)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"a38e15239e9c25fd9d5a7861aabfb823ca088785","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"696c83ef_80f05989","updated":"2021-10-26 15:10:59.000000000","message":"\u003e Patch Set 11: Verified-1\n\u003e \n\u003e Build failed (check pipeline).  For information on how to proceed, see\n\u003e http://docs.openstack.org/infra/manual/developers.html#automated-testing\n\u003e \n\u003e \n\u003e - tripleo-ci-centos-8-ovb-3ctlr_1comp-featureset001 https://review.rdoproject.org/zuul/build/34af321ebabe46958e7eaa1af0df9b48 : FAILURE in 2h 21m 48s\n\u003e - tripleo-ci-centos-8-ovb-3ctlr_1comp_1supp-featureset039 https://review.rdoproject.org/zuul/build/b53cf44a2f444b27a0fd5c2d2b1c1e70 : FAILURE in 3h 13m 24s\n\nunrelated failure as it happens during the introspection phase:\n\n```\nTASK [tripleo.operator.tripleo_overcloud_node_introspect : Introspect node] ****\nTuesday 26 October 2021  13:07:31 +0000 (0:00:00.530)       0:01:58.791 ******* \nfatal: [undercloud]: FAILED! \u003d\u003e {\n    \"changed\": true,\n    \"cmd\": \" openstack overcloud node introspect  --all-manageable --provide      \u003e/home/zuul/overcloud_introspect.log 2\u003e\u00261\",\n    \"delta\": \"1:04:51.417002\",\n    \"end\": \"2021-10-26 14:12:23.653624\",\n    \"rc\": 1,\n    \"start\": \"2021-10-26 13:07:32.236622\"\n}\n```","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"47f8bacf091521a173f3070681f55a258b750020","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"15576a32_cc04e1dd","updated":"2021-10-26 10:28:01.000000000","message":"Tested on a more complex environment, it seems it works: ceph_nfs is correctly upgraded and there are no container leftovers around. ","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"aee97dcde965e957fd9394d6993d68a6062af40a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"0a18f44a_6a70120e","updated":"2021-10-26 11:32:38.000000000","message":"Thanks, Francesco, this looks good to me.  And thanks Luigi for testing it out!","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"466fc7469a21622ae29152c457dba1b3ef504baf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"a93e98ff_f102ae76","updated":"2021-10-28 09:21:41.000000000","message":"check-rdo","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"6f104eacec4ad4b753a2dd7fbd9e9bfb9b3997df","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"c916b250_47111d1a","updated":"2021-10-27 11:52:27.000000000","message":"check-rdo","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"68880f6f33111d98952982b103650452385880d6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"efdbc57b_dcba2e8b","updated":"2021-10-26 15:11:10.000000000","message":"check-rdo","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"change_message_id":"1e7de3f2cf695b56d9a5439d2da9bd998846db97","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"24a56dce_09b7834d","updated":"2021-10-26 12:41:20.000000000","message":"fair enough I guess","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"}],"deployment/ceph-ansible/ceph-nfs.yaml":[{"author":{"_account_id":10459,"name":"Luigi Toscano","email":"ltoscano@redhat.com","username":"ltoscano"},"change_message_id":"14028b68e13a4f3cfda162af6a56ac5f98de225a","unresolved":true,"context_lines":[{"line_number":148,"context_line":"        - {get_attr: [CephBase, role_data, external_upgrade_tasks]}"},{"line_number":149,"context_line":"        - - name: Restart ceph-nfs"},{"line_number":150,"context_line":"            command: pcs resource restart ceph-nfs"},{"line_number":151,"context_line":"            tripleo_delegate_to: \"{{ ceph_nfs_short_bootstrap_node_name }}\""},{"line_number":152,"context_line":"            when: step|int \u003d\u003d 3"},{"line_number":153,"context_line":"      config_settings:"},{"line_number":154,"context_line":"        map_merge:"}],"source_content_type":"text/x-yaml","patch_set":2,"id":"6e6c9a9d_5d167fa3","line":151,"updated":"2021-10-06 07:28:51.000000000","message":"Explicitly tested on a real system I get:\n\n2021-10-05 20:39:18 | ERROR! conflicting action statements: command, tripleo_delegate_to\n2021-10-05 20:39:18 | \n2021-10-05 20:39:18 | The error appears to be in \u0027/var/lib/mistral/a13ece6d-7231-497a-8917-c6e646a5eba3/external_upgrade_steps_tasks.yaml\u0027: line 61, column 3, but may\n2021-10-05 20:39:18 | be elsewhere in the file depending on the exact syntax problem.\n2021-10-05 20:39:18 | \n2021-10-05 20:39:18 | The offending line appears to be:\n2021-10-05 20:39:18 | \n2021-10-05 20:39:18 |   when: step|int \u003d\u003d 0\n2021-10-05 20:39:18 | - command: pcs resource restart ceph-nfs\n2021-10-05 20:39:18 |   ^ here\n2021-10-05 20:39:18 | \n2021-10-05 20:39:18 | Ansible failed, check log at /var/log/containers/mistral/package_update.log.\n\nIn fact tripleo_delegate_to seems to be a variable of the tripleo-container-stop role (the role was renamed tripleo_container_stop in ussuri and later)","commit_id":"ba816031bd99c70c804a5cfb1e5f88c2d8bb2fb2"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"83b4169aecc25cb122ce0fb739c3032012082cbd","unresolved":true,"context_lines":[{"line_number":148,"context_line":"        - {get_attr: [CephBase, role_data, external_upgrade_tasks]}"},{"line_number":149,"context_line":"        - - name: Restart ceph-nfs"},{"line_number":150,"context_line":"            command: pcs resource restart ceph-nfs"},{"line_number":151,"context_line":"            tripleo_delegate_to: \"{{ ceph_nfs_short_bootstrap_node_name }}\""},{"line_number":152,"context_line":"            when: step|int \u003d\u003d 3"},{"line_number":153,"context_line":"      config_settings:"},{"line_number":154,"context_line":"        map_merge:"}],"source_content_type":"text/x-yaml","patch_set":2,"id":"9c1207fb_1ccb879c","line":151,"in_reply_to":"6e6c9a9d_5d167fa3","updated":"2021-10-06 07:58:20.000000000","message":"mmmm ok thanks for looking into it, so delegate_to should be enough, fixing it in the next PS","commit_id":"ba816031bd99c70c804a5cfb1e5f88c2d8bb2fb2"},{"author":{"_account_id":6796,"name":"Giulio Fidente","email":"gfidente@redhat.com","username":"gfidente"},"change_message_id":"3a6fade0dc23100eac674c1d42fe5e6b73796b19","unresolved":true,"context_lines":[{"line_number":156,"context_line":"                  daemon_reload: yes"},{"line_number":157,"context_line":"              - name: restart the ceph-nfs daemon "},{"line_number":158,"context_line":"                command: pcs resource restart ceph-nfs"},{"line_number":159,"context_line":"                delegate_to: \"{{ ceph_nfs_short_bootstrap_node_name }}\""},{"line_number":160,"context_line":"                run_once: true"},{"line_number":161,"context_line":"      config_settings:"},{"line_number":162,"context_line":"        map_merge:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"254ab2b7_501209b9","line":159,"updated":"2021-10-06 10:26:38.000000000","message":"I guess it might be better to delegate this to a pacemaker cluster member, just in case someone is editing the roles trying to move it on some different role?","commit_id":"d109f393718beb922ab688228f70eaab7418d207"},{"author":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"change_message_id":"cef04be58d0788083a2216c0d1d2eaf321ebfbc3","unresolved":true,"context_lines":[{"line_number":156,"context_line":"                  daemon_reload: yes"},{"line_number":157,"context_line":"              - name: restart the ceph-nfs daemon "},{"line_number":158,"context_line":"                command: pcs resource restart ceph-nfs"},{"line_number":159,"context_line":"                delegate_to: \"{{ ceph_nfs_short_bootstrap_node_name }}\""},{"line_number":160,"context_line":"                run_once: true"},{"line_number":161,"context_line":"      config_settings:"},{"line_number":162,"context_line":"        map_merge:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"623bfa20_2d64865b","line":159,"in_reply_to":"1f1f5dd9_be4f6cad","updated":"2021-10-06 12:40:33.000000000","message":"If ceph_nfs is the service name for the ceph-nfs managed by pacemaker, then ceph_nfs_short_bootstrap_node_name is normally the correct name. While in this specific case we probably could use pacemaker_short_bootstrap_node_name, we usually use the first node where the service is configured aka ceph_nfs_short_bootstrap_node_name","commit_id":"d109f393718beb922ab688228f70eaab7418d207"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"0d2841d2d7f384b9d4a63f89704f704faf7d9fb3","unresolved":true,"context_lines":[{"line_number":156,"context_line":"                  daemon_reload: yes"},{"line_number":157,"context_line":"              - name: restart the ceph-nfs daemon "},{"line_number":158,"context_line":"                command: pcs resource restart ceph-nfs"},{"line_number":159,"context_line":"                delegate_to: \"{{ ceph_nfs_short_bootstrap_node_name }}\""},{"line_number":160,"context_line":"                run_once: true"},{"line_number":161,"context_line":"      config_settings:"},{"line_number":162,"context_line":"        map_merge:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"ca4f94e3_b1a1e657","line":159,"in_reply_to":"254ab2b7_501209b9","updated":"2021-10-06 11:24:12.000000000","message":"Right, and in my understanding this variable (which is present in hiera) should contain the node where the daemon is running and that should be used to run the pcs command.\n@michele do you see any issue or is there another way (the right one) to delegate to a pacemaker cluster member?","commit_id":"d109f393718beb922ab688228f70eaab7418d207"},{"author":{"_account_id":6413,"name":"Victoria Martinez de la Cruz","email":"victoria@redhat.com","username":"vkmc"},"change_message_id":"a3da87910d569f64a24bf9049ea27bb4b1966bd0","unresolved":true,"context_lines":[{"line_number":156,"context_line":"                  daemon_reload: yes"},{"line_number":157,"context_line":"              - name: restart the ceph-nfs daemon "},{"line_number":158,"context_line":"                command: pcs resource restart ceph-nfs"},{"line_number":159,"context_line":"                delegate_to: \"{{ ceph_nfs_short_bootstrap_node_name }}\""},{"line_number":160,"context_line":"                run_once: true"},{"line_number":161,"context_line":"      config_settings:"},{"line_number":162,"context_line":"        map_merge:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"a8bcd008_3fe65d33","line":159,"in_reply_to":"623bfa20_2d64865b","updated":"2021-10-07 10:28:49.000000000","message":"A quick search looks like this is correct, the default name for the first node where nfs gnaesha is configured is ceph_nfs_short_bootstrap_node_name https://opendev.org/openstack/puppet-tripleo/src/branch/master/manifests/profile/pacemaker/ceph_nfs.pp#L21-L23. Could this name be overridden by the user and therefore this task fail? Not entirely sure how this works.","commit_id":"d109f393718beb922ab688228f70eaab7418d207"},{"author":{"_account_id":6796,"name":"Giulio Fidente","email":"gfidente@redhat.com","username":"gfidente"},"change_message_id":"b58677aacd746b97e3bca051febf87299a371dab","unresolved":true,"context_lines":[{"line_number":156,"context_line":"                  daemon_reload: yes"},{"line_number":157,"context_line":"              - name: restart the ceph-nfs daemon "},{"line_number":158,"context_line":"                command: pcs resource restart ceph-nfs"},{"line_number":159,"context_line":"                delegate_to: \"{{ ceph_nfs_short_bootstrap_node_name }}\""},{"line_number":160,"context_line":"                run_once: true"},{"line_number":161,"context_line":"      config_settings:"},{"line_number":162,"context_line":"        map_merge:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"1f1f5dd9_be4f6cad","line":159,"in_reply_to":"ca4f94e3_b1a1e657","updated":"2021-10-06 12:31:13.000000000","message":"basically I am trying to make sure the command succeeds when ganesha itself is on some dedicated role and not on controllers","commit_id":"d109f393718beb922ab688228f70eaab7418d207"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"659416d27016a8146213b5b38f11b58e8ce6a6d4","unresolved":true,"context_lines":[{"line_number":142,"context_line":"                copy:"},{"line_number":143,"context_line":"                  dest: \"{{playbook_dir}}/ceph-ansible/group_vars/nfss.yml\""},{"line_number":144,"context_line":"                  content: \"{{ceph_ansible_group_vars_nfss|to_nice_yaml}}\""},{"line_number":145,"context_line":"        - - name: Restart ceph-nfs daemon"},{"line_number":146,"context_line":"            when:"},{"line_number":147,"context_line":"              - step|int \u003d\u003d 3"},{"line_number":148,"context_line":"              - \u0027\"/usr/share/ceph-ansible/infrastructure-playbooks/rolling_update.yml\" in ceph_ansible_playbooks_default\u0027"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"a682479a_fdb3317f","line":145,"range":{"start_line":145,"start_character":18,"end_line":145,"end_character":41},"updated":"2021-10-07 17:09:34.000000000","message":"I need this part in external_deploy_steps, because external_upgrade (with all the steps) is executed before. In addition, this code should be used for minor updates as well (ceph-ansible in any case skips the ceph-nfs restart).","commit_id":"f060eaa287ef6eaabac1c0b4ba420f3a4e2ecfd9"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"659416d27016a8146213b5b38f11b58e8ce6a6d4","unresolved":true,"context_lines":[{"line_number":145,"context_line":"        - - name: Restart ceph-nfs daemon"},{"line_number":146,"context_line":"            when:"},{"line_number":147,"context_line":"              - step|int \u003d\u003d 3"},{"line_number":148,"context_line":"              - \u0027\"/usr/share/ceph-ansible/infrastructure-playbooks/rolling_update.yml\" in ceph_ansible_playbooks_default\u0027"},{"line_number":149,"context_line":"            tags:"},{"line_number":150,"context_line":"              - ceph"},{"line_number":151,"context_line":"            block:"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"f69a3d38_94e7598f","line":148,"range":{"start_line":148,"start_character":14,"end_line":148,"end_character":121},"updated":"2021-10-07 17:09:34.000000000","message":"we can just make sure this is not executed during regular stack updates, but only when an update|upgrade operation is run.","commit_id":"f060eaa287ef6eaabac1c0b4ba420f3a4e2ecfd9"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"659416d27016a8146213b5b38f11b58e8ce6a6d4","unresolved":true,"context_lines":[{"line_number":154,"context_line":"                  daemon_reload: yes"},{"line_number":155,"context_line":"              - name: restart the ceph-nfs daemon"},{"line_number":156,"context_line":"                command: pcs resource restart ceph-nfs"},{"line_number":157,"context_line":"                delegate_to: \u0027{{hostvars[groups[\u0027\u0027overcloud\u0027\u0027][0]][\u0027\u0027ceph_nfs_short_bootstrap_node_name\u0027\u0027]}}\u0027"},{"line_number":158,"context_line":"                run_once: true"},{"line_number":159,"context_line":"      external_update_tasks: {get_attr: [CephBase, role_data, external_update_tasks]}"},{"line_number":160,"context_line":"      external_upgrade_tasks: {get_attr: [CephBase, role_data, external_upgrade_tasks]}"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"02946ebd_02843ea8","line":157,"range":{"start_line":157,"start_character":16,"end_line":157,"end_character":108},"updated":"2021-10-07 17:09:34.000000000","message":"Just using ceph_nfs_short_bootstrap_node_name doesn\u0027t work as this playbook is the external_* one, which behaves differently and we need to access this variable via hostvars.","commit_id":"f060eaa287ef6eaabac1c0b4ba420f3a4e2ecfd9"},{"author":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"change_message_id":"811a2ce7639f9a9c37b061256ea5ff11bff1160f","unresolved":true,"context_lines":[{"line_number":161,"context_line":"                delegate_to: \u0027{{hostvars[groups[\u0027\u0027overcloud\u0027\u0027][0]][\u0027\u0027ceph_nfs_short_bootstrap_node_name\u0027\u0027]}}\u0027"},{"line_number":162,"context_line":"                register: ceph_nfs_id"},{"line_number":163,"context_line":"                run_once: true"},{"line_number":164,"context_line":"              - name: stop ceph3 ceph-nfs containers"},{"line_number":165,"context_line":"                command: \"{{ container_cli }} stop {{ ceph_nfs_id.stdout_lines[0] }}\""},{"line_number":166,"context_line":"                delegate_to: \u0027{{hostvars[groups[\u0027\u0027overcloud\u0027\u0027][0]][\u0027\u0027ceph_nfs_short_bootstrap_node_name\u0027\u0027]}}\u0027"},{"line_number":167,"context_line":"                when: ceph_nfs_id.stdout | length \u003e 0"},{"line_number":168,"context_line":"                run_once: true"},{"line_number":169,"context_line":"              - name: restart the ceph-nfs daemon"}],"source_content_type":"text/x-yaml","patch_set":11,"id":"be8571b2_ae8382ea","line":166,"range":{"start_line":164,"start_character":0,"end_line":166,"end_character":109},"updated":"2021-10-26 11:47:19.000000000","message":"shouldn\u0027t you be using pcs to stop the container here?","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"b785a63288908975cafdc2b218cd3c9d8c8dfce7","unresolved":true,"context_lines":[{"line_number":161,"context_line":"                delegate_to: \u0027{{hostvars[groups[\u0027\u0027overcloud\u0027\u0027][0]][\u0027\u0027ceph_nfs_short_bootstrap_node_name\u0027\u0027]}}\u0027"},{"line_number":162,"context_line":"                register: ceph_nfs_id"},{"line_number":163,"context_line":"                run_once: true"},{"line_number":164,"context_line":"              - name: stop ceph3 ceph-nfs containers"},{"line_number":165,"context_line":"                command: \"{{ container_cli }} stop {{ ceph_nfs_id.stdout_lines[0] }}\""},{"line_number":166,"context_line":"                delegate_to: \u0027{{hostvars[groups[\u0027\u0027overcloud\u0027\u0027][0]][\u0027\u0027ceph_nfs_short_bootstrap_node_name\u0027\u0027]}}\u0027"},{"line_number":167,"context_line":"                when: ceph_nfs_id.stdout | length \u003e 0"},{"line_number":168,"context_line":"                run_once: true"},{"line_number":169,"context_line":"              - name: restart the ceph-nfs daemon"}],"source_content_type":"text/x-yaml","patch_set":11,"id":"15bba2f3_3925450f","line":166,"range":{"start_line":164,"start_character":0,"end_line":166,"end_character":109},"in_reply_to":"be8571b2_ae8382ea","updated":"2021-10-26 11:57:18.000000000","message":"We should, but this container is a leftover from Ceph3.\nThe systemd unit (which is under pcmk control) is updated to Ceph4 and the ExecStop doesn\u0027t work with this container version (and the restart has no effect on this one).\nBy running pcs restart ceph-nfs w/o this stop, the result is having two ceph-nfs containers running (one for ceph3 and one for ceph4).","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"b785a63288908975cafdc2b218cd3c9d8c8dfce7","unresolved":true,"context_lines":[{"line_number":172,"context_line":"                run_once: true"},{"line_number":173,"context_line":"        - - name: Cleanup ceph-nfs daemon"},{"line_number":174,"context_line":"            become: true"},{"line_number":175,"context_line":"            command: \"pcs resource cleanup ceph-nfs\""},{"line_number":176,"context_line":"            delegate_to: \u0027{{hostvars[groups[\u0027\u0027overcloud\u0027\u0027][0]][\u0027\u0027ceph_nfs_short_bootstrap_node_name\u0027\u0027]}}\u0027"},{"line_number":177,"context_line":"            run_once: true"},{"line_number":178,"context_line":"            when:"}],"source_content_type":"text/x-yaml","patch_set":11,"id":"f53277f8_8f8d2661","line":175,"range":{"start_line":175,"start_character":35,"end_line":175,"end_character":42},"updated":"2021-10-26 11:57:18.000000000","message":"I need this because for unknown reasons pcs isn\u0027t happy, the unit is running and we have something like [1] \n\n[1] https://paste.opendev.org/show/810210/","commit_id":"d29e0bfc0f68e31f47b2bada2a6ba5cb80a74bf3"}]}