)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"7f524df46e7a898be550cb9017f7bc92bfc1dd00","unresolved":true,"context_lines":[{"line_number":13,"context_line":"tripleo-client commands to do so."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I982dedb53582fbd76391165c3ca72954c129b84a"},{"line_number":16,"context_line":"Depenends-On: Id6d0699123b9e9b75011ee94949ec4a69012f9af"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":37,"id":"2d8853b1_7e4db48c","line":16,"range":{"start_line":16,"start_character":0,"end_line":16,"end_character":55},"updated":"2022-04-11 11:45:25.000000000","message":"I will remove this depends-on before merging.\nI added it just to see this green with the THT change.\nI\u0027ll then remove it and merge this and then the \"depends on\".","commit_id":"5ea9dc7371b703696222c9172fbcc69c71174e62"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"3bb6c611845c3927c883060bbedd3ace36b4019a","unresolved":false,"context_lines":[{"line_number":13,"context_line":"tripleo-client commands to do so."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I982dedb53582fbd76391165c3ca72954c129b84a"},{"line_number":16,"context_line":"Depenends-On: Id6d0699123b9e9b75011ee94949ec4a69012f9af"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":37,"id":"1ff23334_f8dbb41c","line":16,"range":{"start_line":16,"start_character":0,"end_line":16,"end_character":55},"in_reply_to":"2d8853b1_7e4db48c","updated":"2022-04-11 16:38:54.000000000","message":"Done","commit_id":"5ea9dc7371b703696222c9172fbcc69c71174e62"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"4a6726f6480c2c633303fc58ae50a909782fa9cd","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Add ceph-install playbook to install ceph before overcloud"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Add new playbook to tripleo-quickstart-extras so that CI"},{"line_number":10,"context_line":"can use \"deployed ceph\" [1]. This playbook uses three"},{"line_number":11,"context_line":"tripleo-operator-ansible roles which create a Ceph spec,"},{"line_number":12,"context_line":"create the ceph-admin user, and deploy Ceph by calling"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":38,"id":"e81e1048_f03a4b23","line":9,"range":{"start_line":9,"start_character":8,"end_line":9,"end_character":45},"updated":"2022-04-13 07:31:52.000000000","message":"more like \"Add new tasks file to the standalone role \"","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"dd4321b2b1fa0de9cd5f558da495839ff082283b","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Add ceph-install playbook to install ceph before overcloud"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Add new playbook to tripleo-quickstart-extras so that CI"},{"line_number":10,"context_line":"can use \"deployed ceph\" [1]. This playbook uses three"},{"line_number":11,"context_line":"tripleo-operator-ansible roles which create a Ceph spec,"},{"line_number":12,"context_line":"create the ceph-admin user, and deploy Ceph by calling"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":38,"id":"4c700328_d23dc4fa","line":9,"range":{"start_line":9,"start_character":8,"end_line":9,"end_character":45},"in_reply_to":"e81e1048_f03a4b23","updated":"2022-04-19 16:58:09.000000000","message":"Done","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"4a6726f6480c2c633303fc58ae50a909782fa9cd","unresolved":true,"context_lines":[{"line_number":11,"context_line":"tripleo-operator-ansible roles which create a Ceph spec,"},{"line_number":12,"context_line":"create the ceph-admin user, and deploy Ceph by calling"},{"line_number":13,"context_line":"tripleo-client commands to do so."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I982dedb53582fbd76391165c3ca72954c129b84a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":38,"id":"b7207736_e86e2a13","line":14,"updated":"2022-04-13 07:31:52.000000000","message":"add [1]","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"dd4321b2b1fa0de9cd5f558da495839ff082283b","unresolved":false,"context_lines":[{"line_number":11,"context_line":"tripleo-operator-ansible roles which create a Ceph spec,"},{"line_number":12,"context_line":"create the ceph-admin user, and deploy Ceph by calling"},{"line_number":13,"context_line":"tripleo-client commands to do so."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I982dedb53582fbd76391165c3ca72954c129b84a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":38,"id":"85844029_74bf7578","line":14,"in_reply_to":"b7207736_e86e2a13","updated":"2022-04-19 16:58:09.000000000","message":"Done","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"48ace5c0998b8211164874eee60f817110ad178f","unresolved":true,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Contains a TODO item so that this scenario will also"},{"line_number":24,"context_line":"be tested for W after supporting patches are backported."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Change-Id: I982dedb53582fbd76391165c3ca72954c129b84a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":81,"id":"d42a54c3_3f11924a","line":25,"updated":"2022-06-06 07:07:08.000000000","message":"please consider linking to https://review.opendev.org/c/openstack/tripleo-heat-templates/+/834354 where this is actually wired up for scenario1","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"3a07dac7e9f6fa15e109f9f3f4d6f3e65a09344c","unresolved":true,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Contains a TODO item so that this scenario will also"},{"line_number":24,"context_line":"be tested for W after supporting patches are backported."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Change-Id: I982dedb53582fbd76391165c3ca72954c129b84a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":81,"id":"d4a3fe42_7891e221","line":25,"in_reply_to":"7728b457_031f4866","updated":"2022-06-07 06:55:27.000000000","message":"ok, i checked and saw that is adding \"DeployedCeph: true\" to the ci environment files (and the review commit title is \"Switch scenario001 to Deployed Ceph\") is why I thought it is \u0027wiring it  up\u0027.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"dcb1ace435d6c97588e48f44bc3b9a5a2849875f","unresolved":true,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Contains a TODO item so that this scenario will also"},{"line_number":24,"context_line":"be tested for W after supporting patches are backported."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Change-Id: I982dedb53582fbd76391165c3ca72954c129b84a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":81,"id":"7728b457_031f4866","line":25,"in_reply_to":"d42a54c3_3f11924a","updated":"2022-06-06 11:34:47.000000000","message":"But 834354 doesn\u0027t wire anything. \n\nThis patch deploys ceph and modifies the overcloud deployment to use ceph. Nothing in 834354 is necessary for this patch to work. 834354 is just a clean up to remove parameters that are no longer necessary but are otherwise ignored after this patch merges.\n\n834354 depends on 834352 but not the other way around.\n\nIf 834352 depended on 834354, then we couldn\u0027t merge without a break because 834354 will cause a failure unless 834352 is already in place first.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"e62bf61bee9589cec7d041b4d226bc45fae53bc5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"197443d5_28b2fa33","updated":"2022-03-19 11:21:21.000000000","message":"Initial results of tripleo-ci-centos-9-scenario001-standalone are positive:\n\n- spec created with what looks like a reasonable IP (173.231.255.106) [1]\n- ceph-admin user created with correct initial ceph.conf [2]\n- ansible log [3] captures the above tasks succeeding but then failing\n- bootstrap failure [4] at bottom of ansible log shows reasonable command [5]\n- the failure was is podman registry related\n- to address it I will not skip_container_registry_config in the next patch set [6]\n\n  Error: initializing source docker://173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8: pinging container registry 173.231.255.250:5001: Get \\\"https://173.231.255.250:5001/v2/\\\": http: server gave HTTP response to HTTPS client\\nERROR  \n\n[1] https://8425d46910f579abd66d-f465d94e970ed4b88220a4cc8527bc2b.ssl.cf5.rackcdn.com/834354/1/check/tripleo-ci-centos-9-scenario001-standalone/2227350/logs/undercloud/home/zuul/ceph_spec.yaml\n[2] https://8425d46910f579abd66d-f465d94e970ed4b88220a4cc8527bc2b.ssl.cf5.rackcdn.com/834354/1/check/tripleo-ci-centos-9-scenario001-standalone/2227350/logs/undercloud/home/ceph-admin/assimilate_ceph.conf\n[3] https://8425d46910f579abd66d-f465d94e970ed4b88220a4cc8527bc2b.ssl.cf5.rackcdn.com/834354/1/check/tripleo-ci-centos-9-scenario001-standalone/2227350/logs/undercloud/home/zuul/ansible.log\n\n[4]\n2022-03-18 21:47:30,737 p\u003d19300 u\u003droot n\u003dansible | 2022-03-18 21:47:30.736309 | fa163e9f-328c-ad3a-ee37-00000000006d |      FATAL | Run cephadm bootstrap | standalone.localdomain | error\u003d{\"changed\": true, \"cmd\": \"/usr/sbin/cephadm --image 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 \\\\bootstrap --skip-firewalld --ssh-private-key /home/ceph-admin/.ssh/id_rsa --ssh-public-key /home/ceph-admin/.ssh/id_rsa.pub --ssh-user ceph-admin --allow-fqdn-hostname --output-keyring /etc/ceph/ceph.client.admin.keyring --output-config /etc/ceph/ceph.conf --fsid 56e084ad-f2a5-5cc8-9fc8-798f949107d9 --config /home/ceph-admin/assimilate_ceph.conf \\\\--single-host-defaults \\\\--skip-monitoring-stack --skip-dashboard --log-to-file --skip-prepare-host \\\\--mon-ip 173.231.255.106\\n\", \"delta\": \"0:00:00.583868\", \"end\": \"2022-03-18 21:47:30.708195\", \"msg\": \"non-zero return code\", \"rc\": 1, \"start\": \"2022-03-18 21:47:30.124327\", \"stderr\": \"Skip prepare_host\\nCluster fsid: 56e084ad-f2a5-5cc8-9fc8-798f949107d9\\nVerifying IP 173.231.255.106 port 3300 ...\\nVerifying IP 173.231.255.106 port 6789 ...\\nMon IP `173.231.255.106` is in CIDR network `173.231.255.96/28`\\n- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network\\nAdjusting default settings to suit single-host cluster...\\nPulling container image 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8...\\nNon-zero exit code 125 from /bin/podman pull 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\\n/bin/podman: stderr Trying to pull 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8...\\n/bin/podman: stderr Error: initializing source docker://173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8: pinging container registry 173.231.255.250:5001: Get \\\"https://173.231.255.250:5001/v2/\\\": http: server gave HTTP response to HTTPS client\\nERROR: Failed command: /bin/podman pull 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\", \"stderr_lines\": [\"Skip prepare_host\", \"Cluster fsid: 56e084ad-f2a5-5cc8-9fc8-798f949107d9\", \"Verifying IP 173.231.255.106 port 3300 ...\", \"Verifying IP 173.231.255.106 port 6789 ...\", \"Mon IP `173.231.255.106` is in CIDR network `173.231.255.96/28`\", \"- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network\", \"Adjusting default settings to suit single-host cluster...\", \"Pulling container image 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8...\", \"Non-zero exit code 125 from /bin/podman pull 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\", \"/bin/podman: stderr Trying to pull 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8...\", \"/bin/podman: stderr Error: initializing source docker://173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8: pinging container registry 173.231.255.250:5001: Get \\\"https://173.231.255.250:5001/v2/\\\": http: server gave HTTP response to HTTPS client\", \"ERROR: Failed command: /bin/podman pull 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\"], \"stdout\": \"\", \"stdout_lines\": []}\n\n[5]\n/usr/sbin/cephadm --image 173.231.255.250:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 bootstrap --skip-firewalld --ssh-private-key /home/ceph-admin/.ssh/id_rsa --ssh-public-key /home/ceph-admin/.ssh/id_rsa.pub --ssh-user ceph-admin --allow-fqdn-hostname --output-keyring /etc/ceph/ceph.client.admin.keyring --output-config /etc/ceph/ceph.conf --fsid 56e084ad-f2a5-5cc8-9fc8-798f949107d9 --config /home/ceph-admin/assimilate_ceph.conf --single-host-defaults --skip-monitoring-stack --skip-dashboard --log-to-file --skip-prepare-host --mon-ip 173.231.255.106\n\n[6] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/3/roles/standalone/tasks/ceph-install.yml#71","commit_id":"9db1947cf3760bf448f4c0b123ba069b54facde3"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ba413f387c86dbd21d5d22913ce1ed1dde85041a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"c1dd12f2_6da55c0f","updated":"2022-03-19 13:30:44.000000000","message":"Fails [1] with ERROR: Cannot infer CIDR network for mon IP `158.69.73.214` which is from:\n\nhttps://github.com/ceph/ceph/blob/master/src/cephadm/cephadm#L4448\n\n\n[1]\n FATAL | Run cephadm bootstrap | standalone.localdomain | error\u003d{\\\"changed\\\": true, \\\"cmd\\\": \\\"/usr/sbin/cephadm --image 158.69.71.34:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 \\\\\\\\bootstrap --skip-firewalld --ssh-private-key /home/ceph-admin/.ssh/id_rsa --ssh-public-key /home/ceph-admin/.ssh/id_rsa.pub --ssh-user ceph-admin --allow-fqdn-hostname --output-keyring /etc/ceph/ceph.client.admin.keyring --output-config /etc/ceph/ceph.conf --fsid ae62a15c-9de7-5e99-91ae-ced8617faad4 --config /home/ceph-admin/assimilate_ceph.conf \\\\\\\\--single-host-defaults \\\\\\\\--skip-monitoring-stack --skip-dashboard --log-to-file --skip-prepare-host \\\\\\\\--mon-ip 158.69.73.214\\\\n\\\", \\\"delta\\\": \\\"0:00:00.447858\\\", \\\"end\\\": \\\"2022-03-19 13:18:09.049024\\\", \\\"msg\\\": \\\"non-zero return code\\\", \\\"rc\\\": 1, \\\"start\\\": \\\"2022-03-19 13:18:08.601166\\\", \\\"stderr\\\": \\\"Skip prepare_host\\\\nCluster fsid: ae62a15c-9de7-5e99-91ae-ced8617faad4\\\\nVerifying IP 158.69.73.214 port 3300 ...\\\\nVerifying IP 158.69.73.214 port 6789 ...\\\\nERROR: Cannot infer CIDR network for mon IP `158.69.73.214`: pass --skip-mon-network to configure it later\\\", \\\"stderr_lines\\\": [\\\"Skip prepare_host\\\", \\\"Cluster fsid: ae62a15c-9de7-5e99-91ae-ced8617faad4\\\", \\\"Verifying IP 158.69.73.214 port 3300 ...\\\", \\\"Verifying IP 158.69.73.214 port 6789 ...\\\", \\\"ERROR: Cannot infer CIDR network for mon IP `158.69.73.214`: pass --skip-mon-network to configure it later\\\"], \\\"stdout\\\": \\\"\\\", \\\"stdout_lines\\\": []}\", \"2022-03-19 13:18:09.069884 | fa163ef6-ca21-f3df-791d-00000000006d |     TIMING | tripleo_cephadm : Run cephadm bootstrap | standalone.localdomain | 0:00:10.810751 | 0.73s\", \"\", \"PLAY RECAP *********************************************************************\", \"standalone.localdomain     : ok\u003d12   changed\u003d6    unreachable\u003d0    failed\u003d1    skipped\u003d11   rescued\u003d0    ignored\u003d0   \", \"undercloud                 : ok\u003d3    changed\u003d0    unreachable\u003d0    failed\u003d0    skipped\u003d10   rescued\u003d0    ignored\u003d0   \", \"2022-03-19 13:18:09.073354 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Summary Information ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\", \"2022-03-19 13:18:09.073770 | ~~~~~~~","commit_id":"30f7c2d147d00c2edeb483a592516477209edcc3"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"1f25b7911e0ee0088de8a6414895c12fb59e621f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"f3ed130d_aa77e475","updated":"2022-03-22 21:00:54.000000000","message":"- Failed with \"Get \"https://149.202.162.123:5001/v2/\": http: server gave HTTP response to HTTPS client\" [1].\n\n- I was able to reproduce this in my environment and fix it by updating registries.conf [2]\n\n- The registry conf in this job was not updated [3]\n\n- I wanted the registry to be updated because I set tripleo_ceph_deploy_skip_container_registry_config\u003dfalse in ceph-install.yml\n\n- however, there is no pus_destination\u003dtrue anymore in containers-prepare-parameters.yaml [4]\n\n\n[1] https://f4e87d6e3827a31c5092-da51e0af47ea07ac7d7ede3c335e4573.ssl.cf2.rackcdn.com/834352/11/check/tripleo-ci-centos-9-scenario001-standalone/cb2fccd/logs/undercloud/home/zuul/ansible.log\n\n[2] https://paste.opendev.org/show/b3kqy2aqATjWf0E7AZ1L/\n\n[3] https://f4e87d6e3827a31c5092-da51e0af47ea07ac7d7ede3c335e4573.ssl.cf2.rackcdn.com/834352/11/check/tripleo-ci-centos-9-scenario001-standalone/cb2fccd/logs/undercloud/etc/containers/registries.conf\n\n[4] https://f4e87d6e3827a31c5092-da51e0af47ea07ac7d7ede3c335e4573.ssl.cf2.rackcdn.com/834352/11/check/tripleo-ci-centos-9-scenario001-standalone/cb2fccd/logs/undercloud/home/zuul/containers-prepare-parameters.yaml","commit_id":"495b33db22dc5e1339a677ca9c96d53400e238f5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d0a5733978b2ffcc5988de170ec0b603550dca19","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":14,"id":"1b7b77ff_1720c87e","updated":"2022-03-23 16:07:04.000000000","message":"FATAL | Run cephadm bootstrap failed with a complaint about both a v1/v2 registry.\n\nERROR: Failed command: /bin/podman pull 173.231.255.75:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\\\", \\\"stderr_lines\\\": [\\\"Verifying podman|docker is present...\\\", \\\"Verifying lvm2 is present...\\\", \\\"Verifying time synchronization is in place...\\\", \\\"Unit chronyd.service is enabled and running\\\", \\\"Repeating the final host check...\\\", \\\"podman (/bin/podman) version 4.0.2 is present\\\", \\\"systemctl is present\\\", \\\"lvcreate is present\\\", \\\"Unit chronyd.service is enabled and running\\\", \\\"Host looks OK\\\", \\\"Cluster fsid: 82a55668-da7d-56b0-9b96-a01bc3f819cd\\\", \\\"Verifying IP 173.231.255.106 port 3300 ...\\\", \\\"Verifying IP 173.231.255.106 port 6789 ...\\\", \\\"- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network\\\", \\\"Adjusting default settings to suit single-host cluster...\\\", \\\"Pulling container image 173.231.255.75:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8...\\\", \\\"Non-zero exit code 125 from /bin/podman pull 173.231.255.75:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\\\", \\\"/bin/podman: stderr Error: loading registries configuration \\\\\\\"/etc/containers/registries.conf\\\\\\\": mixing sysregistry v1/v2 is not supported\\\", \\\"ERROR: Failed command: /bin/podman pull 173.231.255.75:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\\\"]","commit_id":"9f0670c2d951d2c8c806a80f5acb3f5290dfee99"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"7aed6ef1550f9aef8a35ca72afd0b7ef53c9a397","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":19,"id":"66109a47_a7cfedb4","updated":"2022-03-24 17:28:21.000000000","message":"- ps18 deployed ceph fine [1] [2]\n- overcloud deployment failure doesn\u0027t look related [3]\n- ps19 should result in tripleo-ci-centos-9-scenario012-standalone not failing\n\n[1] https://5868023f2a04eca9afdb-eb3e5089e1bc48935c47189031f82525.ssl.cf2.rackcdn.com/834352/18/check/tripleo-ci-centos-9-scenario001-standalone/da524d6/logs/undercloud/home/zuul/ansible.log\n\n[2] https://5868023f2a04eca9afdb-eb3e5089e1bc48935c47189031f82525.ssl.cf2.rackcdn.com/834352/18/check/tripleo-ci-centos-9-scenario001-standalone/da524d6/logs/undercloud/home/zuul/deployed_ceph.yaml\n\n[3] https://5868023f2a04eca9afdb-eb3e5089e1bc48935c47189031f82525.ssl.cf2.rackcdn.com/834352/18/check/tripleo-ci-centos-9-scenario001-standalone/da524d6/logs/undercloud/home/zuul/standalone_deploy.log\n","commit_id":"8ce353f211693f76ac80989080d65ecdeb04a2fe"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"caa65046e95c3703aaad7db34bdd0a9e1ca98ce2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"92b35e06_56e287f0","updated":"2022-03-26 16:54:13.000000000","message":"- 012 is green.\n- 001 failed again with the same error [1] \n- Does this have anything to do with deploying ceph in advance?\n- I\u0027ll try running 001 without deploying ceph first in the next ps as an experiment\n\n** Performing Heat stack create.. **\nException: [Errno 16] Device or resource busy: \u0027/home/zuul/tripleo-deploy/heat_launcher/tripleo_deploy-qiu0t7t2\u0027\nTraceback (most recent call last):\n  File \"/usr/lib/python3.9/site-packages/tripleoclient/v1/tripleo_deploy.py\", line 1267, in _standalone_deploy\n    self._deploy_tripleo_heat_templates(orchestration_client,\n  File \"/usr/lib/python3.9/site-packages/tripleoclient/v1/tripleo_deploy.py\", line 808, in _deploy_tripleo_heat_templates\n    stack \u003d orchestration_client.stacks.create(**stack_args)\n  File \"/usr/lib/python3.9/site-packages/heatclient/v1/stacks.py\", line 170, in create\n    resp \u003d self.client.post(\u0027/stacks\u0027,\n  File \"/usr/lib/python3.9/site-packages/heatclient/common/http.py\", line 292, in post\n    return self.client_request(\"POST\", url, **kwargs)\n  File \"/usr/lib/python3.9/site-packages/heatclient/common/http.py\", line 282, in client_request\n    resp, body \u003d self.json_request(method, url, **kwargs)\n  File \"/usr/lib/python3.9/site-packages/heatclient/common/http.py\", line 271, in json_request\n    resp \u003d self._http_request(url, method, **kwargs)\n  File \"/usr/lib/python3.9/site-packages/heatclient/common/http.py\", line 234, in _http_request\n    raise exc.from_response(resp)\nheatclient.exc.HTTPBadRequest: ERROR: HEAT-E99001 Service neutron is not available for resource type OS::TripleO::Network::Ports::ControlPlaneVipPort, reason: neutron network endpoint is not in service catalog.\n\n[1] https://9a6bbf1392a3232530ed-ee3d06e2383dabb0a3bb20a4a4c50a17.ssl.cf5.rackcdn.com/834352/20/check/tripleo-ci-centos-9-scenario001-standalone/df39006/logs/undercloud/home/zuul/standalone_deploy.log","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a13cb496452974d8ee5098119acbdd9016062a00","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"f3c92909_4242a83a","updated":"2022-03-25 21:40:18.000000000","message":"- ceph is happy [1]\n- openstack is not but I don\u0027t know that it\u0027s related [2]\n\n  heatclient.exc.HTTPBadRequest: ERROR: HEAT-E99001 Service neutron is not available for resource type OS::TripleO::Network::Ports::ControlPlaneVipPort, reason: neutron network endpoint is not in service catalog.\n\n\n[1] https://9a6bbf1392a3232530ed-ee3d06e2383dabb0a3bb20a4a4c50a17.ssl.cf5.rackcdn.com/834352/20/check/tripleo-ci-centos-9-scenario001-standalone/df39006/logs/undercloud/home/zuul/ansible.log\n\n[2] https://9a6bbf1392a3232530ed-ee3d06e2383dabb0a3bb20a4a4c50a17.ssl.cf5.rackcdn.com/834352/20/check/tripleo-ci-centos-9-scenario001-standalone/df39006/logs/undercloud/home/zuul/standalone_deploy.log","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"4e12540ea7fe2a1d636a9095865b166212a442c7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"4e476b21_46fdae36","updated":"2022-03-24 19:09:11.000000000","message":"- started bootstrap with --mon-ip 173.231.255.110 fine \n- it ran \u0027ceph ... orch host add standalone.localdomain 173.231.255.110\u0027\n- which failed with \"Can\u0027t communicate with remote host `173.231.255.110`\" [1]\n- I see that /etc/hosts [2] doesn\u0027t have an entry for standalone.localdomain\n- a previous run didn\u0027t have this problem\n- let\u0027s try using 127.0.0.2\n\n[1] http://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_650/834352/19/check/tripleo-ci-centos-9-scenario001-standalone/6500e1e/logs/undercloud/var/log/ceph/cephadm.log\n\n[2] http://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_650/834352/19/check/tripleo-ci-centos-9-scenario001-standalone/6500e1e/logs/undercloud/etc/hosts","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"318b3d876755fe08b1aea17b2b2bdbbf2e396153","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"389a298f_81d14038","updated":"2022-03-25 11:49:16.000000000","message":"001 run skipped because of other failure.\nOther failure from https://bugs.launchpad.net/tripleo/+bug/1966382.\nI\u0027m going to borrow the fix to the above bug in this patch.","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a8cbf80708819737e1ec9ef4208c8760722139ff","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"2c8f8243_db341986","updated":"2022-03-24 22:07:57.000000000","message":"recheck","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"cb6ffe15c6b97922794a0cca63ebfd38871c4311","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"301ac4f0_ef4144ae","updated":"2022-03-25 21:40:39.000000000","message":"recheck","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d1d5562e642225112ec440aaa0318e898cd5fb15","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"cddcae3c_a4052229","updated":"2022-03-24 22:22:27.000000000","message":"recheck","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"81221d3e6d92acadc3f3db23753fe06dbf230688","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"da7a6b7d_6c9c9c29","updated":"2022-03-25 19:30:33.000000000","message":"recheck","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"81221d3e6d92acadc3f3db23753fe06dbf230688","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"f487aea1_cad2003e","in_reply_to":"389a298f_81d14038","updated":"2022-03-25 19:30:33.000000000","message":"https://review.opendev.org/c/openstack/tripleo-quickstart/+/835210","commit_id":"1494a2abc0508841c78ef3d6dfb4ff89bd729dc5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"62cc6fa31f663141961656cf9b25db8a7d4d573c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":21,"id":"8e88793e_89152c16","updated":"2022-03-26 19:02:34.000000000","message":"Why should I get this error? If ceph is deployed before. It looks unrelated.\n\n  https://paste.opendev.org/show/bN1IEkiBred4cv7QtXMx/\n\nDo I need to disable cephadm after if it\u0027s using up too many resources to cause a system overload?\n\n[Errno 16] Device or resource busy: \u0027/home/zuul/tripleo-deploy/heat_launcher/tripleo_deploy-qiu0t7t2\u0027","commit_id":"6152a4b83db33064d2a68adf26b13608051bb09a"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a3516df85b578d44cebe5e061eaed3684aafa2f3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"e71db1de_032d61d8","updated":"2022-03-26 21:17:21.000000000","message":"Now it failed [1] because it cannot ssh to itself and the failure has nothing to do with the change in ps22, it\u0027s identical to ps21 WRT the code which was actually run. Maybe not all VMs from the testing pool are the same?\n\n\"Adding host standalone.localdomain...\", \"Non-zero exit code 22 from /bin/podman run --rm --ipc\u003dhost --stop-signal\u003dSIGTERM --net\u003dhost --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE\u003dquay.io/ceph/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 -e NODE_NAME\u003dstandalone.localdomain -e CEPH_USE_RANDOM_NONCE\u003d1 -v /var/log/ceph/579fa8dc-8e22-575a-9f4f-23a84573041e:/var/log/ceph:z -v /tmp/ceph-tmpto4w8i6j:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpeb3q0s3f:/etc/ceph/ceph.conf:z quay.io/ceph/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 orch host add standalone.localdomain 127.0.0.2\", \"/usr/bin/ceph: stderr Error EINVAL: Can\u0027t communicate with remote host `127.0.0.2`, possibly because python3 is not installed there: cannot send (already closed?)\",\n\n[1] https://22e1778dc1da79f68b69-4b80f87b96cce8897ac2b1f41b9062df.ssl.cf5.rackcdn.com/834352/22/check/tripleo-ci-centos-9-scenario001-standalone/a9c1510/logs/undercloud/home/zuul/ansible.log","commit_id":"e4fe7ad58bbb1eec261417289ca81e35159ccc1b"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"8213f7c90f8b36b88b3d95dbfeae02e3623cc4ca","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"7ed072d1_3b00037a","updated":"2022-03-26 19:32:24.000000000","message":"Patch set 22 disables ceph orch after ceph deployment and before overcloud deployment as an experiment to see if the presence of the orch causes Heat to fail.\n\nIf Heat succeeds then the deployment will get further but fail when standing up RGW because orch is necessary for that. One way to deal with that is to modify scenario 001 so it goes back to RBD only and we rely on 004 to test RGW and Dashboard (which require the orch). We should still be able to create pools if orch is disabled.","commit_id":"e4fe7ad58bbb1eec261417289ca81e35159ccc1b"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"21353578c1d1a0cff7cc8e3763ab6cbc89d06576","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"004823b8_ad2663f7","updated":"2022-03-26 21:17:38.000000000","message":"recheck","commit_id":"e4fe7ad58bbb1eec261417289ca81e35159ccc1b"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"5bc0bce908c20acc4a11d984bc22b65aee1b0e8a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"2b5ddcc8_d714bcf2","updated":"2022-03-28 18:12:09.000000000","message":"\nObservations:\n- Ceph deployment using mon_ip 173.231.255.106 suceeded [1]\n- The [Pause cephadm] play was called [1] with the correct FSID [2]\n- We then see the overcloud deploy fail for reasons seen before [3] and I assume that\u0027s because cephadm was not disabled\n- It looks like pauce cephadm had the opposite effect because \u0027mgr module enable cephadm\u0027 and \u0027orch set backend cephadm\u0027 were called [2]\n- Also, it looks like tripleo_cephadm_action was true because it called the reverse task [4]\n\nChanges in PS24\n- Set \u0027tripleo_ceph_user_enable: false\u0027 as it\u0027s a boolean [5] (not something we write \u0027enable\u0027 or \u0027disable\u0027 in).\n- Set \"tripleo_ceph_user_generate_scripts: true\" for \"Disable cephadm and ceph-admin user\" so we can see the command which was run.\n\n\n[1] https://77160b63d873e76a0b97-a314ed1e81fb229dd8553fdd3b9ca80f.ssl.cf2.rackcdn.com/834352/23/check/tripleo-ci-centos-9-scenario001-standalone/921d9c7/logs/undercloud/home/zuul/ansible.log\n\n[2] https://77160b63d873e76a0b97-a314ed1e81fb229dd8553fdd3b9ca80f.ssl.cf2.rackcdn.com/834352/23/check/tripleo-ci-centos-9-scenario001-standalone/921d9c7/logs/undercloud/home/zuul/deployed_ceph.yaml\n\n[3] https://77160b63d873e76a0b97-a314ed1e81fb229dd8553fdd3b9ca80f.ssl.cf2.rackcdn.com/834352/23/check/tripleo-ci-centos-9-scenario001-standalone/921d9c7/logs/undercloud/home/zuul/standalone_deploy.log\n\n[4] https://review.opendev.org/c/openstack/tripleo-ansible/+/820449/5/tripleo_ansible/roles/tripleo_cephadm/tasks/toggle_cephadm.yml#33\n\n[5] https://github.com/openstack/tripleo-operator-ansible/blob/253618d00640d3da61b65fb298e66790a7c036ae/roles/tripleo_ceph_user/README.md#role-variables\n","commit_id":"7a5d33b095e5f92a09a24d722566cabb244ddfe9"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"78caceda1dc5b2a02734ebc516eac5324378b074","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"9d5dd73b_7054527b","updated":"2022-03-28 20:55:52.000000000","message":"- ceph deployed successfully [1]\n- ceph disable script actually has \u0027openstack overcloud ceph user disable\u0027 as it should [2]\n- as per the console it failed because it needs to pass \u0027--yes\u0027 [3]\n- the role does not contain a --yes option [4] though it will always be needed for disable\n\n[1] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_22a/834352/24/check/tripleo-ci-centos-9-scenario001-standalone/22a2b2b/logs/undercloud/home/zuul/ansible.log\n\n[2] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_22a/834352/24/check/tripleo-ci-centos-9-scenario001-standalone/22a2b2b/logs/undercloud/home/zuul/openstack_overcloud_ceph_user.sh\n\n[3]\n2022-03-28 20:34:36.703118 | primary | fatal: [undercloud]: FAILED! \u003d\u003e {\"ansible_job_id\": \"330767849788.27523\", \"changed\": false, \"cmd\": \"openstack overcloud ceph user disable /home/zuul/ceph_spec.yaml --fsid $FSID --standalone --cephadm-ssh-user $CEPHADM_SSH_USER --stack $DEPLOY_STACK \", \"delta\": \"0:00:02.208693\", \"end\": \"2022-03-28 20:34:27.261661\", \"finished\": 1, \"msg\": \"non-zero return code\", \"rc\": 1, \"start\": \"2022-03-28 20:34:25.052968\", \"stderr\": \"/usr/lib/python3.9/site-packages/ansible/_vendor/__init__.py:42: UserWarning: One or more Python packages bundled by this ansible-core distribution were already loaded (pyparsing). This may result in undefined behavior.\\n  warnings.warn(\u0027One or more Python packages bundled by this ansible-core distribution were already \u0027\\nUser interaction required, cannot confirm.\\nWill not disable cephadm and delete the cephadm SSH user : ceph-admin. See the --yes parameter to override this behavior. \", \"stderr_lines\": [\"/usr/lib/python3.9/site-packages/ansible/_vendor/__init__.py:42: UserWarning: One or more Python packages bundled by this ansible-core distribution were already loaded (pyparsing). This may result in undefined behavior.\", \"  warnings.warn(\u0027One or more Python packages bundled by this ansible-core distribution were already \u0027\", \"User interaction required, cannot confirm.\", \"Will not disable cephadm and delete the cephadm SSH user : ceph-admin. See the --yes parameter to override this behavior. \"], \"stdout\": \"\", \"stdout_lines\": []}\n\n[4] https://github.com/openstack/tripleo-operator-ansible/blob/253618d00640d3da61b65fb298e66790a7c036ae/roles/tripleo_ceph_user/tasks/main.yml#L2","commit_id":"7a5d33b095e5f92a09a24d722566cabb244ddfe9"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"21bb51077b7570196ecaf199de71468220ce00f9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":25,"id":"b34be2cd_7a8e0e2c","updated":"2022-03-29 12:16:11.000000000","message":"- Another successful ceph deploy with mon_ip 173.231.255.246 [0]\n- I confirm the disable is present as per the openstack_overcloud_ceph_user.sh script [1]\n- A depends-on [2] does not use the updated ansible-operator. I think we\u0027ll have to merge that before proceeding.\n\n[0] https://14b110eee126c805f37a-8605f807f0b1cdc197533279c15edd0d.ssl.cf5.rackcdn.com/834352/25/check/tripleo-ci-centos-9-scenario001-standalone/2d8c555/logs/undercloud/home/zuul/ansible.log\n\n[1] https://14b110eee126c805f37a-8605f807f0b1cdc197533279c15edd0d.ssl.cf5.rackcdn.com/834352/25/check/tripleo-ci-centos-9-scenario001-standalone/2d8c555/logs/undercloud/home/zuul/openstack_overcloud_ceph_user.sh\n\n[2] https://review.opendev.org/c/openstack/tripleo-operator-ansible/+/835520 ","commit_id":"3ce5306e90c3a24b2d26ebfea717d9c9299c1352"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"00f2bc4e3068e12672e9bc811930e49c6bf11c4b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"61267bb3_3107de04","updated":"2022-03-29 17:35:25.000000000","message":"- ceph deployed successfully with mon_ip 158.69.73.163 [1]\n- cephadm disabled (\u0027mgr module disable cephadm\u0027 + \u0027remove ssh keys\u0027) without failure [1]\n- heat still failed, though \n\nERROR: HEAT-E99001 Service neutron is not available for resource type OS::TripleO::Network::Ports::ControlPlaneVipPort, reason: neutron network endpoint is not in service catalog.\n\n[1] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_080/834352/26/check/tripleo-ci-centos-9-scenario001-standalone/0801042/logs/undercloud/home/zuul/ansible.log\n\n[2] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_080/834352/26/check/tripleo-ci-centos-9-scenario001-standalone/0801042/logs/undercloud/home/zuul/standalone_deploy.log","commit_id":"56c33fd27f2a3fb47049cb1d7ac5626bbfec7365"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d5dd26fc14895d8ee2630e1c81045372a017bd7a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"a4a5bac2_b1136d66","updated":"2022-03-29 17:44:14.000000000","message":"- Because I\u0027m getting a Heat error I\u0027ll remove my change to the paramters passed to Heat [1] to see if it makes any difference (if it does that would mean I added my heat env file incorrectly).\n- If so, then I expect to see ceph deploy, then openstack will start deploying though it will fail at the point of configuring ceph for openstack usage because it won\u0027t have the deployed_ceph.yaml file\n- Awaiting results\n\n[1] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/26/roles/standalone/tasks/standalone-install.yml","commit_id":"8f164ff0c6f17c63811e59588375e7945d1b4668"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"e76f624473ee261266ecc8cdff72b42b5accca52","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"71ee2afa_c529ef29","in_reply_to":"81df4461_c519c335","updated":"2022-04-11 16:39:45.000000000","message":"Ack","commit_id":"8f164ff0c6f17c63811e59588375e7945d1b4668"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d4ac2221d9058913090946ac651955732e85568d","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":27,"id":"81df4461_c519c335","in_reply_to":"a4a5bac2_b1136d66","updated":"2022-03-30 13:18:02.000000000","message":"\u003e - Because I\u0027m getting a Heat error I\u0027ll remove my change to the paramters passed to Heat [1] to see if it makes any difference (if it does that would mean I added my heat env file incorrectly).\n\u003e - If so, then I expect to see ceph deploy, then openstack will start deploying though it will fail at the point of configuring ceph for openstack usage because it won\u0027t have the deployed_ceph.yaml file\n\u003e - Awaiting results\n\u003e \n\u003e [1] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/26/roles/standalone/tasks/standalone-install.yml\n\nResults arrived as expected. So I need to change how I\u0027m passing my deployed_ceph.yaml env file.\n\nhttps://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c9a/834352/27/check/tripleo-ci-centos-9-scenario001-standalone/c9a53af/logs/undercloud/home/zuul/standalone_deploy.log","commit_id":"8f164ff0c6f17c63811e59588375e7945d1b4668"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"427e08a083ccd44f93a94ece5f9c8733fa92881e","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":28,"id":"5e2c41d3_ec93b174","updated":"2022-03-30 15:07:33.000000000","message":"- I used [1] to make Heat not fail; thus I need to find a way to \"-e deployed_ceph.yaml\" correctly\n- If I use [2] I think I can avoid the problem with [3] where it fails on [4]\n- The failure from [4] will be resolved if I \"-e deployed_ceph.yaml\" correctly since it will have \"DeployedCeph: true\" [5]\n- So we\u0027re now awaiting results of [2] to get a different outcome than [3]\n\n[1] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/26..27/roles/standalone/tasks/standalone-install.yml\n\n[2] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/27..28/roles/standalone/tasks/ceph-install.yml\n\n[3] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c9a/834352/27/check/tripleo-ci-centos-9-scenario001-standalone/c9a53af/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-yn7ab5ka/cephadm/cephadm_command.log\n\n[4] https://github.com/openstack/tripleo-ansible/blob/master/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml#L41\n\n[5] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c9a/834352/27/check/tripleo-ci-centos-9-scenario001-standalone/c9a53af/logs/undercloud/home/zuul/deployed_ceph.yaml","commit_id":"03be5cbc46708c4f18bd650327804e69be9c266a"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"35135c77037e8a01bf4b93c41a95603fb30ca790","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":28,"id":"b761b8d2_0cb8cf75","updated":"2022-03-30 19:00:34.000000000","message":"- We now have a good ceph deployment [1] and a completed openstack deployment [2]\n- cephadm ran during overcloud deployment [3] and it didn\u0027t bootstrap the ceph cluster as it\u0027s idempotent. It also created pools.\n- It also generated ceph_client.yml [4] though it has the wrong FSID; likely because the deployed_ceph.yaml file was not passed [5]\n- The tempest results failed, but this could be explained by ceph clients which are unable to connect to the ceph cluster from not having had the deployed_ceph.yaml file directly passed.\n- The next step should be to focus on passing the deployed_ceph.yaml file correctly\n\n[1] https://212159587b4db1f2f67b-d413fbe3672f31435774ef2983026d41.ssl.cf2.rackcdn.com/834352/28/check/tripleo-ci-centos-9-scenario001-standalone/38fd2ed/logs/undercloud/home/zuul/ansible.log\n\n[2] https://212159587b4db1f2f67b-d413fbe3672f31435774ef2983026d41.ssl.cf2.rackcdn.com/834352/28/check/tripleo-ci-centos-9-scenario001-standalone/38fd2ed/logs/undercloud/home/zuul/standalone_deploy.log\n\n[3] https://212159587b4db1f2f67b-d413fbe3672f31435774ef2983026d41.ssl.cf2.rackcdn.com/834352/28/check/tripleo-ci-centos-9-scenario001-standalone/38fd2ed/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-6ffvfq_6/cephadm/cephadm_command.log\n\n[4] https://212159587b4db1f2f67b-d413fbe3672f31435774ef2983026d41.ssl.cf2.rackcdn.com/834352/28/check/tripleo-ci-centos-9-scenario001-standalone/38fd2ed/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-6ffvfq_6/cephadm/ceph_client.yml\n\n[5] https://212159587b4db1f2f67b-d413fbe3672f31435774ef2983026d41.ssl.cf2.rackcdn.com/834352/28/check/tripleo-ci-centos-9-scenario001-standalone/38fd2ed/logs/undercloud/home/zuul/deployed_ceph.yaml","commit_id":"03be5cbc46708c4f18bd650327804e69be9c266a"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"e76f624473ee261266ecc8cdff72b42b5accca52","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"aec761ef_ae354dcc","in_reply_to":"5e2c41d3_ec93b174","updated":"2022-04-11 16:39:45.000000000","message":"Ack","commit_id":"03be5cbc46708c4f18bd650327804e69be9c266a"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"e76f624473ee261266ecc8cdff72b42b5accca52","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"a7f1db94_fecf3df9","in_reply_to":"b761b8d2_0cb8cf75","updated":"2022-04-11 16:39:45.000000000","message":"Ack","commit_id":"03be5cbc46708c4f18bd650327804e69be9c266a"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"dd5c4061580b186dda95706fb7950b0a7dc41643","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"e81698be_75ef3df9","updated":"2022-03-31 12:00:25.000000000","message":"It\u0027s the intermittent failure again [1]:\n\norch host add standalone.localdomain 173.231.255.110\", \"/usr/bin/ceph: stderr Error EINVAL: Can\u0027t communicate with remote host `173.231.255.110`, possibly because python3 is not installed there: cannot send (already closed?)\", \"ERROR: Failed to add host \u003cstandalone.localdomain\u003e\n\n[1] https://acdcea17c56a1c58d317-d1707fdfeb7c3818749104e3238e1015.ssl.cf1.rackcdn.com/834352/29/check/tripleo-ci-centos-9-scenario001-standalone/0df0858/logs/undercloud/home/zuul/ansible.log","commit_id":"80da2ebac69d81bd83bfc6bc8f3de789c035da2c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"06f0b1eeef88e13a165359f66b3a06875778595e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"c1f68570_02ea1a5b","updated":"2022-03-31 12:00:31.000000000","message":"recheck","commit_id":"80da2ebac69d81bd83bfc6bc8f3de789c035da2c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"662a850eaa0874b8162c09379ec0bc7db87b3b93","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":32,"id":"181e6af7_3343679e","updated":"2022-04-04 18:51:49.000000000","message":"- ceph and overcloud deploy succeeded but tempest failed\n- Why couldn\u0027t I append  env files list https://paste.opendev.org/show/bCIOipY4411S33f7VBpK ? \n","commit_id":"f0d1d14ff039c0919f483c6e6485deef1cb8bbca"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"39157547af8e4775e67b1cdfae32c691d242fd39","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":32,"id":"a64fb542_6a99e22d","updated":"2022-04-04 15:16:33.000000000","message":"recheck","commit_id":"f0d1d14ff039c0919f483c6e6485deef1cb8bbca"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"2a46d911f9722daef6b8c649222c7157ec02e4b0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":33,"id":"d2b4c7bc_309796aa","updated":"2022-04-07 18:17:46.000000000","message":"We\u0027re green!\n\n- ceph deploy successful [1]\n- overcloud deploy successful [2]\n- tempest results green [3]\n\nI now want to update this patch to not use the following:\n\n  tripleo_ceph_deploy_container_namespace: \"quay.io/ceph\"\n\nand instead use a local mirror described in [4]. For example:\n\n  173.231.254.237:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\n\nAs per [5], if [4] has push_destination\u003dtrue, then /etc/hosts and\n/etc/containers/registries.conf would be updated to accept the above.\n\n[1]  https://584d0f69a24cd7ce1d78-bd247f7438880c13edd50dffc9eda9b5.ssl.cf5.rackcdn.com/834352/33/check/tripleo-ci-centos-9-scenario001-standalone/ba95324/logs/undercloud/home/zuul/ansible.log\n[2]  https://584d0f69a24cd7ce1d78-bd247f7438880c13edd50dffc9eda9b5.ssl.cf5.rackcdn.com/834352/33/check/tripleo-ci-centos-9-scenario001-standalone/ba95324/logs/undercloud/home/zuul/standalone_deploy.log\n[3] https://584d0f69a24cd7ce1d78-bd247f7438880c13edd50dffc9eda9b5.ssl.cf5.rackcdn.com/834352/33/check/tripleo-ci-centos-9-scenario001-standalone/ba95324/logs/undercloud/home/zuul/tempest/tempest.log\n[4]  https://584d0f69a24cd7ce1d78-bd247f7438880c13edd50dffc9eda9b5.ssl.cf5.rackcdn.com/834352/33/check/tripleo-ci-centos-9-scenario001-standalone/ba95324/logs/undercloud/home/zuul/containers-prepare-parameters.yaml\n[5] https://review.opendev.org/c/openstack/python-tripleoclient/+/825416\n","commit_id":"03eb21851ed427eaaa0493a789b0454d95bf1b9c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"679d7d0cd818ccf44aec643ab5b1d86cd32f9757","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":34,"id":"80ad3ff4_a90a90c8","updated":"2022-04-07 20:23:12.000000000","message":"After adding a task to accept update the registries.conf [0] with the following:\n\n[registries.insecure]\nregistries \u003d [\u0027158.69.70.88:5001\u0027]\n\nThe deployment failed [1] because mixing sysregistry v1/v2 is not supported:\n\n\n\"Pulling container image 158.69.70.88:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8...\", \"Non-zero exit code 125 from /bin/podman pull 158.69.70.88:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\", \"/bin/podman: stderr Error: loading registries configuration \\\"/etc/containers/registries.conf\\\": mixing sysregistry v1/v2 is not supported\", \"ERROR: Failed command: /bin/podman pull 158.69.70.88:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8\"], \"stdout\": \"\", \"stdout_lines\": []}\n\nMy task to update registries.conf will need to be modified to only use v2 format.\n\n\n[0] https://fcc5e9557a503f1bb645-af919829f86b35e971243ef9572b928e.ssl.cf1.rackcdn.com/834352/34/check/tripleo-ci-centos-9-scenario001-standalone/fbee54b/logs/undercloud/etc/containers/registries.conf\n\n[1] https://fcc5e9557a503f1bb645-af919829f86b35e971243ef9572b928e.ssl.cf1.rackcdn.com/834352/34/check/tripleo-ci-centos-9-scenario001-standalone/fbee54b/logs/undercloud/home/zuul/ansible.log\n\n","commit_id":"4026684ce5a38eff08933a10a32d5329056e924b"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"cd3f34ec34767facab0b72cef9cc879aeb768f8d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":34,"id":"ddf18879_74acfcdb","updated":"2022-04-07 19:21:36.000000000","message":"Cool [1] !\n\n[1] https://584d0f69a24cd7ce1d78-bd247f7438880c13edd50dffc9eda9b5.ssl.cf5.rackcdn.com/834352/33/check/tripleo-ci-centos-9-scenario001-standalone/ba95324/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-mme4hv3b/cephadm/cephadm_command.log","commit_id":"4026684ce5a38eff08933a10a32d5329056e924b"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ab35f116f7e0af9d328e71e7c8e99d0ec6be231e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":36,"id":"59a2beb1_1031d428","updated":"2022-04-08 13:19:44.000000000","message":"As per results [1], updated pull from local mirror worked:\n\n\"Pulling container image 158.69.64.78:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8...\", \"Ceph version: ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)\", \"Extracting ceph user uid/gid from container image...\", \"Creating initial keys...\", \"Creating initial monmap...\", \"Creating mon...\", \"Waiting for mon to start...\", \"Waiting for mon...\", \"mon is available\", \"Assimilating anything we can from ceph.conf...\",\n\nfailed for old reason...\n\n\"ERROR: Failed to add host \u003cstandalone.localdomain\u003e: Failed command: /bin/podman run --rm --ipc\u003dhost --stop-signal\u003dSIGTERM --net\u003dhost --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE\u003d158.69.64.78:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 -e NODE_NAME\u003dstandalone.localdomain -e CEPH_USE_RANDOM_NONCE\u003d1 -v /var/log/ceph/0e606aff-9b0f-56fd-981c-bced78d05e44:/var/log/ceph:z -v /tmp/ceph-tmpwmuvufo5:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpjduqqdok:/etc/ceph/ceph.conf:z 158.69.64.78:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 orch host add standalone.localdomain 158.69.75.196\"], \"stdout\": \"\", \"stdout_lines\": []}\n\nit\u0027s intermittent so I\u0027ll try again\n\n[1]  https://24eade5565127d985eb0-7e6feee1594781d3a430e22d861f8db7.ssl.cf2.rackcdn.com/834352/36/check/tripleo-ci-centos-9-scenario001-standalone/b9f9cb0/logs/undercloud/home/zuul/ansible.log","commit_id":"531f3bd2c6b08982391ef5810f8808180c655c49"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d4d54e326ef958944fbd036a2f3378dc6d27fdfc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":36,"id":"06ca550e_095ac8b9","updated":"2022-04-08 13:19:56.000000000","message":"recheck","commit_id":"531f3bd2c6b08982391ef5810f8808180c655c49"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a72e93f452752083d0d5bd3062cc741c9db18c75","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":36,"id":"44a8360d_cf0a049c","updated":"2022-04-08 14:33:58.000000000","message":"recheck","commit_id":"531f3bd2c6b08982391ef5810f8808180c655c49"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"8950af11d39dc96bbf2f3e60eda16fc8616dfb18","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":37,"id":"1ce0f378_b6a39d72","updated":"2022-04-11 16:34:29.000000000","message":"So if that heat patch goes through we\u0027ll be fine:\n\ntripleo-ci-centos-9-scenario001-standalone https://zuul.opendev.org/t/openstack/build/70d2a48bdfb84314a3a8638e2c3ab430 : SUCCESS in 1h 35m 14s\n\nI\u0027m going to set this up for review now with the dependency going the other way.\n\n","commit_id":"5ea9dc7371b703696222c9172fbcc69c71174e62"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"80c1bd9e6e4f3bdfab6b4e9735d8efabd5034a24","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"5cbca46f_a26d08cf","updated":"2022-04-12 12:55:18.000000000","message":"Failure with tripleo-ci-centos-9-standalone-on-multinode-ipa is unrelated and fixed by:\n\nhttps://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/837410\n\nWhat\u0027s relevant here is that 001 passed again.","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"4a6726f6480c2c633303fc58ae50a909782fa9cd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"c79e6b82_fa3baf97","updated":"2022-04-13 07:31:52.000000000","message":"c","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"54ff04bdafc089820de670a033c554b26ce43303","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"93eed80a_3591789a","updated":"2022-04-21 03:04:14.000000000","message":"If we merge this as is, it will change all CI jobs where \"standalone_ceph: true\". That\u0027s standalone 001, 004, 010\n\nWe should also be sure of the branches too. For now it needs to just be master since standalone deployed ceph needs to be backported to W. Then we should go back no further. \n\nPerhaps this needs an update to use a new depends on to a tripleo CI patch which has a new job parameter \"deployed_ceph\" which is like standalone_ceph but separate and initially only for 001. Then when 001 is merged and stable I will expand to other scenarios and branches in follow up patches. ","commit_id":"332bbe02b366651e6b0da9ea6a5247069469292c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"5a04a596f53fa24ef0483f165dc9a6835eb41d50","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"f09bb739_bfcb331c","updated":"2022-04-20 21:08:03.000000000","message":"recheck","commit_id":"332bbe02b366651e6b0da9ea6a5247069469292c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ebf5eddd3e88d13529ed0cdd546181f8a868c592","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"a3e9ca19_262c2f73","updated":"2022-04-21 02:50:00.000000000","message":"recheck ","commit_id":"332bbe02b366651e6b0da9ea6a5247069469292c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"5a32647f1369cc9244540f8db379e405dff972da","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"dff20634_787adae9","in_reply_to":"93eed80a_3591789a","updated":"2022-04-21 13:41:20.000000000","message":"Using this to try to limit the scope\n\nhttps://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/43/roles/standalone/tasks/main.yml#6","commit_id":"332bbe02b366651e6b0da9ea6a5247069469292c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"055301c7b53dd880457e66642520d0a6936033db","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":44,"id":"f317e47f_271c100a","updated":"2022-04-21 20:47:26.000000000","message":"- tripleo-ci-centos-9-scenario001-standalone is green [1]\n- the failure in tripleo-ci-centos-9-standalone-on-multinode-ipa [2] is not be related\n- are we ready to merge 001-standalone and master for now?\n\n[1] https://zuul.opendev.org/t/openstack/build/80891b1e3cf34303a94883e132ee9400/artifacts\n[2] https://zuul.opendev.org/t/openstack/build/9f2f57dbbfe04ab7b91675b8c92e50b4","commit_id":"f3a137a12b4b29d4b6b45cb7d365b861fbfcbd42"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"209edad944087a42f8e5abeeea950caa9f5eb304","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":44,"id":"d906f881_2acd5806","updated":"2022-04-21 14:25:41.000000000","message":"@Marios,\n\nI agree about using standalone_ceph. For now I added a scenario check only because I want to start with 001. However, I\u0027ll remove the scenario check as per the TODO when 004 and 010 are ready.\n\nShould I be limiting the branches to be tested in another way?","commit_id":"f3a137a12b4b29d4b6b45cb7d365b861fbfcbd42"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"f9c82788c2f922dfad0fe44286ef166d71ded540","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":44,"id":"ec0376c4_5bc68374","updated":"2022-04-22 17:07:18.000000000","message":"recheck","commit_id":"f3a137a12b4b29d4b6b45cb7d365b861fbfcbd42"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a97eed4bf7d420f3680c186ae43d9e1ce709fca2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":46,"id":"bce11b1b_87d4007f","updated":"2022-04-26 15:58:49.000000000","message":"- tripleo-ci-centos-9-scenario001-standalone succeeded [1]\n- tripleo-ci-centos-9-containers-multinode-wallaby failed for tempest [2] but this patch is only for standalone\n- tripleo-ci-centos-9-standalone-on-multinode-ipa failed [3] for RPM installation failure, unreleated [3]\n\n\n\n[1] https://zuul.opendev.org/t/openstack/build/717323b8376e40aabfe340160260aacc \n[2] https://zuul.opendev.org/t/openstack/build/ad07b9e982ef4313bd493d042b321637 \n[3] https://zuul.opendev.org/t/openstack/build/ac636f2b724b4f95a8692866b98f29b0","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a5783de8b1dbdf55914a5f6294bcba81292213dc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":46,"id":"d52c203c_c7accedb","updated":"2022-04-26 15:59:29.000000000","message":"recheck","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"9c1f3723febf5141a31bd7081a0dc80223346411","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":47,"id":"773c9149_4d4d3868","updated":"2022-04-27 15:41:01.000000000","message":"- 001 and 004 are green but this PS is not doing what we need it to\n- when deployed ceph works there\u0027s a deployed_ceph.yaml file in logs/undercloud/home/zuul [1]\n- there would also be openstack_overcloud_ceph_deploy.sh, openstack_overcloud_ceph_spec.sh files [1]\n- However, this latest run of 001 does not have those files [2]\n- I suspect this condition [3] is failing silently and ceph is getting deployed during overcloud which makes the job green.\n- I need to update this patch to get the values from standalone_ceph and zuul.branch correctly\n\n[1]  https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_22a/834352/24/check/tripleo-ci-centos-9-scenario001-standalone/22a2b2b/logs/undercloud/home/zuul/\n\n[2] https://693ae3b551562860f1e3-7b446e9d0ed6b9dfc63ac80c2f10e41b.ssl.cf2.rackcdn.com/834352/47/check/tripleo-ci-centos-9-scenario001-standalone/c6f7c99/logs/undercloud/home/zuul/index.html\n\n[3]\n    - standalone_ceph is defined and standalone_ceph\n    - zuul.branch is defined and zuul.branch in [\u0027master\u0027]\n","commit_id":"b9806cfd55accc471a0eb4f9314b9e7e7af3b2c7"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"66366e794002fc3ac6308d5508c10d16cf213868","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":51,"id":"3f65015c_417905b4","updated":"2022-05-02 19:14:59.000000000","message":"As per the run [1] of the latest PS we see that neither of the following were true [2]:\n\n  when: standalone_ceph is defined and standalone_ceph\n  when: standalone_ceph | default(false)\n\nThe only debug task which was run the one to save all vars which we have [3].\n\nIt contains eight instances of the string \"standalone_ceph: true\". I think the issue is accessing it correctly.\n\nAnyone should be able to save a copy of the file [3] and have python access the value of the variable:\n\n  [fultonj@runcible ansible]$ md5sum debug_vars.yaml\n  e69707853c6d182d81b8fd29cfb374cb  debug_vars.yaml\n  [fultonj@runcible ansible]$ python3\n  Python 3.10.4 (main, Mar 25 2022, 00:00:00) [GCC 11.2.1 20220127 (Red Hat 11.2.1-9)] on linux\n  Type \"help\", \"copyright\", \"credits\" or \"license\" for more information.\n  \u003e\u003e\u003e stream \u003d open(\"/home/fultonj/ansible/debug_vars.yaml\")\n  \u003e\u003e\u003e import yaml\n  \u003e\u003e\u003e d \u003d yaml.load(stream)\n  \u003cstdin\u003e:1: YAMLLoadWarning: calling yaml.load() without Loader\u003d... is deprecated, as the default \n  Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n  \u003e\u003e\u003e d[\u0027hostvars\u0027][\u0027undercloud\u0027][\u0027job\u0027][\u0027standalone_ceph\u0027]\n  True\n  \u003e\u003e\u003e \n\nSo do I need to use some alternative syntax for accessing this variable like I do the IP address?\n\n  \"{{ hostvars[inventory_hostname][\u0027ansible_default_ipv4\u0027][\u0027address\u0027] }}\"\n\n\n\n[1] https://e6a07a277775a85fd383-2c7741880943732ccc10651ec71fbf9c.ssl.cf2.rackcdn.com/834352/51/check/tripleo-ci-centos-9-scenario001-standalone/9de193c/logs/quickstart_install.log\n\n[2]\n~~~\nTASK [standalone : Save all vars to a file] ************************************\nFriday 29 April 2022  13:42:28 +0000 (0:00:00.046)       0:07:46.277 ********** \nchanged: [undercloud]\n\nTASK [standalone : Print if zuul.branch is defined] ****************************\nFriday 29 April 2022  13:42:35 +0000 (0:00:06.511)       0:07:52.789 ********** \nok: [undercloud] \u003d\u003e {\n    \"msg\": \"zuul.branch: master\"\n}\n\nTASK [standalone : Print if standalone_ceph is defined and standalone_ceph] ****\nFriday 29 April 2022  13:42:35 +0000 (0:00:00.056)       0:07:52.846 ********** \nskipping: [undercloud]\n\nTASK [standalone : Print if standalone_ceph|default(false)] ********************\nFriday 29 April 2022  13:42:35 +0000 (0:00:00.045)       0:07:52.891 ********** \nskipping: [undercloud]\n\nTASK [standalone : include_tasks] **********************************************\nFriday 29 April 2022  13:42:35 +0000 (0:00:00.052)       0:07:52.943 ********** \nskipping: [undercloud]\n\nTASK [standalone : include_tasks] **********************************************\nFriday 29 April 2022  13:42:35 +0000 (0:00:00.064)       0:07:53.008 ********** \nincluded: /home/zuul/workspace/.quickstart/usr/local/share/ansible/roles/standalone/tasks/standalone-install.yml for undercloud\n~~~\n\n[3] https://e6a07a277775a85fd383-2c7741880943732ccc10651ec71fbf9c.ssl.cf2.rackcdn.com/834352/51/check/tripleo-ci-centos-9-scenario001-standalone/9de193c/logs/undercloud/home/zuul/debug_vars.yaml\n","commit_id":"ffa445bbe99719d3d70a0f73e7bb77143fcd2cc0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"484275b12926a1c77474af0760996ff86df8558e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":54,"id":"2a2579af_9fca144c","updated":"2022-05-03 11:14:35.000000000","message":"It looks like either of the following will work:\n\n- name: hostvars.undercloud.job.standalone_ceph\n  debug:\n    msg: \"{{ hostvars.undercloud.job.standalone_ceph }}\"\n\n- name: job.standalone_ceph\n  debug:\n    msg: \"{{ job.standalone_ceph }}\"\n\nAs per:\n\n  https://paste.opendev.org/show/bhj0NL0VAXFXJ8qzYTVI/\n\nThis patchset uses the \"job.standalone_ceph | default(false)\". Hopefully that\u0027s all with standalone_ceph variable.","commit_id":"af821f3f8fbd442d47ae76a27593b7420fb23ba2"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"08ed07d26cd0698300b52da046612ca598bf7345","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":55,"id":"e7412162_e979d4d9","updated":"2022-05-03 18:10:17.000000000","message":"- Output from my change [0] confirms I have dependencies including python3 [1].\n- I see the same error in cephadm log [2] in [3] however.\n- _get_connection() [4] threw an exception visible in the ceph-mgr logs w/ debug [5]\n- 104.130.211.202:5001/tripleomastercentos9/daemon/v6.0.7-stable-6.0-pacific-centos-stream8\n\n\n[0] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/54..55/roles/standalone/tasks/ceph-install.yml\n\n[1] https://a1e2d113e9dec0e38dab-f41a14b9b9124c0084e146d917381dc8.ssl.cf5.rackcdn.com/834352/55/check/tripleo-ci-centos-9-scenario001-standalone/05dd715/logs/quickstart_install.log\n\n[2] https://a1e2d113e9dec0e38dab-f41a14b9b9124c0084e146d917381dc8.ssl.cf5.rackcdn.com/834352/55/check/tripleo-ci-centos-9-scenario001-standalone/05dd715/logs/undercloud/var/log/ceph/cephadm.log\n\n[3] https://github.com/ceph/ceph/pull/34250/files\n\n[4]  https://github.com/Daniel-Pivonka/ceph/blob/bb4554a3a67c4fc29b5dad597fbb880ea52394ac/src/pybind/mgr/cephadm/module.py#L1464-L1485\n\n[5] https://a1e2d113e9dec0e38dab-f41a14b9b9124c0084e146d917381dc8.ssl.cf5.rackcdn.com/834352/55/check/tripleo-ci-centos-9-scenario001-standalone/05dd715/logs/undercloud/var/log/ceph/8cc7641a-9465-5bf1-8d5a-6d9b4502dd25/ceph-mgr.standalone.localdomain.mdbrvd.log","commit_id":"5bf9d9154d0b223734514672e58cd78ddf51adab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a803d9fe5e30ab830d1dcc843209d45af7c3af65","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":56,"id":"31ae5bd2_c54e0efe","updated":"2022-05-03 19:32:48.000000000","message":"I\u0027ve added this [1] because the journal had these lines right before the exception from [3] was logged\n\n  May 03 16:58:48 standalone.localdomain ceph-8cc7641a-9465-5bf1-8d5a-6d9b4502dd25-mgr-standalone-localdomain-mdbrvd[25167]: sudo: /etc/sudoers.d/ceph-admin is owned by uid 1000, should be 0\n  ...\n  May 03 16:58:48 standalone.localdomain sudo[26407]: ceph-admin : user NOT in sudoers ; PWD\u003d/home/ceph-admin ; USER\u003droot ; COMMAND\u003d/bin/python3 -c import sys;exec(eval(sys.stdin.readline()))\n  May 03 16:58:48 standalone.localdomain sshd[26406]: Received disconnect from 104.130.209.29 port 56638:11: disconnected by user\n\nIf it helps then I\u0027ll patch [4] to add owner/group. Clearly the user IS in sudoers [5].\n\n[1]  https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/55..56/roles/standalone/tasks/ceph-install.yml#106\n\n[2] https://a1e2d113e9dec0e38dab-f41a14b9b9124c0084e146d917381dc8.ssl.cf5.rackcdn.com/834352/55/check/tripleo-ci-centos-9-scenario001-standalone/05dd715/logs/undercloud/var/log/extra/journal.txt\n\n[3] https://github.com/ceph/ceph/blob/v16.2.7/src/pybind/mgr/cephadm/serve.py#L1391\n\n[4] https://github.com/openstack/tripleo-ansible/blob/master/tripleo_ansible/roles/tripleo_create_admin/tasks/create_user.yml#L23-L28\n\n[5] https://a1e2d113e9dec0e38dab-f41a14b9b9124c0084e146d917381dc8.ssl.cf5.rackcdn.com/834352/55/check/tripleo-ci-centos-9-scenario001-standalone/05dd715/logs/undercloud/etc/sudoers.d/ceph-admin","commit_id":"8d8ad352aace1d7bc6ce42aa6cb2e0ab7d14390b"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"1c7c29eca01f33dc05c044d2f67b6dd5f1d78491","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":56,"id":"0778fe8e_b49cb33e","in_reply_to":"31ae5bd2_c54e0efe","updated":"2022-05-03 21:26:28.000000000","message":"\u003e I\u0027ve added this [1] because the journal had these lines right before the exception from [3] was logged\n\u003e \n\u003e   May 03 16:58:48 standalone.localdomain ceph-8cc7641a-9465-5bf1-8d5a-6d9b4502dd25-mgr-standalone-localdomain-mdbrvd[25167]: sudo: /etc/sudoers.d/ceph-admin is owned by uid 1000, should be 0\n\u003e   ...\n\u003e   May 03 16:58:48 standalone.localdomain sudo[26407]: ceph-admin : user NOT in sudoers ; PWD\u003d/home/ceph-admin ; USER\u003droot ; COMMAND\u003d/bin/python3 -c import sys;exec(eval(sys.stdin.readline()))\n\u003e   May 03 16:58:48 standalone.localdomain sshd[26406]: Received disconnect from 104.130.209.29 port 56638:11: disconnected by user\n\u003e \n\u003e If it helps then I\u0027ll patch [4] to add owner/group. Clearly the user IS in sudoers [5].\n\nWorked for 001/004 in latest run. I\u0027ll do the patch under this bug:\n\n https://bugs.launchpad.net/tripleo/+bug/1971498","commit_id":"8d8ad352aace1d7bc6ce42aa6cb2e0ab7d14390b"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"23e7e70f9e6ce979001c0c834b8f3c3f23eab2fa","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":58,"id":"b03af93d_d4dc2525","updated":"2022-05-05 14:22:35.000000000","message":"Adam King said cephadm CI seems a similar issue during addition of hosts using ceph orchestrator:\n\n http://magna002.ceph.redhat.com/cephci-jenkins/cephci-run-ER8KGZ/Host_addition_with_spec_file_0.log\n\n  Error EINVAL: Can\u0027t communicate with remote host `10.0.211.162`, possibly because python3 is not installed there: cannot send (already closed?\n\n","commit_id":"53868f3f9d4dcae90aa4a582470b1120871e5076"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"605281b74fdb075d30f6f369f64767c2affd50e4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":58,"id":"b5c2b6f0_94a28eea","updated":"2022-05-05 15:25:30.000000000","message":"The following dependencies have been workflowed. I try again after they merge.\n\n- https://review.opendev.org/c/openstack/tripleo-ansible/+/840449\n- https://review.opendev.org/c/openstack/tripleo-ansible/+/840385","commit_id":"53868f3f9d4dcae90aa4a582470b1120871e5076"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"45d226d0532fb7777970ee309251cde51b91f2c5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"d0199bc9_1d57de13","updated":"2022-05-05 21:01:05.000000000","message":"- In 001 ceph and overcloud deployment succeeded as well as temptest [1]\n- In 004 ceph deployed fine but then the overcloud deployment failed [2]\n- The 004 deployment failed while creaing pools [3] as seen in [4]\n- Looks like [5] introduced a bug by defaulting the pools to {} [6]\n- We shouldn\u0027t override CephPools from the default that THT has like that\n- I need to fix [5] before it\u0027s merged\n\n[1] https://paste.opendev.org/show/btVQM8u0fcJJZTddtW8Y/\n[2] https://paste.opendev.org/show/bXl5sR11YNN0wOhFXipl/\n[3] \n2022-05-05 19:30:55,336 p\u003d34908 u\u003droot n\u003dansible | 2022-05-05 19:30:55.335390 | fa163ea5-1ba7-880f-9603-000000001d35 |      FATAL | build resulting pool list | undercloud | error\u003d{\"msg\": \"Unexpected templating type error occurred on ({\u0027openstack_pools\u0027: {{ vms|default([]) + volumes|default([]) + images|default([]) + metrics|default([]) + backup|default([]) + cinder_extra|default([]) + ceph_pools.extra_pools|default([]) }} }): can only concatenate list (not \\\"dict\\\") to list\"}\n[4] https://a34faa8b05592d9ab413-2fe3f6c342189909aad5220492fb4721.ssl.cf5.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/2c64374/logs/undercloud/home/zuul/ansible.log\n\n[5] https://review.opendev.org/c/openstack/tripleo-ansible/+/840449\n\n[6] https://a34faa8b05592d9ab413-2fe3f6c342189909aad5220492fb4721.ssl.cf5.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/2c64374/logs/undercloud/home/zuul/deployed_ceph.yaml\n","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"4c25daa0bda919cd40bb79e2671df5ca350e99ea","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"0c3523a2_8a127791","updated":"2022-05-11 14:58:31.000000000","message":"- RGW was not deployed for 004 in this change.\n- If RGW was deployed, then it would have produced logs; but it didn\u0027t.\n- It did for the green 004 job [1] (without this change) but did not for this change [2]\n- The cephadm.log [3] for the green job contains the string \"rgw\" but not for this job [4]\n- The tripleo-ansible log for the cephadm command for the green job shows RGW [5]\n- The same for this job shows RGW [6] not running at the end though it has an RGW spec with a different network! \n- I see \"radosgw_address_block: 192.168.24.0/24\" and the cluster network in cephadm-extra-vars-heat.yml\n- So it tried to deploy RGW but it failed to actually run it\n- We see the ceph mgr rejecting attempts to use the \u0027192.168.24.0/24\u0027 network [9] [10]\n\n[1] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_dce/840436/2/gate/tripleo-ci-centos-9-scenario004-standalone/dce077e/logs/undercloud/var/log/ceph/4b5c8c0a-ff60-454b-a1b4-9747aa737d19/index.html\n\n[2] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/var/log/ceph/e07b8943-63ce-545a-a0fd-2a61261e02ab/index.html\n\n[3] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_dce/840436/2/gate/tripleo-ci-centos-9-scenario004-standalone/dce077e/logs/undercloud/var/log/ceph/cephadm.log\n\n[4] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/var/log/ceph/cephadm.log\n\n[5] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_dce/840436/2/gate/tripleo-ci-centos-9-scenario004-standalone/dce077e/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-_01k072m/cephadm/cephadm_command.log\n\n[6] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-mtf1kj6p/cephadm/cephadm_command.log\n\n[7]\n        \"service_type: rgw\",\n        \"service_id: rgw\",\n        \"service_name: rgw.rgw\",\n        \"placement:\",\n        \"  hosts:\",\n        \"  - standalone.localdomain\",\n        \"networks:\",\n        \"- 192.168.24.0/24\",\n        \"spec:\",\n        \"  rgw_frontend_port: 8080\",\n        \"  rgw_realm: default\",\n        \"  rgw_zone: default\"\n\n[8] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-mtf1kj6p/cephadm/cephadm-extra-vars-heat.yml\n\n[9] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/var/log/ceph/e07b8943-63ce-545a-a0fd-2a61261e02ab/ceph-mgr.standalone.localdomain.ffnwxk.log\n\n[10]\n2022-05-09T19:33:36.466+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : args: --image 149.202.183.74:5001/tripleomastercentos9/daemon@sha256:aed5d7c55a8065602f1990e46c3c232176bfbbb1341d78b41eba1c4f696a8f5e deploy --fsid e07b8943-63ce-545a-a0fd-2a61261e02ab --name mon.standalone.localdomain --meta-json {\"service_name\": \"mon\", \"ports\": [], \"ip\": null, \"deployed_by\": [\"149.202.183.74:5001/tripleomastercentos9/daemon@sha256:aed5d7c55a8065602f1990e46c3c232176bfbbb1341d78b41eba1c4f696a8f5e\"], \"rank\": null, \"rank_generation\": null} --config-json - --reconfig\n2022-05-09T19:33:36.466+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : stdin: {\"config\": \"# minimal ceph.conf for e07b8943-63ce-545a-a0fd-2a61261e02ab\\n[global]\\n\\tfsid \u003d e07b8943-63ce-545a-a0fd-2a61261e02ab\\n\\tmon_host \u003d [v2:149.202.181.186:3300/0,v1:149.202.181.186:6789/0]\\n[mon.standalone.localdomain]\\npublic network \u003d 192.168.24.0/24\\n\", \"keyring\": \"[mon.]\\n\\tkey \u003d AQCUZ3liHG3RKRAAQnAcnkLHhJCnqRW19QiyrA\u003d\u003d\\n\\tcaps mon \u003d \\\"allow *\\\"\\n\", \"files\": {\"config\": \"[mon.standalone.localdomain]\\npublic network \u003d 192.168.24.0/24\\n\"}}\n2022-05-09T19:33:37.237+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : code: 0\n2022-05-09T19:33:37.237+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : err: Reconfig daemon mon.standalone.localdomain ...\n2022-05-09T19:33:37.248+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : Sleeping for 60 seconds\n2022-05-09T19:33:37.263+0000 7fcb77899700  0 log_channel(cluster) log [DBG] : pgmap v628: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 4.8 MiB used, 14 GiB / 14 GiB avail\n2022-05-09T19:33:39.170+0000 7fcb5e897700  0 log_channel(audit) log [DBG] : from\u003d\u0027client.14185 -\u0027 entity\u003d\u0027client.admin\u0027 cmd\u003d[{\"prefix\": \"osd pool stats\", \"pool_name\": \"images\", \"target\": [\"mon-mgr\", \"\"], \"format\": \"json\"}]: dispatch\n2022-05-09T19:33:39.170+0000 7fcb5e897700 -1 mgr.server reply reply (2) No such file or directory unrecognized pool \u0027images\u0027\n2022-05-09T19:33:39.264+0000 7fcb77899700  0 log_channel(cluster) log [DBG] : pgmap v630: 9 pgs: 9 active+clean; 0 B data, 4.9 MiB used, 14 GiB / 14 GiB avail\n2022-05-09T19:33:41.182+0000 7fcb27701700  0 [progress INFO root] Processing OSDMap change 10..13\n2022-05-09T19:33:41.264+0000 7fcb77899700  0 log_channel(cluster) log [DBG] : pgmap v632: 17 pgs: 8 unknown, 9 active+clean; 0 B data, 4.9 MiB used, 14 GiB / 14 GiB avail\n2022-05-09T19:33:43.264+0000 7fcb77899700  0 log_channel(cluster) log [DBG] : pgmap v635: 17 pgs: 8 unknown, 9 active+clean; 0 B data, 4.9 MiB used, 14 GiB / 14 GiB avail\n2022-05-09T19:33:43.935+0000 7fcb5e897700  0 log_channel(audit) log [DBG] : from\u003d\u0027client.14191 -\u0027 entity\u003d\u0027client.admin\u0027 cmd\u003d[{\"prefix\": \"orch apply\", \"target\": [\"mon-mgr\", \"\"]}]: dispatch\n2022-05-09T19:33:43.939+0000 7fcb56096700  0 [cephadm INFO root] Saving service rgw.rgw spec with placement standalone.localdomain\n2022-05-09T19:33:43.939+0000 7fcb56096700  0 log_channel(cephadm) log [INF] : Saving service rgw.rgw spec with placement standalone.localdomain\n2022-05-09T19:33:43.947+0000 7fcb56096700  0 log_channel(cephadm) log [DBG] : _kick_serve_loop\n2022-05-09T19:33:43.983+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : mon_command: \u0027config dump\u0027 -\u003e 0 in 0.002s\n2022-05-09T19:33:43.983+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : refreshing hosts and daemons\n2022-05-09T19:33:43.984+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : mon_command: \u0027config generate-minimal-conf\u0027 -\u003e 0 in 0.001s\n2022-05-09T19:33:43.986+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : mon_command: \u0027auth get\u0027 -\u003e 0 in 0.001s\n2022-05-09T19:33:43.986+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : Provided hosts: [DaemonPlacement(daemon_type\u003d\u0027mon\u0027, hostname\u003d\u0027standalone.localdomain\u0027, network\u003d\u0027\u0027, name\u003d\u0027\u0027, ip\u003dNone, ports\u003d[], rank\u003dNone, rank_generation\u003dNone)]\n2022-05-09T19:33:43.987+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : _check_for_strays\n2022-05-09T19:33:43.987+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : [{\u0027hostname\u0027: \u0027standalone.localdomain\u0027, \u0027services\u0027: [{\u0027type\u0027: \u0027mgr\u0027, \u0027id\u0027: \u0027standalone.localdomain.ffnwxk\u0027}, {\u0027type\u0027: \u0027mon\u0027, \u0027id\u0027: \u0027standalone.localdomain\u0027}, {\u0027type\u0027: \u0027osd\u0027, \u0027id\u0027: \u00270\u0027}], \u0027ceph_version\u0027: \u0027ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)\u0027}]\n2022-05-09T19:33:43.987+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : 0 OSDs are scheduled for removal: []\n2022-05-09T19:33:43.987+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : Saving [] to store\n2022-05-09T19:33:44.005+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : Applying service mon spec\n2022-05-09T19:33:44.006+0000 7fcb2af48700  0 log_channel(cephadm) log [DBG] : mon public_network(s) is [\u0027192.168.24.0/24\u0027]\n2022-05-09T19:33:44.006+0000 7fcb2af48700  0 [cephadm INFO cephadm.serve] Filtered out host standalone.localdomain: does not belong to mon public_network (192.168.24.0/24)\n2022-05-09T19:33:44.006+0000 7fcb2af48700  0 log_channel(cephadm) log [INF] : Filtered out host standalone.localdomain: does not belong to mon public_network (192.168.24.0/24)\n\n","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"23a64b7ae9bf75b492555837f3711fff2da591e1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"bd602f79_d9d8928b","updated":"2022-05-11 12:04:46.000000000","message":"- Tempest failed for 004 [1]\n- No stackviz reports so it\u0027s hard to find which test failed in the tempest log [2]\n- 004 was good for another job this morning [3] but difference is not yet apparent to me\n\n[1] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/quickstart_install.log\n\nTASK [os_tempest : Execute tempest tests] **************************************\nMonday 09 May 2022  19:51:44 +0000 (0:00:00.049)       0:47:57.387 ************ \nfatal: [undercloud]: FAILED! \u003d\u003e {\"changed\": false, \"cmd\": \"set -e\\nif [ -d /openstack/venvs/tempest-untagged/bin ];\\nthen\\n. /openstack/venvs/tempest-untagged/bin/activate\\nfi\\ntempest run   --concurrency 2   --exclude-list /home/zuul/tempest/etc/tempest_excludelist.txt   --include-list /home/zuul/tempest/etc/tempest_includelist.txt \u003e /var/log/tempest/tempest_run.log\\n\", \"delta\": \"0:02:10.176934\", \"end\": \"2022-05-09 19:53:56.233928\", \"msg\": \"non-zero return code\", \"rc\": 1, \"start\": \"2022-05-09 19:51:46.056994\", \"stderr\": \"\", \"stderr_lines\": [], \"stdout\": \"\", \"stdout_lines\": []}\n\n[2]\nhttps://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/home/zuul/tempest/tempest.log\n\n[3] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_dce/840436/2/gate/tripleo-ci-centos-9-scenario004-standalone/dce077e/logs/undercloud/home/zuul/tempest/tempest.log\n\n\n","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ef13d18a839a68826b1088d14783f12ec750625d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"3b6c9d38_84dd6dbe","updated":"2022-05-11 12:20:38.000000000","message":"- tempest_results.xml [1] is my friend\n- RGW related: test_swift_acl_anonymous_download + test_swift_basic_ops failure\n- Both replied w/ 503 Service Unavailable: No server is available to handle this request.\n\n[1] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/var/log/tempest/tempest_results.xml","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"98ebc238e74024812ce7588642517192a3821e0c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"01d75e83_fe8c66da","updated":"2022-05-11 12:42:13.000000000","message":"Why was the RGW service at http://192.168.24.3:8080 unavailable at these time stamps?\n\n- 2022-05-09 19:53:45,563\n- 2022-05-09 19:53:45,557\n\n{{{ 2022-05-09 19:53:45,563 134653 INFO [tempest.lib.common.rest_client] Request (TestObjectStorageBasicOps:test_swift_basic_ops): 503 GET http://192.168.24.3:8080/swift/v1/AUTH_46a0d97c16cd4647ac0b9d4a14183a9a 0.001s 2022-05-09 19:53:45,563 134653 DEBUG [tempest.lib.common.rest_client] Request - Headers: {\"X-Auth-Token\": \"\u003comitted\u003e\"} Body: None Response - Headers: {\"content-length\": \"107\", \"cache-control\": \"no-cache\", \"content-type\": \"text/html\", \"connection\": \"close\", \"status\": \"503\", \"content-location\": \"http://192.168.24.3:8080/swift/v1/AUTH_46a0d97c16cd4647ac0b9d4a14183a9a\"} Body: b\"\u003chtml\u003e\u003cbody\u003e\u003ch1\u003e503 Service Unavailable\u003c/h1\u003e\\nNo server is available to handle this request.\\n\u003c/body\u003e\u003c/html\u003e\\n\" }}} \n\nTraceback (most recent call last): \nFile \"/usr/lib/python3.9/site-packages/tempest/common/utils/__init__.py\", line 70, in wrapper return f(*func_args, **func_kwargs) \nFile \"/usr/lib/python3.9/site-packages/tempest/scenario/test_object_storage_basic_ops.py\", line 37, in test_swift_basic_ops self.get_swift_stat() \nFile \"/usr/lib/python3.9/site-packages/tempest/scenario/manager.py\", line 1628, in get_swift_stat self.account_client.list_account_containers() \nFile \"/usr/lib/python3.9/site-packages/tempest/lib/services/object_storage/account_client.py\", line 70, in list_account_containers resp, body \u003d self.get(url, headers\u003d{}) \nFile \"/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py\", line 314, in get return self.request(\u0027GET\u0027, url, extra_headers, headers) \nFile \"/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py\", line 703, in request self._error_checker(resp, resp_body) \nFile \"/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py\", line 883, in _error_checker raise exceptions.UnexpectedResponseCode(str(resp.status), tempest.lib.exceptions.UnexpectedResponseCode: Unexpected response code received Details: 503 \n","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"307bcd8f4e77d23e7a24f0cd02f798cee781c6bf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"095eb5de_7cecfd72","updated":"2022-05-05 21:33:37.000000000","message":"recheck","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"a616768e316083985d60ed80653a4456e39bfb53","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"1219f466_5a82f360","updated":"2022-05-09 17:53:56.000000000","message":"recheck","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"08bade196f6b634c7edaffc811bf645f67abd737","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"32224f4b_90607eb5","updated":"2022-05-09 09:52:00.000000000","message":"recheck","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ab8e4b658f3704daf4a1650f0b87aa11a60cde46","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"341b7bea_44459cfd","updated":"2022-05-06 11:38:46.000000000","message":"recheck","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"34cb72b76e468294fade655190776f3c04890a09","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"646c29bf_9e95653d","updated":"2022-05-06 14:40:36.000000000","message":"recheck","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"b7b76afdb1e26d43e91825fcdb1f171b385ed09d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"f22ab34b_bf49691f","updated":"2022-05-09 06:35:41.000000000","message":"recheck","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"21b5861c764dadbb9a3f1fcdc0d21e38b2372da3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":66,"id":"f358828c_232266e5","updated":"2022-05-18 00:53:52.000000000","message":"zuul is only red for an unrelated standalone-on-multinode-ipa failure.\n\ntripleo-ci-centos-9-scenario004-standalone is green\n\nMy override of CephClusterNetwork, CephMonNetwork, and CephRgwNetwork didn\u0027t change the values that Heat used and RGW was brought up fine on .24 network instead of the .42 network. However, it made no difference and the tempest tests passed.\n\nI\u0027ll make an update to remove the override of CephClusterNetwork, CephMonNetwork, and CephRgwNetwork.\n\n- The ceph.conf [1] has 192.168.42.0/24 for the CephClusterNetwork, CephMonNetwork, CephRgwNetwork\n- cephadm-extra-vars-heat.yml [2] has radosgw_address_block: 192.168.24.0/24 though\n- The rgw spec in the logs [3] has the .24 instead of the .42 address\n- The same log [3] shows RGW is running \"rgw: 1 daemon active (1 hosts, 1 zones)\",\n- The ceph mgr log [4] shows it had no problem running RGW on 192.168.24.1\n- Since both 192.168.42.1 and 192.168.24.1 are valid addresses for the host it didn\u0027t seem to matter\n- Tempest results [5] [6] show that swift and manila passed\n\ntripleo-ci-centos-9-scenario001-standalone is also green.\n\n[1]  https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_8a2/834352/66/check/tripleo-ci-centos-9-scenario004-standalone/8a2135e/logs/undercloud/home/zuul/deployed_ceph.yaml\n\n[2] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_8a2/834352/66/check/tripleo-ci-centos-9-scenario004-standalone/8a2135e/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-5qvwo1l0/cephadm/cephadm-extra-vars-heat.yml\n\n[3] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_8a2/834352/66/check/tripleo-ci-centos-9-scenario004-standalone/8a2135e/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-5qvwo1l0/cephadm/cephadm_command.log\n\n[4] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_8a2/834352/66/check/tripleo-ci-centos-9-scenario004-standalone/8a2135e/logs/undercloud/var/log/ceph/b3de7558-81e5-5c78-958b-70e94035fa43/ceph-mgr.standalone.localdomain.dqeiix.log\n\n[5] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_8a2/834352/66/check/tripleo-ci-centos-9-scenario004-standalone/8a2135e/logs/undercloud/var/log/tempest/tempest_results.xml\n\n[6] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_8a2/834352/66/check/tripleo-ci-centos-9-scenario004-standalone/8a2135e/logs/undercloud/var/log/tempest/stestr_results.html","commit_id":"0c7ab0720c7be0a6c5b14ea0b46b8e70371b1e7d"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"c7bffb5faa65f90012a26d18da2685c333a55b11","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":68,"id":"9add0c7e_31a1b2fc","updated":"2022-05-18 13:04:34.000000000","message":"- 004 failure waiting for OSDs to come up [1] \n- 001 failure creating ceph-admin user [2]\n\nHave not seen these yet in this job.\n\n[1] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_bc0/834352/68/check/tripleo-ci-centos-9-scenario004-standalone/bc0e357/logs/undercloud/home/zuul/ansible.log\nstandalone.localdomain | error\u003d{\"attempts\": 40, \"changed\": true, \"cmd\": \"podman run --rm --net\u003dhost --ipc\u003dhost   --volume /etc/ceph:/etc/ceph:z --volume /home/ceph-admin/assimilate_ceph.conf:/home/assimilate_ceph.conf:z  --volume /home/ceph-admin/specs/ceph_spec.yaml:/home/ceph_spec.yaml:z   --entrypoint ceph 149.202.170.89:5001/tripleomastercentos9/daemon:v6.0.7-stable-6.0-pacific-centos-stream8 --fsid 3c43b3a9-311b-598d-aea6-27c053f409c3 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring   status --format json | jq .osdmap.num_up_osds\", \"delta\": \"0:00:00.998657\", \"end\": \"2022-05-18 12:37:02.931787\", \"msg\": \"\", \"rc\": 0, \"start\": \"2022-05-18 12:37:01.933130\", \"stderr\": \"\", \"stderr_lines\": [], \"stdout\": \"0\", \"stdout_lines\": [\"0\"]}\n\n[2] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_074/834352/68/check/tripleo-ci-centos-9-scenario001-standalone/074480e/logs/undercloud/home/zuul/ansible.log\n2022-05-18 12:14:23,801 p\u003d23163 u\u003dzuul n\u003dansible | 2022-05-18 12:14:23.800414 | fa163e5a-88bd-684b-2e0f-000000000016 |      FATAL | create user ceph-admin | standalone.localdomain | error\u003d{\"censored\": \"the output has been hidden due to the fact that \u0027no_log: true\u0027 was specified for this result\", \"changed\": false}\n","commit_id":"6af1644c444ecfc92e2975ab1264ab36b6cb9d39"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"5136891d5f53060bb6321b9b16e2e904d139373d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":73,"id":"b934e729_f8ecb665","updated":"2022-05-23 13:04:10.000000000","message":"- 001 and 010 green with network changes\n- all ceph jobs (001, 004, 010) are getting checked with this patch\n- 004 red with new useradd error unrelated to this patch. a fluke?\n\nUseradd error details:\n\n2022-05-22 17:47:56,876 p\u003d24066 u\u003dzuul n\u003dansible | 2022-05-22 17:47:56.874903 | fa163eac-8634-12d6-8044-000000000016 |      FATAL | create user ceph-admin | undercloud | error\u003d{\"censored\": \"the output has been hidden due to the fact that \u0027no_log: true\u0027 was specified for this result\", \"changed\": false}\n\nMay 22 17:47:56 standalone.localdomain sudo[24122]: pam_unix(sudo:session): session opened for user root(uid\u003d0) by (uid\u003d1000)\nMay 22 17:47:56 standalone.localdomain useradd[24127]: new group: name\u003dceph-admin, GID\u003d1002\nMay 22 17:47:56 standalone.localdomain useradd[24127]: new user: name\u003dceph-admin, UID\u003d1001, GID\u003d1002, home\u003d/home/ceph-admin, shell\u003d/bin/bash, from\u003dnone\nMay 22 17:47:56 standalone.localdomain useradd[24128]: failed adding user \u0027ceph-admin\u0027, exit code: 9\nMay 22 17:47:56 standalone.localdomain sudo[24122]: pam_unix(sudo:session): session closed for user root\nMay 22 17:47:56 standalone.localdomain sudo[24121]: pam_unix(sudo:session): session closed for user root\nMay 22 17:47:57 standalone.localdomain sudo[24147]:  \n\n$ man useradd\n...\n       9\n           username already in use\n...\n$\n\nSo the user already existed? The journal shows no log of the user being created before that.\n\n  https://a340eb04c9bdc60233e3-f794ac2492944f87fa91e71db0988d60.ssl.cf1.rackcdn.com/834352/73/check/tripleo-ci-centos-9-scenario004-standalone/4f9865f/logs/undercloud/var/log/extra/journal.txt\n\nMaybe the host wasn\u0027t correctly cleaned but I see other users getting added, e.g. heat.\n\nI could require that thetripleo_create_admin roleit check the user exists before adding the user. But that role is used in evey job which uses config-download. \n\n  https://github.com/openstack/tripleo-ansible/blob/master/tripleo_ansible/playbooks/ceph-admin-user-playbook.yml#L59\n\n  https://github.com/openstack/tripleo-ansible/blob/master/tripleo_ansible/roles/tripleo_create_admin/tasks/create_user.yml#L19\n\nIs it bug in the python\u0027s user module?\n\n  https://github.com/ansible/ansible/blob/stable-2.12/lib/ansible/modules/user.py#L784\n\nit basically ran \"useradd ...\" and got 9 error code back but maybe there\u0027s something odd going on. Did useradd have a race condition? It was provided by shadow-utils-4.9-4.el9.x86_64\n\nSame exact code was fine when 001 and 010 ran. Let\u0027s see if this happens again.\n\n\n\n","commit_id":"26b3892607132b64aa3eacd6d4c536035fb2b451"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"385c59d405bd7582994d834c003a0afa9c72766d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":73,"id":"07147e37_173a2292","updated":"2022-05-23 17:44:50.000000000","message":"recheck","commit_id":"26b3892607132b64aa3eacd6d4c536035fb2b451"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"304d717251f05dff47ec6032d0bcf7b6a526b682","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":73,"id":"2106a9c5_4221fadc","updated":"2022-05-23 13:06:36.000000000","message":"recheck","commit_id":"26b3892607132b64aa3eacd6d4c536035fb2b451"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"0c63625356fe21184f0ac3d26b8610df7e16b9bf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":75,"id":"59e412d5_58431bad","updated":"2022-05-27 12:31:25.000000000","message":"Here\u0027s why the RGW tempest test failed in 004-standalone:\n\n- The hosts file [1] maps standalone.storage to 192.168.24.1 \n- However, standalone.storage should map to 192.168.42.1 (24 vs 42)\n- haproxy logs [2] show it trying to use standalone.storage to reach RGW [3] 10 seconds before the failing tempest test [4]\n\n[1] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_916/834352/75/check/tripleo-ci-centos-9-scenario004-standalone/9162458/logs/undercloud/etc/hosts\n\n[2] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_916/834352/75/check/tripleo-ci-centos-9-scenario004-standalone/9162458/logs/undercloud/var/log/containers/haproxy/haproxy.log\n\n[3] \nMay 25 21:39:17 standalone haproxy[7]: Server ceph_rgw_be/standalone.storage.localdomain is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.\n\n[4]\nMay 25 21:58:44 standalone haproxy[7]: 192.168.24.3:53186 [25/May/2022:21:58:44.138] ceph_rgw ceph_rgw_be/\u003cNOSRV\u003e 0/-1/-1/-1/0 503 217 - - SC-- 43/1/0/0/0 0/0 \"PUT /swift/v1/AUTH_e3673523d60d4374aab72c34aca1e462/tempest-swift-scenario-container-1011309131 HTTP/1.1\"\nMay 25 21:58:44 standalone haproxy[7]: 192.168.24.3:53188 [25/May/2022:21:58:44.148] ceph_rgw ceph_rgw_be/\u003cNOSRV\u003e 0/-1/-1/-1/0 503 217 - - SC-- 43/1/0/0/0 0/0 \"GET /swift/v1/AUTH_e3673523d60d4374aab72c34aca1e462 HTTP/1.1\"\n","commit_id":"d8baee5e887a7d5960f7259f4d9274aab25ddafc"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"51f1152340749b6d6101cdb03990f244865439d3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":75,"id":"e91fc9e3_5a6dd2ab","updated":"2022-05-25 19:38:51.000000000","message":"recheck","commit_id":"d8baee5e887a7d5960f7259f4d9274aab25ddafc"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"902fd809ee3a51cbc6e59b26478df2e22786be21","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":75,"id":"fcd5c309_4202802e","updated":"2022-05-26 20:06:00.000000000","message":"recheck","commit_id":"d8baee5e887a7d5960f7259f4d9274aab25ddafc"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"6a2141e7a2956539750e8458ced78d11cfdfa7ba","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":75,"id":"21683451_a1dee8b2","in_reply_to":"59e412d5_58431bad","updated":"2022-05-27 17:45:44.000000000","message":"\u003e Here\u0027s why the RGW tempest test failed in 004-standalone:\n\u003e \n\u003e - The hosts file [1] maps standalone.storage to 192.168.24.1 \n\n- I confirmed that passing a NodePortMap [1] results in the correct hosts file.\n- I\u0027ll update the ansible [2] which generates the deployed_network file to include the NodePortMap\n\n[1] \n  NodePortMap:\n    standalone:\n      ctlplane:\n        ip_address: 192.168.24.2\n        ip_subnet: 192.168.24.2/24\n        ip_address_uri: 192.168.24.2\n      storage:\n        ip_address: 192.168.42.1\n        ip_subnet: 192.168.42.1/24\n        ip_address_uri: 192.168.42.1\n      storage_mgmt:\n        ip_address: 192.168.42.1\n        ip_subnet: 192.168.42.1/24\n        ip_address_uri: 192.168.42.1\n\n[2] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/75/roles/standalone/tasks/storage-network.yml#64","commit_id":"d8baee5e887a7d5960f7259f4d9274aab25ddafc"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"650b3a6df6c08896b1e5fbc63107207b7e71d358","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"ba796a03_0788ce6e","updated":"2022-06-10 13:19:54.000000000","message":"Failure in tripleo-ci-centos-9-scenario010-ovn-provider-standalone is unrelated Gather_podman_infos_failed.log\t","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"def3b3051164b778bfe03d74f3de71e2f168140b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"4b07e145_64341045","updated":"2022-06-07 15:45:35.000000000","message":"Thanks for the review Sandeep. I have an answer to your question below.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"193e8852e92273cc0d1836883bb7b83a447253cb","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"f169f1c4_5d634ae4","updated":"2022-06-08 05:29:57.000000000","message":"hitting known gate blocker @ https://bugs.launchpad.net/tripleo/+bug/1977873\n\nhttps://5da7fa74180cdcfe2917-eb0b1bb80d6d8b85b62f24c50591cd92.ssl.cf1.rackcdn.com/834352/81/gate/tripleo-ci-centos-9-standalone/9608e7f/logs/undercloud/home/zuul/standalone_deploy.log\n\n2022-06-07 22:24:42.832375 | fa163eb6-7a32-51c6-2ecb-0000000007d9 |      FATAL | Enable os_enable_vtpm SELinux boolean for vTPM | standalone | error\u003d{\"changed\": false, \"msg\": \"SELinux boolean os_enable_vtpm does not exist.\"}","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"cdf19a42011dd5124aab2755e5dcdbeab05cafd3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"7c2cfe11_58ce28f0","updated":"2022-06-14 09:56:26.000000000","message":"https://e7fbb04e1f1bc4fc681d-d7024b4a9df5a3cf5a52965a2b5a469a.ssl.cf1.rackcdn.com/834352/81/gate/tripleo-ci-centos-9-scenario004-standalone/574cdfb/job-output.txt\n\n```\nTASK | create user ceph-admin\", \"2022-06-14 08:15:45.376184 | bc764e20-14a4-6973-e52d-000000000016 |      FATAL | create user ceph-admin | undercloud | error\u003d{\\\"censored\\\": \\\"the output has been hidden due to the fact that \u0027no_log: true\u0027 was specified for this result\\\", \\\"changed\\\": false}\", \"2022-06-14 08:15:45.377561 | bc764e20-14a4-6973-e52d-000000000016 |     TIMING | tripleo_create_admin : create user {{ tripleo_admin_user }} | undercloud | 0:00:03.755695 | 0.65s\", \"2022-06-14 08:15:45.420135 | bc764e20-14a4-6973-e52d-000000000016 |    CHANGED | create user ceph-admin | standalone.localdomain\", \"2022-06-14 08:15:45.421751 | bc764e20-14a4-6973-e52d-000000000016 |     TIMING | tripleo_create_admin : create user {{ tripleo_admin_user }} | standalone.localdomain | 0:00:03.799893 | 0.71s\", \"2022-06-14 08:15:45.441156 | bc764e20-14a4-6973-e52d-000000000017 |       TASK | grant admin rights to user ceph-admin\", \"2022-06-14 08:15:46.343298 | bc764e20-14a4-6973-e52d-000000000017 |    CHANGED | grant admin rights to user ceph-admin | standalone.localdomain\", \"2022-06-14 08:15:46.344505 | bc764e20-14a4-6973-e52d-000000000017 |     TIMING | tripleo_create_admin : grant admin rights to user {{ tripleo_admin_user }} | standalone.localdomain | 0:00:04.722652 | 0.90s\", \"2022-06-14 08:15:46.358417 | bc764e20-14a4-6973-e52d-000000000018 |\n\n```","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":9976,"name":"Ronelle Landy","email":"rlandy@redhat.com","username":"rlandy"},"change_message_id":"867b39a651c62f57a7bce7b6c389c8c77cda61d1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"e7b2cdd8_d21c52ff","updated":"2022-06-03 11:25:26.000000000","message":"if we merge https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352 w/o the others in the set, will we break something?","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"54bf25f8316e343e55892fed06a8c79d1f5f75f2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"4b75eade_d7d84bbc","updated":"2022-06-13 11:54:23.000000000","message":"recheck","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"cd514dcb807297f419803a1fe807cc7eba550f4c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"543e5bcb_9502df25","updated":"2022-06-14 18:36:57.000000000","message":"recheck","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"0e2d47fbd40588fa7cd26353eba86664971dac17","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"7673354d_4223ae85","updated":"2022-06-12 21:42:12.000000000","message":"recheck","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"0d797b2f1b27588205c3bbae1918b889be48b353","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"97f68309_71961c89","updated":"2022-06-14 02:50:32.000000000","message":"recheck","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"42293760c60e99bd3812edb2cff0ed1ca2b783a7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"ab9e257a_b6417e19","updated":"2022-06-10 13:20:04.000000000","message":"recheck","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"7711938ccb4ac4b9e6199c812646804fb429e54b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"fd6eff46_a5774354","updated":"2022-06-14 14:19:25.000000000","message":"recheck","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"99d7dec7946bce85d3aa4f6cbdbb2c5863dab395","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"247f4eb7_ebcc944a","updated":"2022-06-13 01:23:38.000000000","message":"recheck ","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"170ea78e87f6ceda24844f82d98f6f52e0230b73","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"2f09e40d_6894c506","updated":"2022-06-12 15:59:46.000000000","message":"recheck ","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"b1c37f752cdbfdc70c366c41ae09a1ee211681ca","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"3da7308a_1905b20a","updated":"2022-06-10 07:24:41.000000000","message":"recheck \n\nshould be ok now ","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"cc60a7d4ac5f4e75b5467561a7fb677cbc96c3a1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"30a011aa_5d32f93c","updated":"2022-06-15 12:56:42.000000000","message":"recheck disk full issue","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"aae5ca7b6521e6353f208b1e0eaeb69b871f56a8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"c83bcd46_7acf16e3","updated":"2022-06-15 05:11:26.000000000","message":"recheck sc10 passing in last run","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"3a07dac7e9f6fa15e109f9f3f4d6f3e65a09344c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"ace0c336_c528041c","updated":"2022-06-07 06:55:27.000000000","message":"thanks ","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"48ace5c0998b8211164874eee60f817110ad178f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"6fdf8d32_979225fa","updated":"2022-06-06 07:07:08.000000000","message":"thanks - need at least one more pass \n\nsome inline comments for consideration","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"7711938ccb4ac4b9e6199c812646804fb429e54b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"6dd3f32c_259029b1","in_reply_to":"4ac60a73_7fbb2076","updated":"2022-06-14 14:19:25.000000000","message":"Thank you John for the detailed reply. Let me recheck the patch and if the issue still comes, we can open a bug.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"f30cc917d355a79587d94eb3b63b1592e4e76703","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"4ac60a73_7fbb2076","in_reply_to":"7c2cfe11_58ce28f0","updated":"2022-06-14 14:08:09.000000000","message":"- The tripleo_create_admin role (which creates the tripleo-admin user for everything) failed [0] on one out of four jobs \n\n- The failure is intermittent (was green previously)\n\n- It failed on the \"create user\" task [1] which really just calls the ansible user module [2]\n\n- When correlating the failed task time [3] with the journal log [4] I see \"useradd[31970]: failed adding user \u0027ceph-admin\u0027, exit code: 9\" [5]\n\n- Exit code 9 means \"username already in use\" [6]\n\n- The journal [4] shows \"new user: name\u003dceph-admin\" only once\n\n- We\u0027re using  Ansible Version: 2.8.20 per the job output [7]\n\n- The user module for this version already has a user_exists function so idempotence is attempted [8]\n\n- The user module in 2.8.20 hasn\u0027t been updated in 3 years though the version 2.13 was updated last 3 months ago\n\n- Perhaps for now, as a work-around, I should assume that [1] is not idempotent and add a check before calling it. I could add a WIP patch to depends-on to at least get more debugging information.\n\n- Bottom line, the exact same call is made regardless of if this patch is merged. I.e. this role is used everywhere and this job patch merely calls the role earlier in the deployment process.\n\n[0] https://e7fbb04e1f1bc4fc681d-d7024b4a9df5a3cf5a52965a2b5a469a.ssl.cf1.rackcdn.com/834352/81/gate/tripleo-ci-centos-9-scenario004-standalone/574cdfb/logs/undercloud/home/zuul/ansible.log\n\n[1]  https://github.com/openstack/tripleo-ansible/blob/master/tripleo_ansible/roles/tripleo_create_admin/tasks/create_user.yml#L18-L21\n\n[2] https://docs.ansible.com/ansible/latest/collections/ansible/builtin/user_module.html\n\n[3] 2022-06-14 08:15:45,377 p\u003d31908 u\u003dzuul n\u003dansible | 2022-06-14 08:15:45.376184 | bc764e20-14a4-6973-e52d-000000000016 |      FATAL | create user ceph-admin | undercloud | error\u003d{\"censored\": \"the output has been hidden due to the fact that \u0027no_log: true\u0027 was specified for this result\", \"changed\": false}\n\n[4] https://e7fbb04e1f1bc4fc681d-d7024b4a9df5a3cf5a52965a2b5a469a.ssl.cf1.rackcdn.com/834352/81/gate/tripleo-ci-centos-9-scenario004-standalone/574cdfb/logs/undercloud/var/log/extra/journal.txt\n\n[5] \nJun 14 08:15:45 standalone.localdomain useradd[31969]: new group: name\u003dceph-admin, GID\u003d1002\nJun 14 08:15:45 standalone.localdomain useradd[31969]: new user: name\u003dceph-admin, UID\u003d1001, GID\u003d1002, home\u003d/home/ceph-admin, shell\u003d/bin/bash, from\u003dnone\nJun 14 08:15:45 standalone.localdomain useradd[31970]: failed adding user \u0027ceph-admin\u0027, exit code: 9\n\n[6] https://linux.die.net/man/8/useradd\n\n[7] https://e7fbb04e1f1bc4fc681d-d7024b4a9df5a3cf5a52965a2b5a469a.ssl.cf1.rackcdn.com/834352/81/gate/tripleo-ci-centos-9-scenario004-standalone/574cdfb/job-output.txt\n\n[8] https://github.com/ansible/ansible/blob/v2.8.20/lib/ansible/modules/system/user.py#L848\n\n[9] https://github.com/ansible/ansible/blob/v2.13.0/lib/ansible/modules/user.py","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d9e02362dcba18cbd24459599538c0edad253b54","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":81,"id":"cb21bc38_d89f21d4","in_reply_to":"e7b2cdd8_d21c52ff","updated":"2022-06-03 11:33:34.000000000","message":"No, it won\u0027t. Recommended merge order: 834352 and then 834354\n\n834354 is really just a clean up of THT parameters for after this patch lands.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"}],"roles/standalone/tasks/ceph-install.yml":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"aa9169b77d01e54ebc3f3c5b380d65f889170974","unresolved":true,"context_lines":[{"line_number":2,"context_line":"- name: Set facts for Ceph"},{"line_number":3,"context_line":"  set_fact:"},{"line_number":4,"context_line":"    stack: standalone"},{"line_number":5,"context_line":"    mon_ip: \"{{ hostvars[inventory_hostname][\u0027ansible_default_ipv4\u0027][\u0027address\u0027] }}\""},{"line_number":6,"context_line":"    osd_spec: \"{{ working_dir }}/osd_spec.yaml\""},{"line_number":7,"context_line":"    initial_ceph_conf: \"{{ working_dir }}/initial_ceph.conf\""},{"line_number":8,"context_line":"    ceph_spec: \"{{ working_dir }}/ceph_spec.yaml\""}],"source_content_type":"text/x-yaml","patch_set":1,"id":"b8501c80_52c02132","line":5,"range":{"start_line":5,"start_character":4,"end_line":5,"end_character":83},"updated":"2022-03-18 20:06:44.000000000","message":"Because the 192.168.24.0/24 network containing the cloud IP and VIP are not configured until \u0027openstack tripleo deploy\u0027 is run, we need Ceph to run on a different network if we\u0027re going to deploy Ceph before the overcloud. Any IP on the VM may be used including the IP on the default libvirt network 192.168.122.0/24.","commit_id":"d90ddde01fa7628c498721508d2622d9173811f4"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"72fc1cc28433b7a682d071c5ef5e128da734a7b4","unresolved":false,"context_lines":[{"line_number":2,"context_line":"- name: Set facts for Ceph"},{"line_number":3,"context_line":"  set_fact:"},{"line_number":4,"context_line":"    stack: standalone"},{"line_number":5,"context_line":"    mon_ip: \"{{ hostvars[inventory_hostname][\u0027ansible_default_ipv4\u0027][\u0027address\u0027] }}\""},{"line_number":6,"context_line":"    osd_spec: \"{{ working_dir }}/osd_spec.yaml\""},{"line_number":7,"context_line":"    initial_ceph_conf: \"{{ working_dir }}/initial_ceph.conf\""},{"line_number":8,"context_line":"    ceph_spec: \"{{ working_dir }}/ceph_spec.yaml\""}],"source_content_type":"text/x-yaml","patch_set":1,"id":"f04bf677_862fd5e0","line":5,"range":{"start_line":5,"start_character":4,"end_line":5,"end_character":83},"in_reply_to":"b8501c80_52c02132","updated":"2022-05-17 14:34:15.000000000","message":"In PS61 I\u0027m creating a separate storage network in advance with a dummy interface.\n\nhttps://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/59..61/roles/standalone/tasks/storage-network.yml","commit_id":"d90ddde01fa7628c498721508d2622d9173811f4"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"f9d982ba6af649c63376a997d2e3c555ead25239","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"- name: Create OSD spec file"},{"line_number":12,"context_line":"  copy:"},{"line_number":13,"context_line":"   dest: \"{{ osd_spec }}\""},{"line_number":14,"context_line":"   content: |"},{"line_number":15,"context_line":"     data_devices:"},{"line_number":16,"context_line":"       paths:"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"407a1df3_5d1b1dc5","line":13,"in_reply_to":"bd47ebe0_94761a95","updated":"2022-03-18 20:38:14.000000000","message":"\u003e linters: [error] wrong indentation: expected 4 but found 3 (indentation)\n\nDone.","commit_id":"d90ddde01fa7628c498721508d2622d9173811f4"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"f9d982ba6af649c63376a997d2e3c555ead25239","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"- name: Create initial ceph.conf file"},{"line_number":20,"context_line":"  copy:"},{"line_number":21,"context_line":"   dest: \"{{ initial_ceph_conf }}\""},{"line_number":22,"context_line":"   content: |"},{"line_number":23,"context_line":"     # passing initial conf has side effect of not"},{"line_number":24,"context_line":"     # setting public/cluster networks which is good"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"900c2cb4_5515a584","line":21,"in_reply_to":"0084c396_9e7662b7","updated":"2022-03-18 20:38:14.000000000","message":"\u003e linters: [error] wrong indentation: expected 4 but found 3 (indentation)\n\nDone.","commit_id":"d90ddde01fa7628c498721508d2622d9173811f4"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"14683c147c39546f57afcb1c8951cc006e6d0156","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"50ff29b1_546c92a5","line":82,"updated":"2022-03-18 20:15:29.000000000","message":"tripleo_ceph_deploy_container_image_prepare: \"{{ working_dir }}/containers-prepare-parameters.yaml\"","commit_id":"d90ddde01fa7628c498721508d2622d9173811f4"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"f9d982ba6af649c63376a997d2e3c555ead25239","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"df88e2f8_9b90d35f","line":82,"in_reply_to":"50ff29b1_546c92a5","updated":"2022-03-18 20:38:14.000000000","message":"Done","commit_id":"d90ddde01fa7628c498721508d2622d9173811f4"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"45fcbf4bcb516061f7507ed2ec734a52714c2cfc","unresolved":true,"context_lines":[{"line_number":2,"context_line":"- name: Set facts for Ceph"},{"line_number":3,"context_line":"  set_fact:"},{"line_number":4,"context_line":"    stack: standalone"},{"line_number":5,"context_line":"    # mon_ip: \"{{ hostvars[inventory_hostname][\u0027ansible_default_ipv4\u0027][\u0027address\u0027] }}\""},{"line_number":6,"context_line":"    mon_ip: 127.0.0.2"},{"line_number":7,"context_line":"    osd_spec: \"{{ working_dir }}/osd_spec.yaml\""},{"line_number":8,"context_line":"    initial_ceph_conf: \"{{ working_dir }}/initial_ceph.conf\""},{"line_number":9,"context_line":"    ceph_spec: \"{{ working_dir }}/ceph_spec.yaml\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"1aa5c7da_1c8217de","line":6,"range":{"start_line":5,"start_character":0,"end_line":6,"end_character":21},"updated":"2022-03-21 15:40:05.000000000","message":"Both mon_ip choices result in this failure\n\nERROR: Cannot infer CIDR network for mon IP `127.0.0.2`\nERROR: Cannot infer CIDR network for mon IP `158.69.73.214`\n\nhttps://github.com/ceph/ceph/blob/master/src/cephadm/cephadm#L4448","commit_id":"e57ce8f3b25b61590be584f75f3e31703dd0cd29"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"23758c5aa48202beee63b5a3eb0d9e00d97fb056","unresolved":false,"context_lines":[{"line_number":2,"context_line":"- name: Set facts for Ceph"},{"line_number":3,"context_line":"  set_fact:"},{"line_number":4,"context_line":"    stack: standalone"},{"line_number":5,"context_line":"    # mon_ip: \"{{ hostvars[inventory_hostname][\u0027ansible_default_ipv4\u0027][\u0027address\u0027] }}\""},{"line_number":6,"context_line":"    mon_ip: 127.0.0.2"},{"line_number":7,"context_line":"    osd_spec: \"{{ working_dir }}/osd_spec.yaml\""},{"line_number":8,"context_line":"    initial_ceph_conf: \"{{ working_dir }}/initial_ceph.conf\""},{"line_number":9,"context_line":"    ceph_spec: \"{{ working_dir }}/ceph_spec.yaml\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"e9aeeb8d_0eb683d2","line":6,"range":{"start_line":5,"start_character":0,"end_line":6,"end_character":21},"in_reply_to":"1aa5c7da_1c8217de","updated":"2022-04-07 18:10:04.000000000","message":"I can avoid this issue by passing this:\n\n  tripleo_ceph_deploy_cephadm_extra_args: \"--skip-mon-network\"\n\nCeph is still able to read/write data to pass the tempest test when the above is used.","commit_id":"e57ce8f3b25b61590be584f75f3e31703dd0cd29"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"22a883e371795ea0501d357ca910ba99eaf9fec4","unresolved":true,"context_lines":[{"line_number":58,"context_line":"    tripleo_ceph_user_debug: true"},{"line_number":59,"context_line":"    tripleo_ceph_user_generate_scripts: true"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"- name: Print all available facts (remove me)"},{"line_number":62,"context_line":"  ansible.builtin.debug:"},{"line_number":63,"context_line":"    var: ansible_facts"},{"line_number":64,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":10,"id":"b7571ae9_0afa03ba","line":61,"range":{"start_line":61,"start_character":8,"end_line":61,"end_character":33},"updated":"2022-03-22 15:32:29.000000000","message":"As per the ansible facts there is no broadcast. \n\n https://paste.opendev.org/show/bdaZnrUs5860hkskYaMp","commit_id":"27bce95ab88bb8f64c7628f5f524d69d52d0c604"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"23758c5aa48202beee63b5a3eb0d9e00d97fb056","unresolved":false,"context_lines":[{"line_number":58,"context_line":"    tripleo_ceph_user_debug: true"},{"line_number":59,"context_line":"    tripleo_ceph_user_generate_scripts: true"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"- name: Print all available facts (remove me)"},{"line_number":62,"context_line":"  ansible.builtin.debug:"},{"line_number":63,"context_line":"    var: ansible_facts"},{"line_number":64,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":10,"id":"9ac8b748_0c8c9c6d","line":61,"range":{"start_line":61,"start_character":8,"end_line":61,"end_character":33},"in_reply_to":"b7571ae9_0afa03ba","updated":"2022-04-07 18:10:04.000000000","message":"I can avoid this issue by passing this:\n\n  tripleo_ceph_deploy_cephadm_extra_args: \"--skip-mon-network\"\n\nCeph is still able to read/write data to pass the tempest test when the above is used.","commit_id":"27bce95ab88bb8f64c7628f5f524d69d52d0c604"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"22a883e371795ea0501d357ca910ba99eaf9fec4","unresolved":true,"context_lines":[{"line_number":79,"context_line":"    tripleo_ceph_deploy_config: \"{{ initial_ceph_conf }}\""},{"line_number":80,"context_line":"    tripleo_ceph_deploy_output: \"{{ deployed_ceph }}\""},{"line_number":81,"context_line":"    tripleo_ceph_deploy_container_image_prepare: \"{{ containers_prep }}\""},{"line_number":82,"context_line":"    tripleo_ceph_deploy_cephadm_extra_args: \"--log-to-file --skip-mon-network\""},{"line_number":83,"context_line":"    tripleo_ceph_deploy_force: true"},{"line_number":84,"context_line":"    tripleo_ceph_deploy_become: true"},{"line_number":85,"context_line":"    tripleo_ceph_deploy_overwrite: true"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"0181cbfd_668fc9ef","line":82,"range":{"start_line":82,"start_character":59,"end_line":82,"end_character":77},"updated":"2022-03-22 15:32:29.000000000","message":"Because there\u0027s no broadcast, I think I\u0027m running into this:\n\n  https://tracker.ceph.com/issues/50688#change-196200\n\nLet\u0027s trying --skip-mon-network to let bootstrap complete for now.","commit_id":"27bce95ab88bb8f64c7628f5f524d69d52d0c604"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"23758c5aa48202beee63b5a3eb0d9e00d97fb056","unresolved":false,"context_lines":[{"line_number":79,"context_line":"    tripleo_ceph_deploy_config: \"{{ initial_ceph_conf }}\""},{"line_number":80,"context_line":"    tripleo_ceph_deploy_output: \"{{ deployed_ceph }}\""},{"line_number":81,"context_line":"    tripleo_ceph_deploy_container_image_prepare: \"{{ containers_prep }}\""},{"line_number":82,"context_line":"    tripleo_ceph_deploy_cephadm_extra_args: \"--log-to-file --skip-mon-network\""},{"line_number":83,"context_line":"    tripleo_ceph_deploy_force: true"},{"line_number":84,"context_line":"    tripleo_ceph_deploy_become: true"},{"line_number":85,"context_line":"    tripleo_ceph_deploy_overwrite: true"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"801ba9f8_ae2af4bf","line":82,"range":{"start_line":82,"start_character":59,"end_line":82,"end_character":77},"in_reply_to":"0181cbfd_668fc9ef","updated":"2022-04-07 18:10:04.000000000","message":"I can avoid this issue by passing this:\n\n  tripleo_ceph_deploy_cephadm_extra_args: \"--skip-mon-network\"\n\nCeph is still able to read/write data to pass the tempest test when the above is used.","commit_id":"27bce95ab88bb8f64c7628f5f524d69d52d0c604"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"289ceb9d033a30876a3d1e1541a182b139214e22","unresolved":true,"context_lines":[{"line_number":92,"context_line":""},{"line_number":93,"context_line":"- name: Deploy Ceph via tripleo-operator-ansible"},{"line_number":94,"context_line":"  collections:"},{"line_number":95,"context_line":"    - tripleo.operator"},{"line_number":96,"context_line":"  include_role:"},{"line_number":97,"context_line":"    name: tripleo_ceph_deploy"},{"line_number":98,"context_line":"  vars:"}],"source_content_type":"text/x-yaml","patch_set":58,"id":"af92c223_4429ec74","line":95,"range":{"start_line":95,"start_character":6,"end_line":95,"end_character":22},"updated":"2022-05-05 06:56:16.000000000","message":"quick question: do we have a log of the output of the bootstrap process?","commit_id":"53868f3f9d4dcae90aa4a582470b1120871e5076"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"327412c5d8dc57120106d2dd1cda706a9c8f5f6b","unresolved":true,"context_lines":[{"line_number":92,"context_line":""},{"line_number":93,"context_line":"- name: Deploy Ceph via tripleo-operator-ansible"},{"line_number":94,"context_line":"  collections:"},{"line_number":95,"context_line":"    - tripleo.operator"},{"line_number":96,"context_line":"  include_role:"},{"line_number":97,"context_line":"    name: tripleo_ceph_deploy"},{"line_number":98,"context_line":"  vars:"}],"source_content_type":"text/x-yaml","patch_set":58,"id":"c7408aca_543d9191","line":95,"range":{"start_line":95,"start_character":6,"end_line":95,"end_character":22},"in_reply_to":"af92c223_4429ec74","updated":"2022-05-05 14:31:24.000000000","message":"Yes, the ansible role is called by quickstart and logged here:\n\n  https://f95bab1ced261b8e04a3-4e7b2e69a2914bf326001f07747bad73.ssl.cf5.rackcdn.com/834352/58/check/tripleo-ci-centos-9-scenario001-standalone/801764d/logs/quickstart_install.log\n\nIt actually bootstrapped fine and failed here:\n\n  TASK | Create Deployed Ceph Heat Environment file to pass to overcloud deploy\\nAn exception occurred during task execution. To see the full traceback, use -vvv. The error was: ansible.errors.AnsibleUndefinedVariable: \u0027tripleo_cephadm_pools\u0027 is undefined\\n2022-05-04 10:26:45.604900 | fa163e5d-6600-4fda-4008-00000000014d |      FATAL | Create Deployed Ceph Heat Environment file to pass to overcloud deploy | standalone.localdomain -\u003e localhost | error\u003d{\\\"changed\\\": false, \\\"msg\\\": \\\"AnsibleUndefinedVariable: \u0027tripleo_cephadm_pools\u0027 is undefined\\\"\n\nLooks like I need your new patch:\n\n  https://review.opendev.org/c/openstack/tripleo-ansible/+/840449","commit_id":"53868f3f9d4dcae90aa4a582470b1120871e5076"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"72fc1cc28433b7a682d071c5ef5e128da734a7b4","unresolved":false,"context_lines":[{"line_number":92,"context_line":""},{"line_number":93,"context_line":"- name: Deploy Ceph via tripleo-operator-ansible"},{"line_number":94,"context_line":"  collections:"},{"line_number":95,"context_line":"    - tripleo.operator"},{"line_number":96,"context_line":"  include_role:"},{"line_number":97,"context_line":"    name: tripleo_ceph_deploy"},{"line_number":98,"context_line":"  vars:"}],"source_content_type":"text/x-yaml","patch_set":58,"id":"8b2469e0_265f21a7","line":95,"range":{"start_line":95,"start_character":6,"end_line":95,"end_character":22},"in_reply_to":"c7408aca_543d9191","updated":"2022-05-17 14:34:15.000000000","message":"The relevant patch has merged so I\u0027m resolving this issue in the review.","commit_id":"53868f3f9d4dcae90aa4a582470b1120871e5076"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"4c25daa0bda919cd40bb79e2671df5ca350e99ea","unresolved":true,"context_lines":[{"line_number":41,"context_line":"  copy:"},{"line_number":42,"context_line":"    dest: \"{{ initial_ceph_conf }}\""},{"line_number":43,"context_line":"    content: |"},{"line_number":44,"context_line":"      # passing initial conf has side effect of not"},{"line_number":45,"context_line":"      # setting public/cluster networks which is good"},{"line_number":46,"context_line":"      # for standlone as ctlplane network is default"},{"line_number":47,"context_line":"      # but it is not yet configured"},{"line_number":48,"context_line":"      [global]"},{"line_number":49,"context_line":"      osd pool default size \u003d 1"},{"line_number":50,"context_line":"      [mon]"}],"source_content_type":"text/x-yaml","patch_set":59,"id":"1ec67d95_f5584018","line":47,"range":{"start_line":44,"start_character":0,"end_line":47,"end_character":36},"updated":"2022-05-11 14:58:31.000000000","message":"Need to find a way to pass the networks through to the Heat deployment. Else, we get a spec for RGW with the ctlplane network though the rest of Ceph is using the  mon_ip. \n\nE.g. we see this in the tripleo-ansible cephadm call log [1] becaue cephadm-extra-vars-heat.yml [2] has the ctlplane which is the default.\n\n        \"service_type: rgw\",\n        \"service_id: rgw\",\n        \"service_name: rgw.rgw\",\n        \"placement:\",\n        \"  hosts:\",\n        \"  - standalone.localdomain\",\n        \"networks:\",\n        \"- 192.168.24.0/24\",\n        \"spec:\",\n        \"  rgw_frontend_port: 8080\",\n        \"  rgw_realm: default\",\n        \"  rgw_zone: default\"\n\n\n\n[1]  https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-mtf1kj6p/cephadm/cephadm_command.log\n\n[2] https://dc57d77492030d7db0b0-2344f8d71749bd5aac75027308191d87.ssl.cf2.rackcdn.com/834352/59/check/tripleo-ci-centos-9-scenario004-standalone/6c9754d/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-mtf1kj6p/cephadm/cephadm-extra-vars-heat.yml","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"ee115c6e7cfaa199bc07a3af47d92c566a0aadf8","unresolved":true,"context_lines":[{"line_number":41,"context_line":"  copy:"},{"line_number":42,"context_line":"    dest: \"{{ initial_ceph_conf }}\""},{"line_number":43,"context_line":"    content: |"},{"line_number":44,"context_line":"      # passing initial conf has side effect of not"},{"line_number":45,"context_line":"      # setting public/cluster networks which is good"},{"line_number":46,"context_line":"      # for standlone as ctlplane network is default"},{"line_number":47,"context_line":"      # but it is not yet configured"},{"line_number":48,"context_line":"      [global]"},{"line_number":49,"context_line":"      osd pool default size \u003d 1"},{"line_number":50,"context_line":"      [mon]"}],"source_content_type":"text/x-yaml","patch_set":59,"id":"b59e7c42_e3526d85","line":47,"range":{"start_line":44,"start_character":0,"end_line":47,"end_character":36},"in_reply_to":"1ec67d95_f5584018","updated":"2022-05-11 15:08:44.000000000","message":"mmmm yeah, we need to close the loop here because we\u0027re using a different network, while the old method was supposed on rely on what heat provided.\nThe rgw network (for the spec) is passed in [1] and it comes from heat service map [2].\nMight overriding the CephRgwNetwork work?\nThe other way could be pushing the ansible extra value that overrides the value generated by heat, but not 100% sure which is the working approach.\n\n[1] https://github.com/openstack/tripleo-ansible/blob/master/tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml#L53\n\n[2] https://github.com/openstack/tripleo-heat-templates/blob/master/deployment/cephadm/ceph-rgw.yaml#L78","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"72fc1cc28433b7a682d071c5ef5e128da734a7b4","unresolved":false,"context_lines":[{"line_number":41,"context_line":"  copy:"},{"line_number":42,"context_line":"    dest: \"{{ initial_ceph_conf }}\""},{"line_number":43,"context_line":"    content: |"},{"line_number":44,"context_line":"      # passing initial conf has side effect of not"},{"line_number":45,"context_line":"      # setting public/cluster networks which is good"},{"line_number":46,"context_line":"      # for standlone as ctlplane network is default"},{"line_number":47,"context_line":"      # but it is not yet configured"},{"line_number":48,"context_line":"      [global]"},{"line_number":49,"context_line":"      osd pool default size \u003d 1"},{"line_number":50,"context_line":"      [mon]"}],"source_content_type":"text/x-yaml","patch_set":59,"id":"32a75420_c2835f8d","line":47,"range":{"start_line":44,"start_character":0,"end_line":47,"end_character":36},"in_reply_to":"2b0d2547_4fe59649","updated":"2022-05-17 14:34:15.000000000","message":"Using a combination of the two approaches with PS61\n\nhttps://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/59..61/roles/standalone/tasks/ceph-install.yml","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"6914840d41f0ec00e93b79c5816adf57c13d8e22","unresolved":true,"context_lines":[{"line_number":41,"context_line":"  copy:"},{"line_number":42,"context_line":"    dest: \"{{ initial_ceph_conf }}\""},{"line_number":43,"context_line":"    content: |"},{"line_number":44,"context_line":"      # passing initial conf has side effect of not"},{"line_number":45,"context_line":"      # setting public/cluster networks which is good"},{"line_number":46,"context_line":"      # for standlone as ctlplane network is default"},{"line_number":47,"context_line":"      # but it is not yet configured"},{"line_number":48,"context_line":"      [global]"},{"line_number":49,"context_line":"      osd pool default size \u003d 1"},{"line_number":50,"context_line":"      [mon]"}],"source_content_type":"text/x-yaml","patch_set":59,"id":"2b0d2547_4fe59649","line":47,"range":{"start_line":44,"start_character":0,"end_line":47,"end_character":36},"in_reply_to":"b59e7c42_e3526d85","updated":"2022-05-13 13:03:01.000000000","message":"This issue is something that only happens in standalone deployments as a result of not configuring the network first. In multinode deployments the network would already be configured and --network-file network_data.yaml would be passed to both `openstack overcloud ceph deploy` and `openstack overcloud deploy` and we\u0027d have a consistent network source.\n\nWhat\u0027s happening in this job is that I used an initial ceph.conf which has a side effect I\u0027m taking advantage of because it results in the public/cluster networks not getting the default (192.168.24.0/24). This is basically a workaround which worked fine for RBD but we now see it doesn\u0027t work for RGW.\n\nOne solution is to pass an override which tells the overcloud to use the network the mon_ip is on as the storage network. I.e. I could create an additional environment file using the address from:\n\n    hostvars[inventory_hostname][\u0027ansible_default_ipv4\u0027][\u0027address\u0027]\n\nand override these with that address in an extra THT file:\n\n    CephClusterNetwork: {{ _service_nets.get(\u0027storage_mgmt\u0027, \u0027ctlplane\u0027) }}\n    CephMonNetwork: {{ _service_nets.get(\u0027storage\u0027, \u0027ctlplane\u0027) }}\n    CephRgwNetwork: {{ _service_nets.get(\u0027storage\u0027, \u0027ctlplane\u0027) }}\n\nOR I might prefer to configure a deployment with two networks:\n\n192.168.24.0/24 all services\n192.168.42.0/24 storage network\n\nI\u0027d then create a network_data.yaml file with the above two networks and pass it to both deployments with --network-data.\n\nSo the 42 network would be created by ansible in ceph-install.yaml outside of tripleo which would simulate the actions of networkv2.\n\nI\u0027m not sure though if standalone will work as desired when passed --network-data.","commit_id":"7271055b6ac9b8d7cf3f38f68603e4610d18ab21"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"714d79335502b4e7620a4c34de3e3c7b46d12317","unresolved":true,"context_lines":[{"line_number":130,"context_line":"    deployed_ceph_dict: \"{{ (lookup(\u0027file\u0027, deployed_ceph) |"},{"line_number":131,"context_line":"                             from_yaml).parameter_defaults }}\""},{"line_number":132,"context_line":"  loop:"},{"line_number":133,"context_line":"    - \u0027CephClusterNetwork\u0027"},{"line_number":134,"context_line":"    - \u0027CephMonNetwork\u0027"},{"line_number":135,"context_line":"    - \u0027CephRgwNetwork\u0027"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Write deployed_ceph dictionary to Heat environment file"},{"line_number":138,"context_line":"  copy:"}],"source_content_type":"text/x-yaml","patch_set":61,"id":"7b6df9d6_9489703c","line":135,"range":{"start_line":133,"start_character":0,"end_line":135,"end_character":22},"updated":"2022-05-17 14:39:48.000000000","message":"I\u0027m doing this outside of the project and in the CI only since \"real\" deployments will use --network-data\nMaybe though I should add a --heat-overrides list of key/value pairs","commit_id":"7bf76bf1d5fa3cc3957a6fba7f656ac1b292627f"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"b19eb667317b5a4cb5999ba812b90d76923cc6a8","unresolved":false,"context_lines":[{"line_number":130,"context_line":"    deployed_ceph_dict: \"{{ (lookup(\u0027file\u0027, deployed_ceph) |"},{"line_number":131,"context_line":"                             from_yaml).parameter_defaults }}\""},{"line_number":132,"context_line":"  loop:"},{"line_number":133,"context_line":"    - \u0027CephClusterNetwork\u0027"},{"line_number":134,"context_line":"    - \u0027CephMonNetwork\u0027"},{"line_number":135,"context_line":"    - \u0027CephRgwNetwork\u0027"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Write deployed_ceph dictionary to Heat environment file"},{"line_number":138,"context_line":"  copy:"}],"source_content_type":"text/x-yaml","patch_set":61,"id":"cd65c3e6_6991fcd4","line":135,"range":{"start_line":133,"start_character":0,"end_line":135,"end_character":22},"in_reply_to":"7b6df9d6_9489703c","updated":"2022-06-01 21:13:07.000000000","message":"Using --network-data in standalone too","commit_id":"7bf76bf1d5fa3cc3957a6fba7f656ac1b292627f"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"58f9a53f9d124223c11e5de325e4fb0f151d5d8d","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- block:"},{"line_number":3,"context_line":"    - name: Set facts for Ceph"},{"line_number":4,"context_line":"      set_fact:"},{"line_number":5,"context_line":"        stack: standalone"}],"source_content_type":"text/x-yaml","patch_set":68,"id":"0962d904_072928ec","line":2,"range":{"start_line":2,"start_character":2,"end_line":2,"end_character":7},"updated":"2022-05-18 10:50:02.000000000","message":"+1 on using block here","commit_id":"6af1644c444ecfc92e2975ab1264ab36b6cb9d39"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"c7bffb5faa65f90012a26d18da2685c333a55b11","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- block:"},{"line_number":3,"context_line":"    - name: Set facts for Ceph"},{"line_number":4,"context_line":"      set_fact:"},{"line_number":5,"context_line":"        stack: standalone"}],"source_content_type":"text/x-yaml","patch_set":68,"id":"de444e16_4deb3b0a","line":2,"range":{"start_line":2,"start_character":2,"end_line":2,"end_character":7},"in_reply_to":"0962d904_072928ec","updated":"2022-05-18 13:04:34.000000000","message":"I did that so the var \"cidr: 24\" would be accessible in both.","commit_id":"6af1644c444ecfc92e2975ab1264ab36b6cb9d39"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"48ace5c0998b8211164874eee60f817110ad178f","unresolved":true,"context_lines":[{"line_number":43,"context_line":"        backup: false"},{"line_number":44,"context_line":"        section: \u0027[registry]\u0027"},{"line_number":45,"context_line":"        option: insecure"},{"line_number":46,"context_line":"        value: \u0027true\u0027"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"- name: Create OSD spec file"},{"line_number":49,"context_line":"  copy:"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"16b1f119_75300ce3","line":46,"updated":"2022-06-06 07:07:08.000000000","message":"ack makes sense though I thought we already do this \n\nhaving said that it looks good at https://9ff478a1c949d0bcef31-9c9e874f2700cd2af0f206ec3ca6f5f9.ssl.cf5.rackcdn.com/834354/3/check/tripleo-ci-centos-9-scenario001-standalone/2fb4a32/logs/undercloud/etc/containers/registries.conf\n\n\njust wondering though if this is a duplicate? I assume you had to do this in your env/test ... I pulled up another scen1 job for comparison [1] and it also appears to already have these set.\n\n[1] https://4622e0f521fe7ef77d9d-3fa6c7b03298e65764ba8baf4a8a5f26.ssl.cf1.rackcdn.com/844442/3/check/tripleo-ci-centos-9-scenario001-standalone/b6b6843/logs/undercloud/etc/containers/registries.conf","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"dcb1ace435d6c97588e48f44bc3b9a5a2849875f","unresolved":true,"context_lines":[{"line_number":43,"context_line":"        backup: false"},{"line_number":44,"context_line":"        section: \u0027[registry]\u0027"},{"line_number":45,"context_line":"        option: insecure"},{"line_number":46,"context_line":"        value: \u0027true\u0027"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"- name: Create OSD spec file"},{"line_number":49,"context_line":"  copy:"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"39d88cb1_59a61bc7","line":46,"in_reply_to":"16b1f119_75300ce3","updated":"2022-06-06 11:34:47.000000000","message":"\u003e ack makes sense though I thought we already do this \n\u003e \n\u003e having said that it looks good at https://9ff478a1c949d0bcef31-9c9e874f2700cd2af0f206ec3ca6f5f9.ssl.cf5.rackcdn.com/834354/3/check/tripleo-ci-centos-9-scenario001-standalone/2fb4a32/logs/undercloud/etc/containers/registries.conf\n\u003e\n\u003e \n\u003e just wondering though if this is a duplicate? I assume you had to do this in your env/test ... I pulled up another scen1 job for comparison [1] and it also appears to already have these set.\n\nI had to do this not in my environment but in the CI environment to make the job green.\n\nYes, it\u0027s true the overcloud deployment does this, but WHEN does that modification of registries.conf happen? It happens _during_ overcloud deployment which won\u0027t help the ceph deployment as the ceph deployment happens before overcloud deployment and it also needs the container modification so it can pull the local image.\n\nNow, you might ask, \"is it a bug that we do this before the ceph deployment\"? In normal circumstances, no because we have this:\n\n  https://review.opendev.org/q/topic:bug%252F1958524\n\nHowever, because we are hacking the registries.conf with the containers.yml [1] to use our local mirror, the resultant registries.conf is incompatible with the process above. Thus, in this job I pass --skip-container-registry-config because I modify it directly with these Ansible tasks.\n\n[1] https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/81/roles/standalone/tasks/main.yml#2","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"3a07dac7e9f6fa15e109f9f3f4d6f3e65a09344c","unresolved":false,"context_lines":[{"line_number":43,"context_line":"        backup: false"},{"line_number":44,"context_line":"        section: \u0027[registry]\u0027"},{"line_number":45,"context_line":"        option: insecure"},{"line_number":46,"context_line":"        value: \u0027true\u0027"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"- name: Create OSD spec file"},{"line_number":49,"context_line":"  copy:"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"8e322115_71d4b64c","line":46,"in_reply_to":"39d88cb1_59a61bc7","updated":"2022-06-07 06:55:27.000000000","message":"ok thank you for the explanation makes sense","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":29775,"name":"Sandeep Yadav","email":"sandyada@redhat.com","username":"sandeepyadav93"},"change_message_id":"f663d3ad3e62eec4e7d8ae42d26aaf70ee041fe7","unresolved":true,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"- name: Deploy Ceph via tripleo-operator-ansible"},{"line_number":106,"context_line":"  collections:"},{"line_number":107,"context_line":"    - tripleo.operator"},{"line_number":108,"context_line":"  include_role:"},{"line_number":109,"context_line":"    name: tripleo_ceph_deploy"},{"line_number":110,"context_line":"  vars:"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"c0963ca0_1419d78f","line":107,"range":{"start_line":107,"start_character":0,"end_line":107,"end_character":22},"updated":"2022-06-07 13:49:58.000000000","message":"Where are we logging incase something fails during ceph deploy?","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"def3b3051164b778bfe03d74f3de71e2f168140b","unresolved":true,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"- name: Deploy Ceph via tripleo-operator-ansible"},{"line_number":106,"context_line":"  collections:"},{"line_number":107,"context_line":"    - tripleo.operator"},{"line_number":108,"context_line":"  include_role:"},{"line_number":109,"context_line":"    name: tripleo_ceph_deploy"},{"line_number":110,"context_line":"  vars:"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"3e640471_f0b2bd7f","line":107,"range":{"start_line":107,"start_character":0,"end_line":107,"end_character":22},"in_reply_to":"1fdace6a_3e40604f","updated":"2022-06-07 15:45:35.000000000","message":"The ceph deploy is logged to undercloud/home/zuul/ansible.log [1]\n\nSome tasks configure ceph after deployment during overcloud deployment and are logged in config-download\u0027s cephadm directory [2]\n\nThe ceph server logs (including the cephadm log) are in /var/log/ceph [3]\n\nI suppose the tripleo client could get a --deployment-log\u003dPATH option to set the ansible log env variable and then tripleo-operator-ansible could add a flag for that. \n\n[1] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_f2e/834352/81/check/tripleo-ci-centos-9-scenario004-standalone/f2ec5f8/logs/undercloud/home/zuul/ansible.log\n\n[2]\nhttps://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_f2e/834352/81/check/tripleo-ci-centos-9-scenario004-standalone/f2ec5f8/logs/undercloud/home/zuul/tripleo-deploy/standalone-ansible-c7qy63wh/cephadm/disable_cephadm.log\n\n[3] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_f2e/834352/81/check/tripleo-ci-centos-9-scenario004-standalone/f2ec5f8/logs/undercloud/var/log/ceph/index.html","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":29775,"name":"Sandeep Yadav","email":"sandyada@redhat.com","username":"sandeepyadav93"},"change_message_id":"74d82c32c5e73f6b4839f7249aec3e2fe3c68f3d","unresolved":true,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"- name: Deploy Ceph via tripleo-operator-ansible"},{"line_number":106,"context_line":"  collections:"},{"line_number":107,"context_line":"    - tripleo.operator"},{"line_number":108,"context_line":"  include_role:"},{"line_number":109,"context_line":"    name: tripleo_ceph_deploy"},{"line_number":110,"context_line":"  vars:"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"1fdace6a_3e40604f","line":107,"range":{"start_line":107,"start_character":0,"end_line":107,"end_character":22},"in_reply_to":"c0963ca0_1419d78f","updated":"2022-06-07 13:58:40.000000000","message":"https://github.com/openstack/tripleo-operator-ansible/tree/master/roles/tripleo_ceph_deploy\n\nhmm, I don\u0027t see an option in tripleo_ceph_deploy operator for logs.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"}],"roles/standalone/tasks/main.yml":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"1dcf44d8313d71770fbe2c1e9b8899284795c8fb","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: scenario_path + scenario_file in standalone_custom_env_files"},{"line_number":5,"context_line":"  vars:"},{"line_number":6,"context_line":"    scenario_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":7,"context_line":"    scenario_file: \u0027scenario001-standalone.yaml\u0027"}],"source_content_type":"text/x-yaml","patch_set":38,"id":"b756291f_a98af2e0","line":4,"updated":"2022-04-11 16:43:48.000000000","message":"Later I can change this to the following:\n\nwhen: (scenario_path + scn001 in standalone_custom_env_files or\n       scenario_path + scn004 in standalone_custom_env_files)\n\nI think we should see 001 stable for a little while and then switch 004. \n\nHowever, do we want to do this another way in the job definition instead? If so how?","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"c30614c7ae27b4589a3f25664056d441f4b5cdbc","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: scenario_path + scenario_file in standalone_custom_env_files"},{"line_number":5,"context_line":"  vars:"},{"line_number":6,"context_line":"    scenario_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":7,"context_line":"    scenario_file: \u0027scenario001-standalone.yaml\u0027"}],"source_content_type":"text/x-yaml","patch_set":38,"id":"4cf8e064_cd929dc7","line":4,"in_reply_to":"4785707b_0ff912f9","updated":"2022-04-19 21:00:00.000000000","message":"\u003e \u003e [EDIT2]: we may have to be a bit careful with that standalone_ceph it looks like its referenced in a few more places like https://codesearch.opendev.org/?q\u003dstandalone_ceph\u0026i\u003dnope\u0026literal\u003dnope\u0026files\u003d\u0026excludeFiles\u003d\u0026repos\u003d   https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/zuul.d/standalone-jobs.yaml#L566 scen12? scn10? those must be typos? i think we need to clear those up before you can safely use it\n\nI\u0027m cleaning up 012 \n\n  https://review.opendev.org/c/openstack/tripleo-ci/+/838557\n\n010 uses ceph\n\nhttps://github.com/openstack/tripleo-heat-templates/blob/master/ci/environments/scenario010-standalone.yaml","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"7b5e949640b2e73de904160289d3e3c1368f2560","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: scenario_path + scenario_file in standalone_custom_env_files"},{"line_number":5,"context_line":"  vars:"},{"line_number":6,"context_line":"    scenario_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":7,"context_line":"    scenario_file: \u0027scenario001-standalone.yaml\u0027"}],"source_content_type":"text/x-yaml","patch_set":38,"id":"10469416_c8ed2911","line":4,"in_reply_to":"4cf8e064_cd929dc7","updated":"2022-06-06 11:50:51.000000000","message":"Resolving this. 001/004/010 use Ceph and are passing here. 012 does not use Ceph so it has been removed.","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"dd4321b2b1fa0de9cd5f558da495839ff082283b","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: scenario_path + scenario_file in standalone_custom_env_files"},{"line_number":5,"context_line":"  vars:"},{"line_number":6,"context_line":"    scenario_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":7,"context_line":"    scenario_file: \u0027scenario001-standalone.yaml\u0027"}],"source_content_type":"text/x-yaml","patch_set":38,"id":"4785707b_0ff912f9","line":4,"in_reply_to":"b2725b94_fba0bc7d","updated":"2022-04-19 16:58:09.000000000","message":"\u003e [EDIT]: we already have something called \u0027standalone_ceph\u0027 and it is on the scen1/scen4 job definitions eg. there https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/zuul.d/standalone-jobs.yaml#L654 \n\u003e \n\u003e looks like its consumed for the loopback task in https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/roles/run-test/tasks/main.yaml#L63-L66\n\u003e \n\u003e can you try using that?\n\nYes, line 6 of PS40 will add it to the AND for now to confirm it\u0027s available. \n\n\u003e [EDIT2]: we may have to be a bit careful with that standalone_ceph it looks like its referenced in a few more places like https://codesearch.opendev.org/?q\u003dstandalone_ceph\u0026i\u003dnope\u0026literal\u003dnope\u0026files\u003d\u0026excludeFiles\u003d\u0026repos\u003d   https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/zuul.d/standalone-jobs.yaml#L566 scen12? scn10? those must be typos? i think we need to clear those up before you can safely use it\n\nThe following standalone_ceph true only for 001 and 004. Let\u0027s confirm it doesn\u0027t break anything.\n\nhttps://review.opendev.org/c/openstack/tripleo-ci/+/838557","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"4a6726f6480c2c633303fc58ae50a909782fa9cd","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: scenario_path + scenario_file in standalone_custom_env_files"},{"line_number":5,"context_line":"  vars:"},{"line_number":6,"context_line":"    scenario_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":7,"context_line":"    scenario_file: \u0027scenario001-standalone.yaml\u0027"}],"source_content_type":"text/x-yaml","patch_set":38,"id":"b2725b94_fba0bc7d","line":4,"in_reply_to":"b756291f_a98af2e0","updated":"2022-04-13 07:31:52.000000000","message":"yeah i mean we could even add a new var on the job definition if you want it? like when: \"zuul.var_name|default(false)\" \n\n[EDIT]: we already have something called \u0027standalone_ceph\u0027 and it is on the scen1/scen4 job definitions eg. there https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/zuul.d/standalone-jobs.yaml#L654 \n\nlooks like its consumed for the loopback task in https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/roles/run-test/tasks/main.yaml#L63-L66\n\ncan you try using that?\n\n\n[EDIT2]: we may have to be a bit careful with that standalone_ceph it looks like its referenced in a few more places like https://codesearch.opendev.org/?q\u003dstandalone_ceph\u0026i\u003dnope\u0026literal\u003dnope\u0026files\u003d\u0026excludeFiles\u003d\u0026repos\u003d   https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/zuul.d/standalone-jobs.yaml#L566 scen12? scn10? those must be typos? i think we need to clear those up before you can safely use it","commit_id":"bc33c6e91983e7c5b48f6bfe5ef2aec9e873c7ab"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"71ef0361cf7434800b1ef4c79094b117b6a553e7","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: standalone_ceph"},{"line_number":5,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":40,"id":"54bc7598_cf759480","line":4,"updated":"2022-04-19 18:31:31.000000000","message":"It seems that the standalone_ceph variable is not defined as per the most recent run [1]\n\n\n2022-04-19 18:22:54.688703 | primary | TASK [standalone : include_tasks] **********************************************\n2022-04-19 18:22:54.688753 | primary | Tuesday 19 April 2022  18:22:54 +0000 (0:00:00.036)       0:05:39.192 *********\n2022-04-19 18:22:54.709691 | primary | fatal: [undercloud]: FAILED! \u003d\u003e {\"msg\": \"The conditional check \u0027standalone_ceph\u0027 failed. The error was: error while evaluating conditional (standalone_ceph): \u0027standalone_ceph\u0027 is undefined\\n\\nThe error appears to be in \u0027/home/zuul/workspace/.quickstart/usr/local/share/ansible/roles/standalone/tasks/main.yml\u0027: line 3, column 3, but may\\nbe elsewhere in the file depending on the exact syntax problem.\\n\\nThe offending line appears to be:\\n\\n- include_tasks: containers.yml\\n- include_tasks: ceph-install.yml\\n  ^ here\\n\"}\n\n\n[1] https://zuul.opendev.org/t/openstack/build/457013b5296344afaf17898526d1daf0","commit_id":"4447b2ae83b171adfe6762afd80f4e746550ca68"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"0caee76cbdc39c6b91b4920a52f35d24ae4dd600","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: standalone_ceph"},{"line_number":5,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":40,"id":"fbd66880_48860fca","line":4,"in_reply_to":"37ba71b1_fe14e488","updated":"2022-04-20 14:12:56.000000000","message":"OK, PS42 uses it the same way. Let\u0027s see how it does. Thanks.","commit_id":"4447b2ae83b171adfe6762afd80f4e746550ca68"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"fb51472bbf9bbbdf7b8015e05a9ac59a271b41b9","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: standalone_ceph"},{"line_number":5,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":40,"id":"37ba71b1_fe14e488","line":4,"in_reply_to":"54bc7598_cf759480","updated":"2022-04-20 06:48:18.000000000","message":"can you try just \u0027standalone_ceph\u0027 without the \"zuul.\" bit (that should work not sure why it didn\u0027t).\n\nit is definitely there - you can see it in the zuul vars there https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_457/834352/40/check/tripleo-ci-centos-9-scenario001-standalone/457013b/zuul-info/inventory.yaml\n\n    standalone_ceph: true\n\n\nAs a reference that is how we use it in the ci play: https://opendev.org/openstack/tripleo-ci/src/commit/96e6db44ca886c1edd2a0a151ca16610a47342df/roles/run-test/tasks/main.yaml#L63-L66","commit_id":"4447b2ae83b171adfe6762afd80f4e746550ca68"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"5a32647f1369cc9244540f8db379e405dff972da","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when: standalone_ceph"},{"line_number":5,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":40,"id":"99d22daf_e3fa2a84","line":4,"in_reply_to":"fbd66880_48860fca","updated":"2022-04-21 13:41:20.000000000","message":"We\u0027re now using the desired boolean plus a branches regex_match.","commit_id":"4447b2ae83b171adfe6762afd80f4e746550ca68"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"2836bf24943dbda2da1731343d5775f385ee765b","unresolved":true,"context_lines":[{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - branches is defined and branches | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"}],"source_content_type":"text/x-yaml","patch_set":44,"id":"ae31fbb9_8558b53c","line":6,"updated":"2022-04-26 12:22:49.000000000","message":"we have zuul.branch - see https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_457/834352/40/check/tripleo-ci-centos-9-scenario001-standalone/457013b/zuul-info/inventory.yaml \n\n    zuul:\n...\n      artifacts:\n      - branch: master\n\nyou should be able to use zuul.branch e.g. we have it there as an example https://opendev.org/openstack/tripleo-ci/src/commit/15761b77d91ab3e398f6fa1d10d2f5da267b7931/roles/tripleo-repos/tasks/main.yml#L51","commit_id":"f3a137a12b4b29d4b6b45cb7d365b861fbfcbd42"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"487c7aaff5427598685b72b06e4143791ec72fdb","unresolved":false,"context_lines":[{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - branches is defined and branches | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"}],"source_content_type":"text/x-yaml","patch_set":44,"id":"d697399b_2e13d497","line":6,"in_reply_to":"ae31fbb9_8558b53c","updated":"2022-04-26 12:29:46.000000000","message":"Done","commit_id":"f3a137a12b4b29d4b6b45cb7d365b861fbfcbd42"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"b917cededfa416ffd2b2ba0c0472a956d0fa69d7","unresolved":true,"context_lines":[{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"}],"source_content_type":"text/x-yaml","patch_set":46,"id":"cfd78dfc_d1b51fb9","line":6,"range":{"start_line":6,"start_character":33,"end_line":6,"end_character":59},"updated":"2022-04-27 07:34:03.000000000","message":"could also be expressed as\n\nzuul.branch in [\u0027master\u0027] \n\nor even\n\nzuul.branch \u003d\u003d \u0027master\u0027","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"7b5e949640b2e73de904160289d3e3c1368f2560","unresolved":false,"context_lines":[{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"}],"source_content_type":"text/x-yaml","patch_set":46,"id":"619e4887_3ef19a72","line":6,"range":{"start_line":6,"start_character":33,"end_line":6,"end_character":59},"in_reply_to":"6906bcaa_b73970ae","updated":"2022-06-06 11:50:51.000000000","message":"Done","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"4df0134eda2500f8ac10bd4ab2648d08cc09f5d5","unresolved":true,"context_lines":[{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"}],"source_content_type":"text/x-yaml","patch_set":46,"id":"6906bcaa_b73970ae","line":6,"range":{"start_line":6,"start_character":33,"end_line":6,"end_character":59},"in_reply_to":"cfd78dfc_d1b51fb9","updated":"2022-04-27 10:49:05.000000000","message":"Yes, it could. As per the comment on line 4, I intend do this for master or wallaby in the future.\n\nI\u0027ll update it to be like this:\n\n- zuul.branch is defined and zuul.branch in [\u0027master\u0027] \n\nand in a follow up patch I\u0027ll switch it to this:\n\n- zuul.branch is defined and zuul.branch in [\u0027master\u0027, \u0027wallaby\u0027]","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"b917cededfa416ffd2b2ba0c0472a956d0fa69d7","unresolved":true,"context_lines":[{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":10,"context_line":"    scen_001: \u0027scenario001-standalone.yaml\u0027"},{"line_number":11,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":46,"id":"6360474d_21277a19","line":10,"range":{"start_line":7,"start_character":0,"end_line":10,"end_character":4},"updated":"2022-04-27 07:34:03.000000000","message":"i see.. so this whole thing will go away right? the conditional should be fine above.\n\nby the way, i added scen 4 triggering against the new file there https://review.opendev.org/c/openstack/tripleo-ci/+/839476\n\nif you add a depends-on we should see scen4 running in this change too","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"4df0134eda2500f8ac10bd4ab2648d08cc09f5d5","unresolved":true,"context_lines":[{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":10,"context_line":"    scen_001: \u0027scenario001-standalone.yaml\u0027"},{"line_number":11,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":46,"id":"8149884f_f78111fc","line":10,"range":{"start_line":7,"start_character":0,"end_line":10,"end_character":4},"in_reply_to":"6360474d_21277a19","updated":"2022-04-27 10:49:05.000000000","message":"Yes, as per the comment in line 8, lines 7-10 will be removed in a follow up patch.\n\nI\u0027ll remove lines 7-10 out for now so we can see the depends-on run for scen 4.\n\nWe can add them back if we wish to only change 001 with the first merge and then add 004 in a follow up. I thought we should only change one job at a time.","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d34de0f88eabb8b4d00a9beeb4336b031dfa1bcc","unresolved":false,"context_lines":[{"line_number":4,"context_line":"  when:  # TODO(fultonj) use regex_search(\u0027master|wallaby\u0027) after backport"},{"line_number":5,"context_line":"    - standalone_ceph is defined and standalone_ceph"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch | regex_search(\u0027master\u0027)"},{"line_number":7,"context_line":"    - scen_path + scen_001 in standalone_custom_env_files"},{"line_number":8,"context_line":"  vars:  # TODO(fultonj) remove scen_ checking when stable for 001,004,010"},{"line_number":9,"context_line":"    scen_path: \u0027/usr/share/openstack-tripleo-heat-templates/ci/environments/\u0027"},{"line_number":10,"context_line":"    scen_001: \u0027scenario001-standalone.yaml\u0027"},{"line_number":11,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":46,"id":"3bb59d3e_d8f5767f","line":10,"range":{"start_line":7,"start_character":0,"end_line":10,"end_character":4},"in_reply_to":"8149884f_f78111fc","updated":"2022-06-06 11:53:00.000000000","message":"At this point we can see 001/004/010 green. If we merge this as is we\u0027ll convert them all to the new method. If we want to do them one at a time, then I can add extra conditions. I\u0027m not sure it\u0027s beneficial to spread it out over time though.","commit_id":"04c46b2751c1096546edf9ba249491a0f19c12c0"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"4cdd9a55f307038dfc01b04c003e0994377b8ba3","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- debug:"},{"line_number":6,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""},{"line_number":7,"context_line":"  when: standalone_ceph is defined"},{"line_number":8,"context_line":"- debug:"},{"line_number":9,"context_line":"    msg: \"zuul.branch: {{ zuul.branch }}\""},{"line_number":10,"context_line":"  when: zuul.branch is defined"},{"line_number":11,"context_line":"# \u003c/remove\u003e"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":14,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"}],"source_content_type":"text/x-yaml","patch_set":48,"id":"3bda9d45_9473edfd","line":11,"range":{"start_line":4,"start_character":0,"end_line":11,"end_character":11},"updated":"2022-04-27 17:39:06.000000000","message":"My results [1] indicate that zuul.branch is defined but that standalone_ceph is NOT defined.\n\n[1] https://paste.opendev.org/show/bm65JnZI5qkxvXPWByJb","commit_id":"e1d87cf184c34a5e5db9a078076e2dd9baff6a99"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"fc3ed3d6df17aa19a2ad7bc933b03c4417500047","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- debug:"},{"line_number":6,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""},{"line_number":7,"context_line":"  when: standalone_ceph is defined"},{"line_number":8,"context_line":"- debug:"},{"line_number":9,"context_line":"    msg: \"zuul.branch: {{ zuul.branch }}\""},{"line_number":10,"context_line":"  when: zuul.branch is defined"},{"line_number":11,"context_line":"# \u003c/remove\u003e"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":14,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"}],"source_content_type":"text/x-yaml","patch_set":48,"id":"478c5168_35dcd068","line":11,"range":{"start_line":4,"start_character":0,"end_line":11,"end_character":11},"in_reply_to":"040c71e0_6a3653ae","updated":"2022-04-28 12:18:19.000000000","message":"As per line 7, casting to bool shouldn\u0027t make a difference since it\u0027s not even defined. However in PS49, I\u0027ve added |bool to line 22. I\u0027ll also save the content of all vars so we can see what ansible believes it has at this stage.\n\nhttps://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/48..49/roles/standalone/tasks/main.yml","commit_id":"e1d87cf184c34a5e5db9a078076e2dd9baff6a99"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"3dff448703511ae381f65d57e4b17661724decc9","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- debug:"},{"line_number":6,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""},{"line_number":7,"context_line":"  when: standalone_ceph is defined"},{"line_number":8,"context_line":"- debug:"},{"line_number":9,"context_line":"    msg: \"zuul.branch: {{ zuul.branch }}\""},{"line_number":10,"context_line":"  when: zuul.branch is defined"},{"line_number":11,"context_line":"# \u003c/remove\u003e"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":14,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"}],"source_content_type":"text/x-yaml","patch_set":48,"id":"040c71e0_6a3653ae","line":11,"range":{"start_line":4,"start_character":0,"end_line":11,"end_character":11},"in_reply_to":"0e3ec758_e87d200c","updated":"2022-04-28 12:01:40.000000000","message":"maybe we need to be more explicit (like is it reading it as a string?) - as you said you can see it in the zuul inventory\n\ne.g. \n \n  when: standalone_ceph is defined and standalone_ceph|bool\n\nlooking at where we use it, we do have the \u0027and standalone_ceph\u0027 bit (not the |bool but i think it cant hurt...) i wonder if that\u0027s it? https://opendev.org/openstack/tripleo-ci/src/commit/6830c9bf9da681d189ce90ff1a15dcca238a2e1d/roles/run-test/tasks/main.yaml#L66","commit_id":"e1d87cf184c34a5e5db9a078076e2dd9baff6a99"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"fa08ca3989c1ece995926cdc4fb5d0cc39a199fe","unresolved":true,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- debug:"},{"line_number":6,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""},{"line_number":7,"context_line":"  when: standalone_ceph is defined"},{"line_number":8,"context_line":"- debug:"},{"line_number":9,"context_line":"    msg: \"zuul.branch: {{ zuul.branch }}\""},{"line_number":10,"context_line":"  when: zuul.branch is defined"},{"line_number":11,"context_line":"# \u003c/remove\u003e"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":14,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"}],"source_content_type":"text/x-yaml","patch_set":48,"id":"0e3ec758_e87d200c","line":11,"range":{"start_line":4,"start_character":0,"end_line":11,"end_character":11},"in_reply_to":"3bda9d45_9473edfd","updated":"2022-04-27 18:28:20.000000000","message":"Though the inventory from previous run [1] has it and I confirmed it works locally [2].\n\n[1] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_457/834352/40/check/tripleo-ci-centos-9-scenario001-standalone/457013b/zuul-info/inventory.yaml\n\n[2]\n [fultonj@runcible ansible]$ cat get_var.yaml \n ---\n - name: Test example\n   gather_facts: false\n   hosts: all\n   tasks:\n     - name: print vars\n       debug:\n         msg: \"standalone_ceph: {{ standalone_ceph }}\"\n       when:\n         - standalone_ceph | bool\n [fultonj@runcible ansible]$ md5sum inventory.yaml \n 21312ed4a5d1eb7bf4c9f62374950870  inventory.yaml\n [fultonj@runcible ansible]$ ansible-playbook -i inventory.yaml get_var.yaml\n\n PLAY [Test example] \n **************************************************************************\n \n TASK [print vars] \n ****************************************************************************\n ok: [primary] \u003d\u003e {\n     \"msg\": \"standalone_ceph: True\"\n }\n \n PLAY RECAP \n ****************************************************************************\n primary                    : ok\u003d1    changed\u003d0    unreachable\u003d0    failed\u003d0    \n skipped\u003d0    rescued\u003d0    ignored\u003d0   \n \n [fultonj@runcible ansible]$","commit_id":"e1d87cf184c34a5e5db9a078076e2dd9baff6a99"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"7b5e949640b2e73de904160289d3e3c1368f2560","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- debug:"},{"line_number":6,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""},{"line_number":7,"context_line":"  when: standalone_ceph is defined"},{"line_number":8,"context_line":"- debug:"},{"line_number":9,"context_line":"    msg: \"zuul.branch: {{ zuul.branch }}\""},{"line_number":10,"context_line":"  when: zuul.branch is defined"},{"line_number":11,"context_line":"# \u003c/remove\u003e"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":14,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"}],"source_content_type":"text/x-yaml","patch_set":48,"id":"915bdfb2_668917b2","line":11,"range":{"start_line":4,"start_character":0,"end_line":11,"end_character":11},"in_reply_to":"478c5168_35dcd068","updated":"2022-06-06 11:50:51.000000000","message":"Done","commit_id":"e1d87cf184c34a5e5db9a078076e2dd9baff6a99"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"11e96b6e2dc1d2558826c8ccad48715c5f91714b","unresolved":true,"context_lines":[{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- name: Save all vars to a file"},{"line_number":6,"context_line":"  copy:"},{"line_number":7,"context_line":"    content: \"{{ vars | to_nice_yaml }}\""},{"line_number":8,"context_line":"    dest: \"{{ working_dir }}/debug_vars.yaml\""},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"- debug:"},{"line_number":11,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""}],"source_content_type":"text/x-yaml","patch_set":50,"id":"8ed563f5_d27d1ebe","line":8,"range":{"start_line":7,"start_character":0,"end_line":8,"end_character":45},"updated":"2022-04-28 19:26:19.000000000","message":"I dumped all available variables and standalone_ceph is not there. So what variable do I use to trigger this condition?\n\nhttps://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_a46/834352/50/check/tripleo-ci-centos-9-scenario004-standalone/a46d951/logs/undercloud/home/zuul/debug_vars.yaml","commit_id":"ab6242eb9ebefa3dea553a5cdb3e5ebb3035fe96"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"52b8c83b9da303cf1cce349e933115276fc6fe5e","unresolved":false,"context_lines":[{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- name: Save all vars to a file"},{"line_number":6,"context_line":"  copy:"},{"line_number":7,"context_line":"    content: \"{{ vars | to_nice_yaml }}\""},{"line_number":8,"context_line":"    dest: \"{{ working_dir }}/debug_vars.yaml\""},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"- debug:"},{"line_number":11,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""}],"source_content_type":"text/x-yaml","patch_set":50,"id":"668a36c3_f8057c7f","line":8,"range":{"start_line":7,"start_character":0,"end_line":8,"end_character":45},"in_reply_to":"23e59c56_74575a09","updated":"2022-06-06 11:48:29.000000000","message":"Revolving old comment.","commit_id":"ab6242eb9ebefa3dea553a5cdb3e5ebb3035fe96"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"cda810d394498f3743e291b489e794524b4c00fe","unresolved":true,"context_lines":[{"line_number":4,"context_line":"# \u003cremove\u003e"},{"line_number":5,"context_line":"- name: Save all vars to a file"},{"line_number":6,"context_line":"  copy:"},{"line_number":7,"context_line":"    content: \"{{ vars | to_nice_yaml }}\""},{"line_number":8,"context_line":"    dest: \"{{ working_dir }}/debug_vars.yaml\""},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"- debug:"},{"line_number":11,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""}],"source_content_type":"text/x-yaml","patch_set":50,"id":"23e59c56_74575a09","line":8,"range":{"start_line":7,"start_character":0,"end_line":8,"end_character":45},"in_reply_to":"8ed563f5_d27d1ebe","updated":"2022-04-28 19:33:20.000000000","message":"I mean it _is_ there :/ let me check how to reference it.\n\n[fultonj@runcible ansible]$ grep standalone_ceph debug_vars.yaml \n            standalone_ceph: true\n                standalone_ceph: true\n            standalone_ceph: true\n                standalone_ceph: true\n            standalone_ceph: true\n                standalone_ceph: true\n    standalone_ceph: true\n        standalone_ceph: true\n[fultonj@runcible ansible]$","commit_id":"ab6242eb9ebefa3dea553a5cdb3e5ebb3035fe96"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"ee20c8fbac5e178b78d444301fcf7c899dd93f75","unresolved":true,"context_lines":[{"line_number":9,"context_line":""},{"line_number":10,"context_line":"- debug:"},{"line_number":11,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""},{"line_number":12,"context_line":"  when: standalone_ceph is defined"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"- debug:"},{"line_number":15,"context_line":"    msg: \"zuul.branch: {{ zuul.branch }}\""}],"source_content_type":"text/x-yaml","patch_set":50,"id":"e952eb83_cd5f571f","line":12,"range":{"start_line":12,"start_character":27,"end_line":12,"end_character":34},"updated":"2022-04-29 06:55:54.000000000","message":"add here \u0027and standalone_ceph\u0027 see if it prints then","commit_id":"ab6242eb9ebefa3dea553a5cdb3e5ebb3035fe96"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ca0234d50d55f6f3a7672a4cf9bdb6ebb3f50867","unresolved":false,"context_lines":[{"line_number":9,"context_line":""},{"line_number":10,"context_line":"- debug:"},{"line_number":11,"context_line":"    msg: \"standalone_ceph: {{ standalone_ceph }}\""},{"line_number":12,"context_line":"  when: standalone_ceph is defined"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"- debug:"},{"line_number":15,"context_line":"    msg: \"zuul.branch: {{ zuul.branch }}\""}],"source_content_type":"text/x-yaml","patch_set":50,"id":"c3f21f5f_7cf7e68e","line":12,"range":{"start_line":12,"start_character":27,"end_line":12,"end_character":34},"in_reply_to":"e952eb83_cd5f571f","updated":"2022-05-18 12:36:48.000000000","message":"job.standalone_ceph works","commit_id":"ab6242eb9ebefa3dea553a5cdb3e5ebb3035fe96"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"ee20c8fbac5e178b78d444301fcf7c899dd93f75","unresolved":true,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":20,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"},{"line_number":21,"context_line":"    - standalone_ceph is defined and standalone_ceph|bool"},{"line_number":22,"context_line":"    - zuul.branch is defined and zuul.branch in [\u0027master\u0027]"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":50,"id":"67e2549b_af517d0b","line":21,"range":{"start_line":21,"start_character":52,"end_line":21,"end_character":57},"updated":"2022-04-29 06:55:54.000000000","message":"not sure what\u0027s going on here John.\n\none thing is that in our invocation we don\u0027t have the |bool so its just\n\n    when: standalone_ceph is defined and standalone_ceph\n\n\nanother thing you can try is like\n\n    when: \"{{ standalone_ceph|default(false) }}\"\n\ni went digging and that was the conditional we had ^^^ before it changed to what we have today for yamllint","commit_id":"ab6242eb9ebefa3dea553a5cdb3e5ebb3035fe96"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"8ecf84235b3227f67cbc22bb5f548a1d6edc930b","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":20,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"},{"line_number":21,"context_line":"    - standalone_ceph is defined and standalone_ceph|bool"},{"line_number":22,"context_line":"    - zuul.branch is defined and zuul.branch in [\u0027master\u0027]"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":50,"id":"b06deb3f_852aec8e","line":21,"range":{"start_line":21,"start_character":52,"end_line":21,"end_character":57},"in_reply_to":"67e2549b_af517d0b","updated":"2022-06-06 11:46:37.000000000","message":"Just resolving this comment.","commit_id":"ab6242eb9ebefa3dea553a5cdb3e5ebb3035fe96"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"f3919248d207e9a16274b7aa5d89e05d715e9191","unresolved":true,"context_lines":[{"line_number":44,"context_line":""},{"line_number":45,"context_line":"- name: hostvars.primary.hostvars.undercloud.job.standalone_ceph"},{"line_number":46,"context_line":"  debug:"},{"line_number":47,"context_line":"    msg: \"{{ hostvars.primary.hostvars.undercloud.job.standalone_ceph }}\""},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"- name: job.standalone_ceph"},{"line_number":50,"context_line":"  debug:"}],"source_content_type":"text/x-yaml","patch_set":52,"id":"7661ce04_fcdcc4bf","line":47,"updated":"2022-05-02 22:08:31.000000000","message":"There is no hostvars.primary. I\u0027ll try one of these three\n\n  TASK [standalone : hostvars] ***************************************************\n  Monday 02 May 2022  21:39:46 +0000 (0:00:00.030)       0:05:39.794 ************ \n  ok: [undercloud] \u003d\u003e {\n      \"msg\": [\n          \"localhost\",\n          \"127.0.0.2\",\n          \"undercloud\"\n      ]\n  }\n\n  TASK [standalone : hostvars.primary|list] **************************************\n  Monday 02 May 2022  21:39:46 +0000 (0:00:00.040)       0:05:39.834 ************ \n  fatal: [undercloud]: FAILED! \u003d\u003e {\"msg\": \"The task includes an option with an \n  undefined variable. The error was: \\\"hostvars[\u0027primary\u0027]\\\" is undefined","commit_id":"e1d03bc60aa2dbb5411076507f4a5c1ca358f034"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"89124c90b511c9526c48f85f8faf2636d5646943","unresolved":false,"context_lines":[{"line_number":44,"context_line":""},{"line_number":45,"context_line":"- name: hostvars.primary.hostvars.undercloud.job.standalone_ceph"},{"line_number":46,"context_line":"  debug:"},{"line_number":47,"context_line":"    msg: \"{{ hostvars.primary.hostvars.undercloud.job.standalone_ceph }}\""},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"- name: job.standalone_ceph"},{"line_number":50,"context_line":"  debug:"}],"source_content_type":"text/x-yaml","patch_set":52,"id":"2a9d89af_ec2d163a","line":47,"in_reply_to":"7661ce04_fcdcc4bf","updated":"2022-05-17 14:45:27.000000000","message":"Since job.standalone_ceph is working I\u0027m resolving this thread.","commit_id":"e1d03bc60aa2dbb5411076507f4a5c1ca358f034"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"48ace5c0998b8211164874eee60f817110ad178f","unresolved":true,"context_lines":[{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"},{"line_number":5,"context_line":"    - job.standalone_ceph | default(false)"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch in [\u0027master\u0027]"},{"line_number":7,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"677aa0b7_8f693b83","line":5,"range":{"start_line":5,"start_character":0,"end_line":5,"end_character":6},"updated":"2022-06-06 07:07:08.000000000","message":"so is this now working ok as a switch?\n\nBut does it catch all of them scen1 scen4 scen10? because the https://review.opendev.org/c/openstack/tripleo-heat-templates/+/834354 that depends-on this is only wiring up scen1","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":12393,"name":"chandan kumar","display_name":"Chandan Kumar","email":"chkumar@redhat.com","username":"chkumar246"},"change_message_id":"e935ef8db11366173fe7623c2d90f467758f06c2","unresolved":true,"context_lines":[{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"},{"line_number":5,"context_line":"    - job.standalone_ceph | default(false)"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch in [\u0027master\u0027]"},{"line_number":7,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"ef01155b_408e9e1b","line":5,"range":{"start_line":5,"start_character":0,"end_line":5,"end_character":6},"in_reply_to":"134a51d2_734c7702","updated":"2022-06-07 13:38:56.000000000","message":"job.standalone_ceph is defined here https://opendev.org/openstack/tripleo-ci/src/branch/master/zuul.d/standalone-jobs.yaml#L434 already.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"dcb1ace435d6c97588e48f44bc3b9a5a2849875f","unresolved":true,"context_lines":[{"line_number":2,"context_line":"- include_tasks: containers.yml"},{"line_number":3,"context_line":"- include_tasks: ceph-install.yml"},{"line_number":4,"context_line":"  when:  # TODO(fultonj) use [\u0027master\u0027, \u0027wallaby\u0027] after backport"},{"line_number":5,"context_line":"    - job.standalone_ceph | default(false)"},{"line_number":6,"context_line":"    - zuul.branch is defined and zuul.branch in [\u0027master\u0027]"},{"line_number":7,"context_line":"- include_tasks: standalone-install.yml"}],"source_content_type":"text/x-yaml","patch_set":81,"id":"134a51d2_734c7702","line":5,"range":{"start_line":5,"start_character":0,"end_line":5,"end_character":6},"in_reply_to":"677aa0b7_8f693b83","updated":"2022-06-06 11:34:47.000000000","message":"\u003e so is this now working ok as a switch?\n\nYes, it is. The trick was to use job.standalone_ceph and not standalone_ceph. Thank you for suggesting this flag.\n\n\u003e But does it catch all of them scen1 scen4 scen10? because the https://review.opendev.org/c/openstack/tripleo-heat-templates/+/834354 that depends-on this is only wiring up scen1\n\nYes, you can see standalone 001/004/010 is tested per the zuul results in this job.\n\nI\u0027ve updated 834354 so that 001/004/010 standalone are all updated accordingly. 834354 is a clean up of THT to remove parameters that are overridden by this job.","commit_id":"be8dccb7455b62266da2ce1c6d7e7ac667d43084"}],"roles/standalone/tasks/standalone-install.yml":[{"author":{"_account_id":24245,"name":"Harald Jensås","email":"hjensas@redhat.com","username":"harald.jensas"},"change_message_id":"39372f4b225206d86b504a13885f8f85a0028855","unresolved":true,"context_lines":[{"line_number":8,"context_line":"    - tripleo.operator"},{"line_number":9,"context_line":"  include_role:"},{"line_number":10,"context_line":"    name: tripleo_deploy"},{"line_number":11,"context_line":"  vars:"},{"line_number":12,"context_line":"    tripleo_deploy_debug: true"},{"line_number":13,"context_line":"    tripleo_deploy_reproducer_command: \"{{ release not in [\u0027wallaby\u0027, \u0027victoria\u0027, \u0027ussuri\u0027, \u0027train\u0027, \u0027stein\u0027] }}\""},{"line_number":14,"context_line":"    tripleo_deploy_debug_arg: \"{{ standalone_tripleo_deploy_debug_arg }}\""}],"source_content_type":"text/x-yaml","patch_set":68,"id":"8900f2a1_0a3dc1b5","line":11,"range":{"start_line":11,"start_character":2,"end_line":11,"end_character":6},"updated":"2022-05-18 14:02:03.000000000","message":"We need to pass some network-data to here to ensure THT is rendered with the Storage network. Looks like var[1] exist to set it. You also need at env file with DeployedNetworkEnvironment and `OS::TripleO::Network: /usr/share/openstack-tripleo-heat-templates/network/deployed_networks.yaml` resource_registry entry.\n\nI\u0027d suggest a copy of [2] then remove everything that is not for the storage network and change the IP/dns values the match the ones used.\n\n[1] https://opendev.org/openstack/tripleo-operator-ansible/src/branch/master/roles/tripleo_deploy/defaults/main.yml#L28\n[2] https://logserver.rdoproject.org/74/842274/1/openstack-check/tripleo-ci-centos-9-ovb-3ctlr_1comp-featureset001/cfaa434/logs/undercloud/home/zuul/overcloud-networks-deployed.yaml.txt.gz","commit_id":"6af1644c444ecfc92e2975ab1264ab36b6cb9d39"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ee1ebe4137cadbcf7ad4e92b7b7c28b29e7e3a9c","unresolved":true,"context_lines":[{"line_number":8,"context_line":"    - tripleo.operator"},{"line_number":9,"context_line":"  include_role:"},{"line_number":10,"context_line":"    name: tripleo_deploy"},{"line_number":11,"context_line":"  vars:"},{"line_number":12,"context_line":"    tripleo_deploy_debug: true"},{"line_number":13,"context_line":"    tripleo_deploy_reproducer_command: \"{{ release not in [\u0027wallaby\u0027, \u0027victoria\u0027, \u0027ussuri\u0027, \u0027train\u0027, \u0027stein\u0027] }}\""},{"line_number":14,"context_line":"    tripleo_deploy_debug_arg: \"{{ standalone_tripleo_deploy_debug_arg }}\""}],"source_content_type":"text/x-yaml","patch_set":68,"id":"faa742b9_3d71ac27","line":11,"range":{"start_line":11,"start_character":2,"end_line":11,"end_character":6},"in_reply_to":"8900f2a1_0a3dc1b5","updated":"2022-05-20 12:54:31.000000000","message":"OK, testing this approach w/ PS70.\n\n https://review.opendev.org/c/openstack/tripleo-quickstart-extras/+/834352/68..70","commit_id":"6af1644c444ecfc92e2975ab1264ab36b6cb9d39"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"52b8c83b9da303cf1cce349e933115276fc6fe5e","unresolved":false,"context_lines":[{"line_number":8,"context_line":"    - tripleo.operator"},{"line_number":9,"context_line":"  include_role:"},{"line_number":10,"context_line":"    name: tripleo_deploy"},{"line_number":11,"context_line":"  vars:"},{"line_number":12,"context_line":"    tripleo_deploy_debug: true"},{"line_number":13,"context_line":"    tripleo_deploy_reproducer_command: \"{{ release not in [\u0027wallaby\u0027, \u0027victoria\u0027, \u0027ussuri\u0027, \u0027train\u0027, \u0027stein\u0027] }}\""},{"line_number":14,"context_line":"    tripleo_deploy_debug_arg: \"{{ standalone_tripleo_deploy_debug_arg }}\""}],"source_content_type":"text/x-yaml","patch_set":68,"id":"880d88f8_2de9f3aa","line":11,"range":{"start_line":11,"start_character":2,"end_line":11,"end_character":6},"in_reply_to":"faa742b9_3d71ac27","updated":"2022-06-06 11:48:29.000000000","message":"Done","commit_id":"6af1644c444ecfc92e2975ab1264ab36b6cb9d39"}]}