)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"02e69e7eaf8604b79f9b3fc6258de85b4a24a1c0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"dcfc833e_42179c77","updated":"2022-02-16 21:41:03.000000000","message":"Even in a clean deployment with this patch I\u0027m getting:\n\n2022-02-16 19:24:08.531264 |                                      |    WARNING | ERROR: Can\u0027t run container nova_libvirt_init_secret\nstderr: Usage: grep [OPTION]... PATTERN [FILE]...\nTry \u0027grep --help\u0027 for more information.\nerror: failed to get secret \u0027--base64\u0027\nerror: uuidstr in virSecretLookupByUUIDString must be a valid UUID\n2022-02-16 19:24:08.533519 | 52540094-d704-ab0d-2b01-000000008ff0 |      FATAL | Create containers managed by Podman for /var/lib/tripleo-config/container-startup-config/step_4 | compute-0 |\n error\u003d{\"changed\": false, \"msg\": \"Failed containers: nova_libvirt_init_secret\"}\n","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"b952568ab185959878bc10bf8a0008cf59874eb9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"a47cf903_186e9d79","updated":"2022-03-06 09:33:25.000000000","message":"check-rdo","commit_id":"a2674a176d7d6573c1f84e4f242ad37dab73aa7c"}],"container_config_scripts/nova_libvirt_init_secret.sh":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"564da4964c49904244f2c59d3d8c2ad778bfff15","unresolved":true,"context_lines":[{"line_number":9,"context_line":"    exit 1"},{"line_number":10,"context_line":"fi"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"# TODO(fultonj): Make this handle CephConfigPath"},{"line_number":13,"context_line":"echo \"------------------------------------------------\""},{"line_number":14,"context_line":"echo \"Initializing virsh secrets for: ${CEPH_INFO[@]}\""},{"line_number":15,"context_line":""}],"source_content_type":"text/x-sh","patch_set":4,"id":"ac4321ab_98c48c4e","line":12,"updated":"2022-02-17 22:01:17.000000000","message":"Need to do this before merging","commit_id":"ca961244d5cf79afdcc0562969d43f01ae15dd9c"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"cb05f1a3b2cd2a1ce3af9ee8274df39fc577c666","unresolved":false,"context_lines":[{"line_number":9,"context_line":"    exit 1"},{"line_number":10,"context_line":"fi"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"# TODO(fultonj): Make this handle CephConfigPath"},{"line_number":13,"context_line":"echo \"------------------------------------------------\""},{"line_number":14,"context_line":"echo \"Initializing virsh secrets for: ${CEPH_INFO[@]}\""},{"line_number":15,"context_line":""}],"source_content_type":"text/x-sh","patch_set":4,"id":"96f89617_2b8444ba","line":12,"in_reply_to":"38a0280d_d8678ec8","updated":"2022-03-04 22:37:45.000000000","message":"yeah, I can confirm that within the containers you always have /etc/ceph: CephConfigPath (on the host) is always mapped into /etc/ceph in containers (for nova, cinder, glance, manila, etc).","commit_id":"ca961244d5cf79afdcc0562969d43f01ae15dd9c"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"5cff49a86ca6bb1d4885764fef88e4f3b70dfc05","unresolved":false,"context_lines":[{"line_number":9,"context_line":"    exit 1"},{"line_number":10,"context_line":"fi"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"# TODO(fultonj): Make this handle CephConfigPath"},{"line_number":13,"context_line":"echo \"------------------------------------------------\""},{"line_number":14,"context_line":"echo \"Initializing virsh secrets for: ${CEPH_INFO[@]}\""},{"line_number":15,"context_line":""}],"source_content_type":"text/x-sh","patch_set":4,"id":"38a0280d_d8678ec8","line":12,"in_reply_to":"ac4321ab_98c48c4e","updated":"2022-03-04 21:18:24.000000000","message":"CephConfigPath is passed to the THT that calls this script:\n\n  https://github.com/openstack/tripleo-heat-templates/blob/master/deployment/nova/nova-modular-libvirt-container-puppet.yaml#L91-L95\n\nThat THT then mounts wherever CephConfigPath is set to is mounted into /etc/ceph on the nova container\n\n  https://github.com/openstack/tripleo-heat-templates/blob/master/deployment/nova/nova-modular-libvirt-container-puppet.yaml#L719-L730\n\nThe nova contianer is called with CLUSTER:USER:\n\n  https://github.com/openstack/tripleo-heat-templates/blob/master/deployment/nova/nova-modular-libvirt-container-puppet.yaml#L731-L739\n\nBut using /etc/ceph should be correct as that\u0027s the path inside the container.","commit_id":"ca961244d5cf79afdcc0562969d43f01ae15dd9c"}],"deployment/cephadm/ceph-client.yaml":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"c16089ec80f61e0b2f7988270f8aed960087fdd5","unresolved":true,"context_lines":[{"line_number":115,"context_line":"              - name: Populate CephClientConfigVars from THT"},{"line_number":116,"context_line":"                copy:"},{"line_number":117,"context_line":"                  dest: \"{{ tripleo_ceph_client_vars }}\""},{"line_number":118,"context_line":"                  content: |"},{"line_number":119,"context_line":"                    tripleo_ceph_client_fsid: {{ tripleo_ceph_client_fsid }}"},{"line_number":120,"context_line":"                    external_cluster_mon_ips: {{ external_cluster_mon_ips }}"},{"line_number":121,"context_line":"                when:"},{"line_number":122,"context_line":"                  - ceph_input_vars.stat.exists \u003d\u003d False"},{"line_number":123,"context_line":"                  - external_cluster_mon_ips | length \u003e 0"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"a07b4b46_15a94c82","line":120,"range":{"start_line":118,"start_character":0,"end_line":120,"end_character":76},"updated":"2022-03-05 20:38:34.000000000","message":"I think the variables here need to be quoted like so:\n\n -                    tripleo_ceph_client_fsid: {{ tripleo_ceph_client_fsid }}\n -                    external_cluster_mon_ips: {{ external_cluster_mon_ips }}\n +                    tripleo_ceph_client_fsid: \"{{ tripleo_ceph_client_fsid }}\"\n +                    external_cluster_mon_ips: \"{{ external_cluster_mon_ips }}\"\n\nBecause when I pass this env file:\n\n\"\"\"\nparameter_defaults:\n  CephClientKey: \"AQAIFyJirRYTNRAAWUHbirFZ9+807L2y0nkx/Q\u003d\u003d\"\n  CephClusterFSID: \"7713814b-c6bf-57af-bf42-39047d06279c\"\n  CephExternalMonHost: \"[v2:172.16.11.33:3300/0,v1:172.16.11.33:6789/0] [v2:172.16.11.153:3300/0,v1:172.16.11.153:6789/0] [v2:172.16.11.202:3300/0,v1:172.16.11.202:6789/0]\"\n\"\"\"\n\nto this task:\n\nhttps://github.com/openstack/tripleo-ansible/blob/dbc04aad8805b8c7129cbfbeede46ac778014dcb/tripleo_ansible/roles/tripleo_ceph_client/tasks/main.yml#L50-L52\n\n\nI get this error:\n\n2022-03-04 22:24:38.126387 | 244201c7-cba8-15d3-893b-000000004cf0 |       TASK | Load variables produced b\ny the cephadm provisioning process                                                                       \n2022-03-04 22:24:38.153553 | 244201c7-cba8-15d3-893b-000000004cf0 |      FATAL | Load variables produced by the cephadm provisioning process | undercloud | error\u003d{\"ansible_facts\": {}, \"ansible_included_var_files\": [], \"changed\": false, \"message\": \"We were unable to read either as JSON nor YAML, these are the errors we got from each:\\nJSON: Expecting value: line 1 column 1 (char 0)\\n\\nSyntax Error while loading YAML.\\n  found unexpected \u0027:\u0027\\n\\nThe error appears to be in \u0027/home/stack/overcloud-deploy/swift/config-download/swift/cephadm/ceph_client.yml\u0027: line 2, column 30, but may\\nbe elsewhere in the file depending on the exact syntax problem.\\n\\nThe offending line appears to be:\\n\\ntripleo_ceph_client_fsid: 7713814b-c6bf-57af-bf42-39047d06279c\\nexternal_cluster_mon_ips: [v2:172.16.11.33:3300/0,v1:172.16.11.33:6789/0] [v2:172.16.11.153:3300/0,v1:172.16.11.153:6789/0] [v2:172.16.11.202:3300/0,v1:172.16.11.202:6789/0]\\n                         \n    ^ here\\n\"}\n\nBecause we\u0027re building a full YAML file with copy/content we should quote.","commit_id":"fa6ecd71cd3bf5da85cff45a4b4ef2e1172ed426"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"a3295b5bf7d1e204552b4888daa7400f6e16ca21","unresolved":false,"context_lines":[{"line_number":115,"context_line":"              - name: Populate CephClientConfigVars from THT"},{"line_number":116,"context_line":"                copy:"},{"line_number":117,"context_line":"                  dest: \"{{ tripleo_ceph_client_vars }}\""},{"line_number":118,"context_line":"                  content: |"},{"line_number":119,"context_line":"                    tripleo_ceph_client_fsid: {{ tripleo_ceph_client_fsid }}"},{"line_number":120,"context_line":"                    external_cluster_mon_ips: {{ external_cluster_mon_ips }}"},{"line_number":121,"context_line":"                when:"},{"line_number":122,"context_line":"                  - ceph_input_vars.stat.exists \u003d\u003d False"},{"line_number":123,"context_line":"                  - external_cluster_mon_ips | length \u003e 0"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"b16d7294_2050854d","line":120,"range":{"start_line":118,"start_character":0,"end_line":120,"end_character":76},"in_reply_to":"a07b4b46_15a94c82","updated":"2022-03-06 09:33:18.000000000","message":"Ack","commit_id":"fa6ecd71cd3bf5da85cff45a4b4ef2e1172ed426"}],"environments/external-ceph.yaml":[{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"df4a413a5f2a90723b678102d6c924938e1b30a3","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"ba48a338_a88277aa","line":2,"updated":"2022-02-16 23:45:08.000000000","message":"Without this change I don\u0027t see the THT generated ansible in config-download including the tripleo_run_cephadm role like it is now:\n\n(undercloud) [stack@undercloud-0 overcloud]$ tail -42 external_deploy_steps_tasks_step2.yaml | head \n      | default([]) and groups[\u0027ceph_rbdmirror\u0027] | default([]) | length \u003e 0)))\n  - include_role:\n      name: tripleo_run_cephadm\n    name: Deploy or configure the cephadm Ceph cluster\n    when: groups[\u0027ceph_mon\u0027] | default([]) | length \u003e 0 or groups[\u0027ceph_nfs\u0027] | default([])\n      | length \u003e 0\n  name: ceph_base_external_deploy_task\n  tags:\n  - ceph\n  when: step|int \u003d\u003d 2\n(undercloud) [stack@undercloud-0 overcloud]$ \n\nwhich in turn generates the necessary config files:\n\n[heat-admin@controller-0 ~]$ sudo ls -l /var/lib/tripleo-config/ceph/\ntotal 8\n-rw-------. 1 root root 205 Feb 16 17:22 ceph.client.openstack.keyring\n-rw-r--r--. 1 root root 179 Feb 16 17:22 ceph.conf\n[heat-admin@controller-0 ~]$\n\nBut that\u0027s not sufficient. \n\nI see this error failing with grep:\n\n~~~\nERROR: Can\u0027t run container nova_libvirt_init_secret\n\nstderr: Usage: grep [OPTION]... PATTERN [FILE]...\nTry \u0027grep --help\u0027 for more information.\nerror: failed to get secret \u0027--base64\u0027\nerror: uuidstr in virSecretLookupByUUIDString must be a valid UUID\n\nFATAL Create containers managed by Podman for /var/lib/tripleo-config/container-startup-config/step_4 | compute-0 | error\u003d{\"changed\": false, \"msg\": \"Failed containers: nova_libvirt_init_secret\"}\n ~~~\n\nSo who\u0027s calling grep with invalid syntax?\n\n[root@compute-0 step_4]# pwd\n/var/lib/tripleo-config/container-startup-config/step_4\n[root@compute-0 step_4]# cat nova_libvirt_init_secret.json | jq .volumes\n[\n  \"/etc/hosts:/etc/hosts:ro\",\n  \"/etc/localtime:/etc/localtime:ro\",\n  \"/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro\",\n  \"/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro\",\n  \"/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro\",\n  \"/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro\",\n  \"/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro\",\n  \"/dev/log:/dev/log\",\n  \"/etc/puppet:/etc/puppet:ro\",\n  \"/var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova\",\n  \"/etc/libvirt:/etc/libvirt\",\n  \"/run/libvirt:/run/libvirt:shared\",\n  \"/var/lib/libvirt:/var/lib/libvirt:shared\",\n  \"/var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro\",\n  \"/var/lib/tripleo-config/ceph:/etc/ceph:ro\"\n]\n[root@compute-0 step_4]# \n\n[root@compute-0 step_4]# cat /var/lib/container-config-scripts/nova_libvirt_init_secret.sh | curl -F \u0027f:1\u003d\u003c-\u0027 ix.io \nhttp://ix.io/3PO3\n[root@compute-0 step_4]#","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"5185d0066b8480f82fec57d8fb0235896e688e62","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"c0014b7e_1d952f65","line":2,"in_reply_to":"227752a6_ddde0474","updated":"2022-02-16 23:59:27.000000000","message":"I\u0027m now testing a deployment with \"CephConfigPath: /etc/ceph\".\n\nIf that works then I think this patch needs to be rebased on this patch:\n\n https://review.opendev.org/c/openstack/tripleo-heat-templates/+/804538\n\nand then this patch should also update nova_libvirt_init_secret.sh to deal with whatever CephConfigPath is set.","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"c7f9dfb7f99400d250ef39cadc8f8a0ee7b0f0a7","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"b6ff6d0c_69a9b7d5","line":2,"in_reply_to":"50333a34_6cbcff1d","updated":"2022-02-17 11:53:42.000000000","message":"When I said \"rebase 829611 on 804538\" I used the wrong term. I meant that when the solution for the bug lands it should include 804538. Towards that end I\u0027ve proposed a backport of it which should hopefully land before this bug is fixed and cherry picked.\n\n https://review.opendev.org/c/openstack/tripleo-heat-templates/+/829639","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"9e00367389069c8084721dffd69c68609dab78e3","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"50333a34_6cbcff1d","line":2,"in_reply_to":"708a5e22_3c01cf9a","updated":"2022-02-17 11:08:08.000000000","message":"Yes, it\u0027s not in the wallaby compose I\u0027m testing.","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d154e4b7da0195afaa2726ea2479d40748741689","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"68d6b49a_ad44c9bd","line":2,"in_reply_to":"9799bc9b_1042bba1","updated":"2022-02-17 12:50:42.000000000","message":"I still need to modify nova_libvirt_init_secret.sh to take CephConfigPath into account.","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"2cbe41fecd011bcefbb9274ad510d901e14e0f23","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"708a5e22_3c01cf9a","line":2,"in_reply_to":"a21c354d_cace8128","updated":"2022-02-17 09:23:11.000000000","message":"The patch you mentioned [1] is pretty old (merged Aug 2021): do you mean this is missing in the Wallaby compose you\u0027re testing? \n\n\n[1] https://review.opendev.org/c/openstack/tripleo-heat-templates/+/804538","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"904d4bed70514ce1eec30200e934162e04d0860f","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"9799bc9b_1042bba1","line":2,"in_reply_to":"b6ff6d0c_69a9b7d5","updated":"2022-02-17 12:06:36.000000000","message":"++ thanks John","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"599bb48b5b89551dd2444ef2d6c5ef06ef1f30aa","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"227752a6_ddde0474","line":2,"in_reply_to":"ba48a338_a88277aa","updated":"2022-02-16 23:52:45.000000000","message":"I\u0027m testing wallaby but it\u0027s missing this patch:\n\nhttps://github.com/openstack/tripleo-heat-templates/commit/61f67eff1046fba283c7993c10d1f37870679c18","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"bd220cb795ce931ffe20759da2837bc56c80c78b","unresolved":true,"context_lines":[{"line_number":1,"context_line":"resource_registry:"},{"line_number":2,"context_line":"  OS::TripleO::Services::CephExternal: ../deployment/cephadm/ceph-client.yaml"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"parameter_defaults:"},{"line_number":5,"context_line":"  # NOTE: These example parameters are required when using CephExternal"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"a21c354d_cace8128","line":2,"in_reply_to":"c0014b7e_1d952f65","updated":"2022-02-17 09:21:01.000000000","message":"Ack and thanks for clarifying.","commit_id":"3e87605db7a68586f9a7e402106dc628df985b92"}]}