)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"912f935be49c0e089ce1138ff9781c695f86cf99","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"a245025f_416a71e1","updated":"2021-12-06 19:12:16.000000000","message":"AFAICT The CI test failed on this error: https://review.rdoproject.org/zuul/build/bd562c88196049ad8429dee20ebbaf86/log/logs/undercloud/home/zuul/overcloud_node_provision.log.txt.gz#60\n\n2021-12-06 18:02:17.528639 | fa163e56-710f-ee52-6a6f-000000000018 |      FATAL | Provision instances | localhost | error\u003d{\"changed\": false, \"logging\": \"Created port overcloud-controller-1-ctlplane (UUID 58c58940-2ed9-4f63-8c2d-ee1fef64a6bc) for node baremetal-820217-3-27621-3 (UUID 1ec48976-cd12-418d-bdec-224bfed7e657) with {\u0027network_id\u0027: \u0027d9cfc6c9-2e66-4b71-b918-35e117fe216b\u0027, \u0027name\u0027: \u0027overcloud-controller-1-ctlplane\u0027}\\nCreated port overcloud-novacompute-0-ctlplane (UUID fd3ba287-0f27-49b4-a081-da232e99284f) for node baremetal-820217-3-27621-2 (UUID 22b6afcc-5cb1-4f5b-95f7-a5168e715396) with {\u0027network_id\u0027: \u0027d9cfc6c9-2e66-4b71-b918-35e117fe216b\u0027, \u0027name\u0027: \u0027overcloud-novacompute-0-ctlplane\u0027}\\nCreated port overcloud-controller-2-ctlplane (UUID d1c992d5-6248-470b-af2c-3f74850ad9e7) for node baremetal-820217-3-27621-0 (UUID 6a0ef1d3-f28d-4299-943a-9acfa8faf3c3) with {\u0027network_id\u0027: \u0027d9cfc6c9-2e66-4b71-b918-35e117fe216b\u0027, \u0027name\u0027: \u0027overcloud-controller-2-ctlplane\u0027}\\nCreated port overcloud-controller-0-ctlplane (UUID ad2da935-a433-4e41-bc42-cdd82226d401) for node baremetal-820217-3-27621-1 (UUID c844e4e6-2c78-4a9e-bb3e-6228348255b6) with {\u0027network_id\u0027: \u0027d9cfc6c9-2e66-4b71-b918-35e117fe216b\u0027, \u0027name\u0027: \u0027overcloud-controller-0-ctlplane\u0027}\\nAttached port overcloud-controller-1-ctlplane (UUID 58c58940-2ed9-4f63-8c2d-ee1fef64a6bc) to node baremetal-820217-3-27621-3 (UUID 1ec48976-cd12-418d-bdec-224bfed7e657)\\nAttached port overcloud-controller-2-ctlplane (UUID d1c992d5-6248-470b-af2c-3f74850ad9e7) to node baremetal-820217-3-27621-0 (UUID 6a0ef1d3-f28d-4299-943a-9acfa8faf3c3)\\nAttached port overcloud-novacompute-0-ctlplane (UUID fd3ba287-0f27-49b4-a081-da232e99284f) to node baremetal-820217-3-27621-2 (UUID 22b6afcc-5cb1-4f5b-95f7-a5168e715396)\\nAttached port overcloud-controller-0-ctlplane (UUID ad2da935-a433-4e41-bc42-cdd82226d401) to node baremetal-820217-3-27621-1 (UUID c844e4e6-2c78-4a9e-bb3e-6228348255b6)\\nProvisioning started on node baremetal-820217-3-27621-3 (UUID 1ec48976-cd12-418d-bdec-224bfed7e657)\\nProvisioning started on node baremetal-820217-3-27621-2 (UUID 22b6afcc-5cb1-4f5b-95f7-a5168e715396)\\nProvisioning started on node baremetal-820217-3-27621-1 (UUID c844e4e6-2c78-4a9e-bb3e-6228348255b6)\\nProvisioning started on node baremetal-820217-3-27621-0 (UUID 6a0ef1d3-f28d-4299-943a-9acfa8faf3c3)\\n\", \"msg\": \"Node 22b6afcc-5cb1-4f5b-95f7-a5168e715396 reached failure state \\\"deploy failed\\\"; the last error is Failed to prepare to deploy. Exception: Version requested but version discovery document was not found and allow_version_hack was False\"}\n\nI\u0027m going to go ahead and guess that this is not related and attempt a recheck to see if it\u0027s transient.","commit_id":"d05129e40195b139d9e4c22bdcccefeec4de3cd4"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"1d4e12558bc93fefc3e9c3d1002d5120ee9ea7b1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"638307d8_7c45f2ec","updated":"2021-12-07 20:08:35.000000000","message":"Error was during build of the IPA node. Nothing to do with my code.\n\n2021-12-07 18:28:32 | + sudo dnf install -yq ansible ansible-tripleo-ipa\n2021-12-07 18:28:36 | Error: Error downloading packages:\n2021-12-07 18:28:36 |   ansible-2.9.23-1.el8.noarch: Cannot download, all mirrors were already tried without success\n2021-12-07 18:28:36 | \n\n\nLooks like more networking/timeout issues of some sort. Let\u0027s try again!","commit_id":"d05129e40195b139d9e4c22bdcccefeec4de3cd4"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"0a29771f511849c05828b3c688a4f01aee1cebbc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"abf9efd2_d041a037","updated":"2021-12-08 11:29:25.000000000","message":"LGTM, please add a release note also","commit_id":"d05129e40195b139d9e4c22bdcccefeec4de3cd4"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"a5d3934094b12a1886b0daa9adb9e0709cc18750","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"65e3cea0_6a6e4dca","updated":"2021-12-07 17:20:39.000000000","message":"The last error was indeed transient. The new errors are equally suspicious:\n\n\ntripleo-ci-centos-8-standalone-on-multinode-ipa\n\nfatal: [subnode-1]: FAILED! \u003d\u003e {\"changed\": false, \"msg\": \"Failed to download packages: perl-IO-Compress-2.081-1.el8.noarch: Cannot download, all mirrors were already tried without success\", \"results\": []}\n\n\ntripleo-ci-centos-8-scenario000-multinode-oooq-container-updates \n\n2021-12-06 19:03:03.245287 | fa163e40-91ce-c327-68ff-000000000e0e |      FATAL | Container image prepare | undercloud | error\u003d{\"changed\": false, \"error\": \"Not found image: http://188.95.226.186:5001/v2/tripleomaster/openstack-ironic-pxe/manifests/5e6ef0d9cd3dec521e0ad59a2e8467d3\", \"msg\": \"Error running container image prepare: Not found image: http://188.95.226.186:5001/v2/tripleomaster/openstack-ironic-pxe/manifests/5e6ef0d9cd3dec521e0ad59a2e8467d3\", \"params\": {}, \"success\": false}\n\n\ntripleo-ci-centos-8-undercloud-upgrade\n\n2021-12-06 18:19:17 | 2021-12-06 18:19:17.408180 | bc764e20-14d8-2da6-7726-000000000e06 |      FATAL | Container image prepare | undercloud | error\u003d{\"changed\": false, \"error\": \"Not found image: http://188.95.226.186:5001/v2/tripleomaster/openstack-ironic-pxe/manifests/5e6ef0d9cd3dec521e0ad59a2e8467d3\", \"msg\": \"Error running container image prepare: Not found image: http://188.95.226.186:5001/v2/tripleomaster/openstack-ironic-pxe/manifests/5e6ef0d9cd3dec521e0ad59a2e8467d3\", \"params\": {}, \"success\": false}\n\n\nhttps://www.youtube.com/watch?v\u003dzylpQuEcRa4\u0026t\u003d50s","commit_id":"d05129e40195b139d9e4c22bdcccefeec4de3cd4"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"dacf9c2bc94131df9abbf6a2d4d99d443e1d9678","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"4d1e6678_07ca8bb1","updated":"2021-12-07 20:08:39.000000000","message":"recheck","commit_id":"d05129e40195b139d9e4c22bdcccefeec4de3cd4"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"4095eb2213a8f9c90b2b828d041a3ec9935692ff","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"b76d6ea3_60536960","updated":"2021-12-06 19:12:22.000000000","message":"recheck","commit_id":"d05129e40195b139d9e4c22bdcccefeec4de3cd4"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"359acc4f1823029fd2c8e5435f9f6ae2322cd8a0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"e3571fee_c7b7278a","updated":"2021-12-07 17:20:44.000000000","message":"recheck","commit_id":"d05129e40195b139d9e4c22bdcccefeec4de3cd4"},{"author":{"_account_id":4264,"name":"Matthias Runge","email":"mrunge@redhat.com","username":"mrunge"},"change_message_id":"c005f40d4f665fee22e4705defc8a1e99eb6c3e9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"0232a3d4_8ae1d18f","updated":"2021-12-09 19:19:04.000000000","message":"thank you, looks good to me.","commit_id":"c3bb91338627c600d3def901113e995095c2e2b5"}],"deployment/haproxy/haproxy-container-puppet.yaml":[{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"9eb26bd4b5d3e6d695fb076eadb3834995e4b956","unresolved":true,"context_lines":[{"line_number":37,"context_line":"    type: json"},{"line_number":38,"context_line":"    default:"},{"line_number":39,"context_line":"      tag: openstack.haproxy"},{"line_number":40,"context_line":"      file: /var/log/containers/haproxy/haproxy.log"},{"line_number":41,"context_line":"  HAProxySyslogAddress:"},{"line_number":42,"context_line":"    default: /dev/log"},{"line_number":43,"context_line":"    description: Syslog address where HAproxy will send its log"}],"source_content_type":"text/x-yaml","patch_set":2,"id":"343bc38c_28fa4e7f","line":40,"updated":"2021-12-03 12:51:22.000000000","message":"don\u0027t you need to specify some startmsg.regex here as well?","commit_id":"bf59301c6723884aa5cdf12fb83e098f2f3e0775"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"9e3e46c7b7faa5cab36adbd9f402d23b4e4f3609","unresolved":true,"context_lines":[{"line_number":37,"context_line":"    type: json"},{"line_number":38,"context_line":"    default:"},{"line_number":39,"context_line":"      tag: openstack.haproxy"},{"line_number":40,"context_line":"      file: /var/log/containers/haproxy/haproxy.log"},{"line_number":41,"context_line":"  HAProxySyslogAddress:"},{"line_number":42,"context_line":"    default: /dev/log"},{"line_number":43,"context_line":"    description: Syslog address where HAproxy will send its log"}],"source_content_type":"text/x-yaml","patch_set":2,"id":"7e5f12cf_0c75df8e","line":40,"in_reply_to":"343bc38c_28fa4e7f","updated":"2021-12-03 15:48:15.000000000","message":"Thanks for the eyes, Bogdan! I\u0027m not 100% sure the answer since I have not had an opportunity to try deploying haproxy without pacemaker to check what the logs look like in that case.\n\nThe default regex of \"^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) \" applies to all services unless they are overridden here, and virtually all services use the default because they output lines like this:\n\n2021-12-03 15:39:00.323 11 INFO nova.api.openstack.requestlog [...]\n\nPacemaker outputs a different format, which is why we supply the regex when running under pacemaker[1]. Here is a sample of HAProxy log while running under pacemaker:\n\nDec  3 15:40:12 controller-0 haproxy[21]: [...]\n\n[1] https://bugzilla.redhat.com/show_bug.cgi?id\u003d2026290#c5","commit_id":"bf59301c6723884aa5cdf12fb83e098f2f3e0775"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"bb0dd4717d0e113c389beee95f2fa65fcfe6215f","unresolved":true,"context_lines":[{"line_number":37,"context_line":"    type: json"},{"line_number":38,"context_line":"    default:"},{"line_number":39,"context_line":"      tag: openstack.haproxy"},{"line_number":40,"context_line":"      file: /var/log/containers/haproxy/haproxy.log"},{"line_number":41,"context_line":"  HAProxySyslogAddress:"},{"line_number":42,"context_line":"    default: /dev/log"},{"line_number":43,"context_line":"    description: Syslog address where HAproxy will send its log"}],"source_content_type":"text/x-yaml","patch_set":2,"id":"76fa8ae2_0308733c","line":40,"in_reply_to":"7e5f12cf_0c75df8e","updated":"2021-12-06 15:37:46.000000000","message":"Okay, while everything I wrote about is true, the pep8 tests refuse to pass my deployment templates, CI is failing with:\n\nMismatched parameter definitions found for \"HAProxyLoggingSource\"\nDefinitions found:\n  ./deployment/haproxy/haproxy-pacemaker-puppet.yaml:\n    {\u0027type\u0027: \u0027json\u0027, \u0027default\u0027: {\u0027tag\u0027: \u0027openstack.haproxy\u0027, \u0027file\u0027: \u0027/var/log/containers/haproxy/haproxy.log\u0027, \u0027startmsg.regex\u0027: \u0027^[a-zA-Z]{3} [0-9]{2} [:0-9]{8}\u0027}}\n  ./deployment/haproxy/haproxy-container-puppet.yaml:\n    {\u0027type\u0027: \u0027json\u0027, \u0027default\u0027: {\u0027tag\u0027: \u0027openstack.haproxy\u0027, \u0027file\u0027: \u0027/var/log/containers/haproxy/haproxy.log\u0027}}\n\nLocally, I tried adding the correct startmsg.regex to the container-puppet.yaml, but the tests still complain they are mismatched: \n\nMismatched parameter definitions found for \"HAProxyLoggingSource\"                                                                                                                        Definitions found:                                                                                                                                                                      \n  ./deployment/haproxy/haproxy-container-puppet.yaml:                                                                                                                                   \n    {\u0027type\u0027: \u0027json\u0027, \u0027default\u0027: {\u0027tag\u0027: \u0027openstack.haproxy\u0027, \u0027file\u0027: \u0027/var/log/containers/haproxy/haproxy.log\u0027, \u0027startmsg.regex\u0027: \u0027^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\n(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) \u0027}}                                                                                                                                        \n  ./deployment/haproxy/haproxy-pacemaker-puppet.yaml:                                                                                                                                   \n    {\u0027type\u0027: \u0027json\u0027, \u0027default\u0027: {\u0027tag\u0027: \u0027openstack.haproxy\u0027, \u0027file\u0027: \u0027/var/log/containers/haproxy/haproxy.log\u0027, \u0027startmsg.regex\u0027: \u0027^[a-zA-Z]{3} [0-9]{2} [:0-9]{8}\u0027}}\n\n\nI will rename this parameter in one copy of the file to get around this \"problem\"","commit_id":"bf59301c6723884aa5cdf12fb83e098f2f3e0775"}],"deployment/haproxy/haproxy-pacemaker-puppet.yaml":[{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"3647bacb517b6f36a8e5124e8a1d8c9da2dc4099","unresolved":true,"context_lines":[{"line_number":81,"context_line":"  HAProxyInternalTLSKeysDirectory:"},{"line_number":82,"context_line":"    default: \u0027/etc/pki/tls/private/haproxy\u0027"},{"line_number":83,"context_line":"    type: string"},{"line_number":84,"context_line":"  HAProxyPMLoggingSource:"},{"line_number":85,"context_line":"    type: json"},{"line_number":86,"context_line":"    default:"},{"line_number":87,"context_line":"      tag: openstack.haproxy"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"29b31d2a_65a2030c","line":84,"range":{"start_line":84,"start_character":2,"end_line":84,"end_character":24},"updated":"2021-12-08 22:01:14.000000000","message":"Why is this a separate var from the non-pacemaker version? IIRC you cannot run both at the same time so we should be able to leverage the same variable for both.","commit_id":"95fd2d705402b89f3c21438fdbeede2cf209b6f4"},{"author":{"_account_id":30893,"name":"Chris Sibbitt","email":"csibbitt@redhat.com","username":"csibbitt"},"change_message_id":"60a2c611cb242128dc890d0b035e69d26c780a8d","unresolved":true,"context_lines":[{"line_number":81,"context_line":"  HAProxyInternalTLSKeysDirectory:"},{"line_number":82,"context_line":"    default: \u0027/etc/pki/tls/private/haproxy\u0027"},{"line_number":83,"context_line":"    type: string"},{"line_number":84,"context_line":"  HAProxyPMLoggingSource:"},{"line_number":85,"context_line":"    type: json"},{"line_number":86,"context_line":"    default:"},{"line_number":87,"context_line":"      tag: openstack.haproxy"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"7a8adac0_f7d1b77e","line":84,"range":{"start_line":84,"start_character":2,"end_line":84,"end_character":24},"in_reply_to":"0e6ae1df_1063e405","updated":"2021-12-09 15:10:34.000000000","message":"Ah, thanks for this! I was not sure where to find that non-pacemaker log or how to set up my environment to use it. I can see that the default regex (from the puppet) doesn\u0027t apply here, so both should have the startmsg.regex config, and can share a name. I will submit a new patch. Thanks, again!","commit_id":"95fd2d705402b89f3c21438fdbeede2cf209b6f4"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"256f0e145ca923278f173e9e8f02e926b9dfd9a9","unresolved":true,"context_lines":[{"line_number":81,"context_line":"  HAProxyInternalTLSKeysDirectory:"},{"line_number":82,"context_line":"    default: \u0027/etc/pki/tls/private/haproxy\u0027"},{"line_number":83,"context_line":"    type: string"},{"line_number":84,"context_line":"  HAProxyPMLoggingSource:"},{"line_number":85,"context_line":"    type: json"},{"line_number":86,"context_line":"    default:"},{"line_number":87,"context_line":"      tag: openstack.haproxy"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"0e6ae1df_1063e405","line":84,"range":{"start_line":84,"start_character":2,"end_line":84,"end_character":24},"in_reply_to":"29b31d2a_65a2030c","updated":"2021-12-08 22:26:12.000000000","message":"I saw the commentary about the different log formats but they are the same if they are under pacemaker or not.\n\n\nuncontainerized https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_815/820594/1/check/tripleo-ci-centos-8-containers-multinode/8157e9d/logs/subnode-1/var/log/containers/haproxy/haproxy.log\n\nDec  7 01:06:12 centos-8-stream-rax-ord-0027620314 haproxy[12]: 192.168.24.2:34294 [07/Dec/2021:01:06:11.523] nova_osapi nova_osapi/centos-8-stream-rax-ord-0027620314.internalapi.localdomain 0/0/0/691/691 200 2152 - - ---- 48/2/1/1/0 0/0 \"GET /v2.1/servers/69088f91-80dc-4ac8-9cbb-1d85ad9695d8 HTTP/1.1\"\n\n\n\n\ncontainerized https://d29123bcde0971fd2051-b66a943d9312fbe3ccb3a2881f2cac57.ssl.cf2.rackcdn.com/820404/2/gate/tripleo-ci-centos-8-containers-multinode/a2bf747/logs/subnode-1/var/log/containers/haproxy/haproxy.log\n\n\nDec  6 20:48:30 centos-8-stream-rax-dfw-0027618124 haproxy[12]: 192.168.24.2:43344 [06/Dec/2021:20:48:30.346] nova_osapi nova_osapi_be/centos-8-stream-rax-dfw-0027618124.ctlplane.localdomain 0/0/0/4/4 200 523 - - ---- 29/1/0/1/0 0/0 \"GET / HTTP/1.1\"","commit_id":"95fd2d705402b89f3c21438fdbeede2cf209b6f4"}]}