)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"46e61d74cf3b7966b3cb65022dc14da125b2d3ce","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":46,"id":"82fb4af3_6eae741f","updated":"2021-10-27 21:18:05.000000000","message":"recheck","commit_id":"1a5072d2fdbe5e3e7c526f152976f18588b2eebc"},{"author":{"_account_id":34120,"name":"Andre Aranha","display_name":"afariasa","email":"afariasa@redhat.com","username":"afariasa"},"change_message_id":"d89366482b2600062cf1744a16b0cc5b69b100d5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":48,"id":"44462402_5e7b1f3f","updated":"2021-11-05 15:44:34.000000000","message":"recheck","commit_id":"de819aac7cb0043d8076c18e1fcd25069d394b90"},{"author":{"_account_id":25402,"name":"Francesco Pantano","email":"fpantano@redhat.com","username":"fmount"},"change_message_id":"15b29d1728f66e9f184b31d417ce106f296d55ce","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":49,"id":"faf99b53_14847332","updated":"2021-11-16 16:40:10.000000000","message":"Hard to say why tempest fails in scenario004 complaining with \"503 - No server is available to handle this request\" for rgw during swift_basic_ops [1].\nI tried to dig more into scenario004 logs but looks like rgw is happy [2] and a lot of call are properly forwarded from haproxy to rgw, and the processes are up \u0026 running.\nHowever I\u0027m not sure about the pcs status at the time of the tempest run [5], and I would check that first.\nLet\u0027s recheck this change and double check the failure is consistent.\nAlso, does the same issue happening in scenario001 (where RGW is deployed as well) ?\n\n\n\n\n[1] https://zuul.opendev.org/t/openstack/build/c46f616c7a0a4e9194d4f2b20fbfc9c4/log/logs/undercloud/var/log/tempest/tempest_run.log\n[2] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c46/808215/49/check/tripleo-ci-centos-8-scenario004-standalone/c46f616/logs/undercloud/var/log/extra/podman/containers/ceph-4b5c8c0a-ff60-454b-a1b4-9747aa737d19-rgw.rgw.standalone.nivipg/stdout.log\n[3] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c46/808215/49/check/tripleo-ci-centos-8-scenario004-standalone/c46f616/logs/undercloud/var/log/containers/haproxy/haproxy.log\n[4] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c46/808215/49/check/tripleo-ci-centos-8-scenario004-standalone/c46f616/logs/undercloud/var/log/extra/netstat.txt\n[5] https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c46/808215/49/check/tripleo-ci-centos-8-scenario004-standalone/c46f616/logs/undercloud/var/log/pacemaker/pacemaker.log","commit_id":"4d23489b8bf4a6a62c0b2f11d4c0d4b20df3c565"},{"author":{"_account_id":34120,"name":"Andre Aranha","display_name":"afariasa","email":"afariasa@redhat.com","username":"afariasa"},"change_message_id":"556ffd0499d25b2dcbe3e5ea063f6d52691a0e0d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":49,"id":"3543191b_0d96a1ed","updated":"2021-11-16 11:50:34.000000000","message":"recheck","commit_id":"4d23489b8bf4a6a62c0b2f11d4c0d4b20df3c565"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"a90f49a6ff38b350b3af69278c8cbb4917a11fb1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":49,"id":"4b7e68f5_75b8b00f","updated":"2021-11-08 07:01:52.000000000","message":"recheck","commit_id":"4d23489b8bf4a6a62c0b2f11d4c0d4b20df3c565"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"8cd96267e702c61ab899e56e629d814abcd56f8f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":49,"id":"883b33ff_d09a5282","updated":"2021-11-08 15:04:51.000000000","message":"recheck","commit_id":"4d23489b8bf4a6a62c0b2f11d4c0d4b20df3c565"},{"author":{"_account_id":34120,"name":"Andre Aranha","display_name":"afariasa","email":"afariasa@redhat.com","username":"afariasa"},"change_message_id":"afc864a5100a63631157336a5b88c0a44e8c9c96","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":49,"id":"981add4e_1f5e7051","updated":"2021-11-08 14:57:26.000000000","message":"recheck","commit_id":"4d23489b8bf4a6a62c0b2f11d4c0d4b20df3c565"},{"author":{"_account_id":34120,"name":"Andre Aranha","display_name":"afariasa","email":"afariasa@redhat.com","username":"afariasa"},"change_message_id":"67e21d038f514e2a34470b3ec9525a1269d8bec1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":49,"id":"e009764d_8843dd34","updated":"2021-11-15 10:47:15.000000000","message":"recheck","commit_id":"4d23489b8bf4a6a62c0b2f11d4c0d4b20df3c565"},{"author":{"_account_id":30002,"name":"Douglas Viroel","email":"viroel@gmail.com","username":"dviroel"},"change_message_id":"c9b5ed5290eb67533a4e0045afeec305544ea887","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":55,"id":"c605107a_db1460e1","updated":"2021-11-23 12:02:04.000000000","message":"recheck\n\ntest increase container collect log timeouts","commit_id":"09a499b60f14ec8eb4461d55a49d7938b76393be"},{"author":{"_account_id":30002,"name":"Douglas Viroel","email":"viroel@gmail.com","username":"dviroel"},"change_message_id":"7aa8077f1024cf322757dfa2759aeb4bf085a048","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":58,"id":"c54d6038_a78ff2b8","updated":"2021-11-29 18:19:23.000000000","message":"recheck\n\none or two dependencies were rebased","commit_id":"39645f6a10d2dacdaf8d643db02ee8b6f425d4b8"},{"author":{"_account_id":30002,"name":"Douglas Viroel","email":"viroel@gmail.com","username":"dviroel"},"change_message_id":"4875baed0baf27cf9a5cffa1670862c4d2cb4954","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":58,"id":"02d92883_8b56914f","updated":"2021-11-29 18:46:29.000000000","message":"recheck\n\ntempest dependencies rebased","commit_id":"39645f6a10d2dacdaf8d643db02ee8b6f425d4b8"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"9a2db53d8cd9ec1a3744dafab9792a6436993942","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"07893741_46936a6a","updated":"2021-11-30 04:10:25.000000000","message":"recheck","commit_id":"7e64e4722403903d37fe65b2f3b86000708f7d18"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"46de45cd543e6801221b70a302e96f0d73cb4ec8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"37788873_93e1e176","updated":"2021-11-30 06:19:55.000000000","message":"recheck","commit_id":"7e64e4722403903d37fe65b2f3b86000708f7d18"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"061f81e4a9cfd7b1cbc082944e0b6dbfd6fe6160","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"79ba7af0_e5d391ab","updated":"2021-11-30 14:51:43.000000000","message":"recheck","commit_id":"7e64e4722403903d37fe65b2f3b86000708f7d18"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"c34a546f412e5fb9cd016dc33ecd2227ef6d0f1e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":59,"id":"f343602b_dcd97585","updated":"2021-11-30 07:05:27.000000000","message":"recheck","commit_id":"7e64e4722403903d37fe65b2f3b86000708f7d18"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ec03103fd0d8928be7f02bc95de74f432019f985","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":60,"id":"f94f2aab_d46aac51","updated":"2021-12-02 19:37:07.000000000","message":"- Tempest gets 500 style errors \"bad gateway\" between 2021-12-01 21:20:26,953 and 2021-12-01 21:20:32,979 [1]\n- Beast first starts running about 10 seconds later 2021-12-01T21:20:41.602 [2]\n\nIf tempest tries to use haproxy to reach beast and best isn\u0027t running, haproxy will report back a bad gateway.\n\nIf we delay the tempest test with a sleep 1m and it passes that might confirm we\u0027re hitting a race.\n\nIs there a FIPs reason RGW wasn\u0027t ready yet?\n\n\n[1] https://03bdeb201ad5b10b1d29-c8e8f0465e65b3e97bf6a5ab87ec6f98.ssl.cf5.rackcdn.com/808215/60/check/tripleo-ci-centos-8-scenario004-standalone/dbf11a5/logs/undercloud/var/log/tempest/stestr_results.html\n[2] https://03bdeb201ad5b10b1d29-c8e8f0465e65b3e97bf6a5ab87ec6f98.ssl.cf5.rackcdn.com/808215/60/check/tripleo-ci-centos-8-scenario004-standalone/dbf11a5/logs/undercloud/var/log/extra/podman/containers/ceph-4b5c8c0a-ff60-454b-a1b4-9747aa737d19-rgw.rgw.standalone.owrezy/stdout.log\n\n\n","commit_id":"b31ae3e2d04bab45d85aae020a26ba9d758e9388"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"9f881b75a9c152b1f679dc456adc8700e0ffdb95","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":60,"id":"d82b58b3_9d6a378d","updated":"2021-12-02 19:01:37.000000000","message":"Try using v6.0.5-stable-6.0-pacific-centos-8-x86_64 to address the \"swift\" tempest failure\n\nDetails\n- I see the tempest anonymous access test failing [1].\n- You\u0027re using v6.0.4-stable-6.0-pacific-centos-8-x86_64 [2] which is 5 months old.\n- https://github.com/ceph/ceph/pull/37438 backported the fix for anonymous access to Pacific\n- v6.0.5 is the latest stable pacific build [3] and might contain the fix (if it fails the same way we might have to request a new build)\n- Downstream the fix is verified in RHCSv5 4-57 as per BZ 1960306\n- v6.0.5-stable-6.0-pacific-centos-8-x86_64 doesn\u0027t break anything as per our DNM THT patch [0] (so we should probably promote it)\n\n\n[1] https://03bdeb201ad5b10b1d29-c8e8f0465e65b3e97bf6a5ab87ec6f98.ssl.cf5.rackcdn.com/808215/60/check/tripleo-ci-centos-8-scenario004-standalone/dbf11a5/logs/undercloud/var/log/tempest/stestr_results.html\n\n[2] https://03bdeb201ad5b10b1d29-c8e8f0465e65b3e97bf6a5ab87ec6f98.ssl.cf5.rackcdn.com/808215/60/check/tripleo-ci-centos-8-scenario004-standalone/dbf11a5/logs/undercloud/home/zuul/containers-prepare-parameters.yaml\n\n[3] https://quay.io/repository/ceph/daemon?tab\u003dtags\n\n[4] https://review.opendev.org/c/openstack/tripleo-heat-templates/+/778915","commit_id":"b31ae3e2d04bab45d85aae020a26ba9d758e9388"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"548617fb8ad519ad685bf8060d7efd3b6037fdd7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":60,"id":"213553a0_19afdfaa","in_reply_to":"6585e57d_4af89da8","updated":"2022-01-28 08:02:37.000000000","message":"Ack","commit_id":"b31ae3e2d04bab45d85aae020a26ba9d758e9388"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"b809b4a797f1ca79d39da68c4bce3ff21733b7ed","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":60,"id":"6585e57d_4af89da8","in_reply_to":"d82b58b3_9d6a378d","updated":"2021-12-02 19:21:58.000000000","message":"\u003e Try using v6.0.5-stable-6.0-pacific-centos-8-x86_64 to address the \"swift\" tempest failure\n\nIgnore that. It\u0027s not going to be helped by https://github.com/ceph/ceph/pull/37438 since tempest is getting not a diff type of 400 reponse. It\u0027s getting a 500 response!","commit_id":"b31ae3e2d04bab45d85aae020a26ba9d758e9388"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"d458c51bded13cb2ed92163e8785a4283314bff1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":61,"id":"f1807fd8_bbf6a212","updated":"2021-12-02 22:50:34.000000000","message":"recheck","commit_id":"2432fe7c1f43b2f21f54d55253bae50de223500b"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"ad2a353e73fdbf02e4a7025b3d9245aca5f9d7a9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"f146fe70_524fea3e","updated":"2021-12-09 13:13:48.000000000","message":"- RGW did not segfault [1] when using quay.ceph.io/ceph-ci/daemon:latest-devel\n- The Tempest object storage test failed [2] again but in a very different way; a timeout\n- Tempest did a GET which returned a 204 [3] and obtained rgw status information [4]\n- It then tried to create a container [5] which timed out [6]\n- We have Tempest 1:29.2.1-0.20211209010117.2b3cba3 [7] as per the dnf log [8]\n- The failure was in test_object_storage_basic_ops.py [9] [10]\n- The same create container timeout happend for both test_swift_basic_ops and test_swift_acl_anonymous_download [2]\n- The update_container() method was called [11] and it does a PUT with an empty body and some headers but timed out\n- As per the docs for the headers [12] it should pass something like X -Container-Meta-{name} header, where {name} is the name of the metadata item.\n- The name was defined as container_name \u003d data_utils.rand_name(name\u003d\u0027TestContainer\u0027) [13]\n- Nothing remarkable about the name generation that I can see [14]\n- Timeout was between 2021-12-09 04:07:59,999 and 2021-12-09 04:17:00,481 [2]\n- There is nothing telling in the RGW logs during that time [1] (just health check 200)\n- HAproxy logs [15] show the 8 PUTs to ceph_rgw [16]\n- So the tempest client and haproxy sent 8 PUTs to RGW but RGW says it doesn\u0027t know anything about that\n- We\u0027re running RGW with full debug AFAICT [17]\n- We might need RGW developer help\n\n[1] https://32647e7af55c64e6cb5a-4ceca8fad823094091d79c08094977d0.ssl.cf2.rackcdn.com/808215/63/check/tripleo-ci-centos-8-scenario004-standalone/db53dd4/logs/undercloud/var/log/ceph/4b5c8c0a-ff60-454b-a1b4-9747aa737d19/ceph-client.rgw.rgw.standalone.ctuokr.log\n\n[2] https://32647e7af55c64e6cb5a-4ceca8fad823094091d79c08094977d0.ssl.cf2.rackcdn.com/808215/63/check/tripleo-ci-centos-8-scenario004-standalone/db53dd4/logs/undercloud/var/log/tempest/stestr_results.html\n\n[3] 2021-12-09 04:20:00,646 242656 INFO     [tempest.lib.common.rest_client] Request (TestObjectStorageBasicOps:test_swift_basic_ops): 204 GET http://192.168.24.3:8080/swift/v1/AUTH_6d2932a26f824eb28a905f3fd2b82aea 0.018s\n\n[4] 2021-12-09 04:20:00,648 242656 DEBUG    [tempest.scenario.manager] Swift status information obtained successfully\n\n[5] container_name \u003d self.create_container()\n\n[6] urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host\u003d\u0027192.168.24.3\u0027, port\u003d8080): Max retries exceeded with url: /swift/v1/AUTH_6d2932a26f824eb28a905f3fd2b82aea/tempest-swift-scenario-container-13863261 (Caused by ReadTimeoutError(\"HTTPConnectionPool(host\u003d\u0027192.168.24.3\u0027, port\u003d8080): Read timed out. (read timeout\u003d60)\",))\n\n[7]\n2021-12-09T04:00:56+0000 DEBUG Installed: openstack-tempest-1:29.2.1-0.20211209010117.2b3cba3.el8.noarch\n2021-12-09T04:00:56+0000 DEBUG Installed: python3-tempest-1:29.2.1-0.20211209010117.2b3cba3.el8.noarch\n\n[8] https://32647e7af55c64e6cb5a-4ceca8fad823094091d79c08094977d0.ssl.cf2.rackcdn.com/808215/63/check/tripleo-ci-centos-8-scenario004-standalone/db53dd4/logs/undercloud/var/log/dnf.log\n\n[9] File \"/usr/lib/python3.6/site-packages/tempest/scenario/test_object_storage_basic_ops.py\", line 68, in test_swift_acl_anonymous_download\n\n[10] https://github.com/openstack/tempest/blob/master/tempest/scenario/test_object_storage_basic_ops.py#L60\n\n[11] https://github.com/openstack/tempest/blob/f9d31f2c46204abbbef53d3c0ae0f80125878fc9/tempest/lib/services/object_storage/container_client.py#L37\n\n[12] https://docs.openstack.org/api-ref/object-store/#create-container\n\n[13] https://github.com/openstack/tempest/blob/f9d31f2c46204abbbef53d3c0ae0f80125878fc9/tempest/api/object_storage/base.py#L112\n\n[14] https://github.com/openstack/tempest/blob/f9d31f2c46204abbbef53d3c0ae0f80125878fc9/tempest/lib/common/utils/data_utils.py#L43\n\n[15] https://32647e7af55c64e6cb5a-4ceca8fad823094091d79c08094977d0.ssl.cf2.rackcdn.com/808215/63/check/tripleo-ci-centos-8-scenario004-standalone/db53dd4/logs/undercloud/var/log/containers/haproxy/haproxy.log\n\n[16] Dec  9 04:08:59 standalone haproxy[14]: 192.168.24.3:44410 [09/Dec/2021:04:06:59.942] ceph_rgw ceph_rgw_be/standalone.ctlplane.localdomain 0/0/0/-1/120001 504 194 - - sH-- 38/2/1/1/0 0/0 \"PUT /swift/v1/AUTH_6d2932a26f824eb28a905f3fd2b82aea/tempest-swift-scenario-container-687038714 HTTP/1.1\"\n\n[17] https://review.opendev.org/c/openstack/tripleo-heat-templates/+/819697/5/ci/environments/scenario004-standalone.yaml","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"c9b24cebec765e18cd32e915cbe79422b12a4153","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"cd493158_dfc990ed","updated":"2021-12-06 14:32:50.000000000","message":"- The bad gateway 500 error happens at 2021-12-03 01:53:01,599 [1]\n- Beast gets running at 2021-12-03T01:55:21.721+0000 [2]\n- Thus, whatever process which delays the test until the beast is running did not delay enough\n\nTwo questions:\n\n1. Where is the code that added the delay?\n2. How about we modify it to send requests to the beast and sleep until it doesn\u0027t throw a 500 and the nwe run the tempest test?\n\n\n[1] https://5f48111572afd39f0f3d-de9c9aa92c96f031613f73d2359ce388.ssl.cf2.rackcdn.com/808215/62/check/tripleo-ci-centos-8-scenario004-standalone/bae314e/logs/undercloud/var/log/tempest/stestr_results.html\n[2] https://5f48111572afd39f0f3d-de9c9aa92c96f031613f73d2359ce388.ssl.cf2.rackcdn.com/808215/62/check/tripleo-ci-centos-8-scenario004-standalone/bae314e/logs/undercloud/var/log/extra/podman/containers/ceph-4b5c8c0a-ff60-454b-a1b4-9747aa737d19-rgw.rgw.standalone.vbovdd/stdout.log","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"1643b3e0c24aecf92356b1d251044801765bdb9c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"ed60afd4_3171ec10","updated":"2021-12-17 16:02:38.000000000","message":"Examining the train provider failure I have discovered that the tempest *patches* are the culprit. \nThis is the gist of it: Tempest is branch less, changes there impact all releases.\n\nWhen we run train job or any other, it will use the new code at some point. That would be fine, but when the build process starts, other patches are applied, those supplied in rpm. That\u0027s when the error shows up:\n\n----\nDEBUG: + /usr/bin/cat /builddir/build/SOURCES/0001-Revert-Remove-tempest-manager.py-after-4-year-deprec.patch\nDEBUG: + /usr/bin/git apply --index -\nDEBUG: error: patch failed: tools/generate-tempest-plugins-list.py:54\nDEBUG: error: tools/generate-tempest-plugins-list.py: patch does not apply\n----\n\nI\u0027m not certain, but it seems possible the we are dealing with merge problem here.\nOne of the tempest patches, possibly: \n\nhttps://review.opendev.org/c/openstack/tempest/+/820270\nhttps://review.opendev.org/c/openstack/tempest/+/810808\n\nWould have to touch some file altered by the patches in rmp. \n\nBut that\u0027s just my best hypothesis.","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"c5da452d18eb5cd18d9f24cfd731c8ce844529c8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"e3fe4d28_ffd30a60","updated":"2021-12-08 15:27:06.000000000","message":"If you want to test Ceph without RGW (but test it with RBD) and see if scenario001/004 get further, then you can direct tripleo to deploy swift for object storage in place of RGW by replacing this env file:\n\n  -e /usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm.yaml\n\n\nwith this one:\n\n  -e /usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm-rbd-only.yaml\n\nas per:\n\n  https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/features/cephadm.html#deploying-ceph-during-overcloud-deployment\n","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"d5b3d871be8f739b205e8f9436c5a1028b7ce916","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"e6b0dc0f_8a71b2d5","updated":"2021-12-06 19:35:15.000000000","message":"Let\u0027s see how things this time around with this change:\n\nhttps://review.opendev.org/c/openstack/tripleo-heat-templates/+/819697/3/ci/environments/scenario004-standalone.yaml\n","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"7a0482731a0036d40222d54e580059965d504590","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"85f0e142_895f39fc","updated":"2021-12-08 16:16:27.000000000","message":"Related to RGW failing with FIPS https://bugzilla.redhat.com/show_bug.cgi?id\u003d1789237 ","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"f07ccf1bdd16c5ffbff35516b8a6593a1d55c80e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"73939aa0_f059d890","updated":"2022-01-10 10:38:13.000000000","message":"The failure of the tempest DLRN build in the train branch is probably related to the changes between train and wallaby branches. Since the tempest is branch less, the differences between releases are handled by distgits and tags. \n\nIn our case, the application of \"0001-Revert-Remove-tempest-manager.py-after-4-year-deprec.patch\"\ncauses build failure as the patch in question is already in the master branch of the openstack/tempest project under commit[1] :\n\n---\nDEBUG: + /usr/bin/cat /builddir/build/SOURCES/0001-Revert-Remove-tempest-manager.py-after-4-year-deprec.patch\nDEBUG: + /usr/bin/git apply --index -\nDEBUG: error: patch failed: tools/generate-tempest-plugins-list.py:54\nDEBUG: error: tools/generate-tempest-plugins-list.py: patch does not apply\nDEBUG: RPM build errors:\nDEBUG: error: Bad exit status from /var/tmp/rpm-tmp.iIGPcv (%prep)\nDEBUG:     Bad exit status from /var/tmp/rpm-tmp.iIGPcv (%prep)\nDEBUG: Child return code was: 1\nINFO: EXCEPTION: [Error()]\nTraceback (most recent call last):\n  File \"/usr/lib/python3.6/site-packages/mockbuild/trace_decorator.py\", line 95, in trace\n    result \u003d func(*args, **kw)\n  File \"/usr/lib/python3.6/site-packages/mockbuild/util.py\", line 746, in do_with_status\n    raise exception.Error(\"Command failed: \\n # %s\\n%s\" % (command, output), child.returncode)\nmockbuild.exception.Error: Command failed: \n # /usr/bin/systemd-nspawn -q -M c3067fbca3d946c7a89853c99f843167 -D /var/lib/mock/dlrn-centos8-x86_64-1/root -a --capability\u003dcap_ipc_lock --bind\u003d/tmp/mock-resolv.b7ndkpwn:/etc/resolv.conf --setenv\u003dTERM\u003dvt100 --setenv\u003dSHELL\u003d/bin/bash --setenv\u003dHOME\u003d/builddir --setenv\u003dHOSTNAME\u003dmock --setenv\u003dPATH\u003d/usr/bin:/bin:/usr/sbin:/sbin --setenv\u003dPROMPT_COMMAND\u003dprintf \"\\033]0;\u003cmock-chroot\u003e\\007\" --setenv\u003dPS1\u003d\u003cmock-chroot\u003e \\s-\\v\\$  --setenv\u003dLANG\u003den_US.UTF-8 -u mockbuild bash --login -c /usr/bin/rpmbuild -bb --target x86_64 --nodeps /builddir/build/SPECS/openstack-tempest.spec\n----\n\n[1]https://wimp.usersys.redhat.com/?change_id_input\u003dIea813d478a93a1638be597af48929cd50a5e6366\n","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"2eb47f8a145dcbe178981ab790b3815bbb41f54e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"9d5b1384_7d803977","updated":"2021-12-07 00:12:17.000000000","message":"We can see 004 failed and RGW had four Segmentation faults (try ctrl-f \"Segmentation fault\" in the following)\n\nhttps://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_5c2/808215/63/check/tripleo-ci-centos-8-scenario004-standalone/5c27e42/logs/undercloud/var/log/ceph/4b5c8c0a-ff60-454b-a1b4-9747aa737d19/ceph-client.rgw.rgw.standalone.rxhxbg.log\n\nAs per the tempest test:\n\nhttps://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_5c2/808215/63/check/tripleo-ci-centos-8-scenario004-standalone/5c27e42/logs/undercloud/var/log/tempest/stestr_results.html\n\nTempest requested an object at 2021-12-06 23:02:37,877 and got a 502:\n\n2021-12-06 23:02:37,877 225881 INFO     [tempest.lib.common.rest_client] Request (TestObjectStorageBasicOps:test_swift_acl_anonymous_download): 502 PUT http://192.168.24.3:8080/swift/v1/AUTH_a4efdf467bca451caec25fd8eaa55fe9/tempest-swift-scenario-container-1034745309/tempest-swift-scenario-object-1994526551 4.705s\n2021-12-06 23:02:37,878 225881 DEBUG    [tempest.lib.common.rest_client] Request - Headers: {\u0027Content-Type\u0027: \u0027application/json\u0027, \u0027Accept\u0027: \u0027application/json\u0027, \u0027X-Auth-Token\u0027: \u0027\u003comitted\u003e\u0027}\n\nand RGW had segfauted two seconds earlier at 2021-12-06T23:02:35.160. \n\nceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)\n\nI think we now have sufficient data to open a Ceph RGW bug.\n","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"29ecb7fd38d1da652a375874e0e01708752cf370","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"31c89b29_ff445724","updated":"2021-12-08 20:50:34.000000000","message":"We conjecture this is [1] which hasn\u0027t yet been backported to pacific (which we\u0027re using)\nWe can test with latest-devel [2] to see if it reproduces.\n\n[1] https://tracker.ceph.com/issues/52900\n[2] https://quay.io/repository/ceph/daemon?tab\u003dtags","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"8cbaa9da190df11fffe1efbea954f9c978d5462a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"4693a9fa_e5aecbff","updated":"2021-12-17 19:46:43.000000000","message":"note --- checking now to see if failures still occur when swift is used instead of ceph rgw in scenario 4","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"78701d3e75e7de782dfe047912d82598b1105f4d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"28997f96_1a8a8b68","updated":"2021-12-06 19:34:14.000000000","message":"recheck","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"074195624b40036345f57ef334b078a1817367bd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"3215a115_71bc34b5","updated":"2021-12-06 19:40:51.000000000","message":"recheck","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"75383bcc699971c96c83365cf61a05c18f3af989","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"3385eb07_4c61956c","updated":"2021-12-09 00:32:05.000000000","message":"recheck","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"4f4e34b3a8c09af57e43524f568a9843d682e530","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"36b2dce4_dc8c97fe","updated":"2021-12-08 22:27:02.000000000","message":"recheck","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"887d801d590feb6f07ec0a1ed0b4c6fc21d3084e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"cd170892_9e7475d5","updated":"2021-12-17 19:45:49.000000000","message":"recheck","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":18002,"name":"John Fulton","email":"fulton@redhat.com","username":"fultonj"},"change_message_id":"f77a01f5d6b95f448595631efa38998347c1ec86","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":63,"id":"de6afc9b_d7930e3e","updated":"2021-12-08 21:22:06.000000000","message":"recheck","commit_id":"07b310bd3187f2b3ee73c62a10c862c311124af5"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"27966606a3d8fbaed9cd01ae5f7307251aa91ac8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":67,"id":"95769178_4bc467a0","updated":"2022-01-18 15:35:03.000000000","message":"recheck","commit_id":"4809aadfb1b5d04d60dcb5469ca96d16c6f4cfde"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"69ec6a3809f9e398af58bdd65fee609bdc262ede","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":67,"id":"9ba22815_8171bc5a","updated":"2022-01-18 14:37:52.000000000","message":"recheck","commit_id":"4809aadfb1b5d04d60dcb5469ca96d16c6f4cfde"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"5bca2bcab85a0b4034ac0bbaad108ce82ce0316a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":75,"id":"02c16297_3745eb08","updated":"2022-02-02 09:14:27.000000000","message":"Temporary ceph repo for nautilus was established, hopefully alleviating the https://bugs.launchpad.net/tripleo/+bug/1959656 somewhat. \n\nrecheck ","commit_id":"0d1c5da87b9ca4b59c9dfd822795473ae310cbdf"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"078c3510a3274f888a5645dacc2c74c5acd76c93","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":75,"id":"92cbff73_bee8d4dc","updated":"2022-02-04 09:37:57.000000000","message":"recheck","commit_id":"0d1c5da87b9ca4b59c9dfd822795473ae310cbdf"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"9572d085713ebedaec6755bef5023b41b4ce0ae7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":75,"id":"18c0d832_a3c59cbd","updated":"2022-02-04 14:32:29.000000000","message":"tripleo-ci-centos-8-scenario001-standalone is during deploy failing with: \nError: Could not prefetch mysql_user provider \u0027mysql\u0027: Execution of \u0027/usr/bin/mysql --defaults-extra-file\u003d/root/.my.cnf -NBe SELECT CONCAT(User, \u0027@\u0027,Host) AS User FROM mysql.user\u0027 returned 1: ERROR 2002 (HY000): Can\u0027t connect to local MySQL server through socket \u0027/var/lib/mysql/mysql.sock\u0027 (2) \n\ntripleo-ci-centos-8-scenario004-standalone fails during tempest tests \u0027tempest.lib.common.rest_client\u0027 with:\n\nBody: b\u0027{\"error\":{\"code\":400,\"message\":\"Invalid input for field \\\u0027auth_url\\\u0027: \\\u0027a1ab92143ad040c9a384df80ca24907b\\\u0027 does not match \\\u0027^[a-zA-Z0-9+.-]+:.+\\\u0027\\\\n\\\\nFailed validating \\\u0027pattern\\\u0027 in schema[\\\u0027properties\\\u0027][\\\u0027auth_url\\\u0027]:\\\\n    {\\\u0027maxLength\\\u0027: 225,\\\\n     \\\u0027minLength\\\u0027: 0,\\\\n     \\\u0027pattern\\\u0027: \\\u0027^[a-zA-Z0-9+.-]+:.+\\\u0027,\\\\n     \\\u0027type\\\u0027: \\\u0027string\\\u0027}\\\\n\\\\nOn instance[\\\u0027auth_url\\\u0027]:\\\\n    \\\u0027a1ab92143ad040c9a384df80ca24907b\\\u0027\",\"title\":\"Bad Request\"}}\\n\u0027","commit_id":"0d1c5da87b9ca4b59c9dfd822795473ae310cbdf"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"d5eccfb9de2fb9a816d7c66d03503c2cffc40e45","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":76,"id":"53001d58_679422df","updated":"2022-02-04 19:49:30.000000000","message":"recheck","commit_id":"9ca0ffc13efc65688e116d81086b412645628bc6"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"4c356a61e70668c9c4554266ec68c3d81bcf7519","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":78,"id":"1ba5858c_8bc077ff","updated":"2022-03-02 10:58:18.000000000","message":"Failure of the content provider job appears to be caused by rabbit container pulling centos-8 repos.[n+1]\n\n\nTrace:\n------\nPull Policy for pull [ifnewer]\"\\nerror building at STEP \"RUN dnf config-manager --add-repo http://people.redhat.com/jeckersb/centos-stream-8-rabbitmq-latest/centos-stream-8-rabbitmq-latest.repo\": error while running runtime: exit status 1\\nFailed to write to log, write /home/zuul/container-builds/35970492-571b-48e9-a0ee-56b9f38b609b/base/rabbitmq/rabbitmq-build.log: file already closed\\nFailed to write to log, write /home/zuul/container-builds/35970492-571b-48e9-a0ee-56b9f38b609b/base/rabbitmq/rabbitmq-build.log: file already closed\\n\u0027 clean_up /usr/lib/python3.9/site-packages/osc_lib/shell.py:496\u001b[00m\n\n[n+1]https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_d6f/808215/78/check/tripleo-ci-centos-9-content-provider/d6f706d/logs/undercloud/home/zuul/container_image_build.log","commit_id":"337803ac0c0a6cf23c7415a36477cbb5fdf8cbda"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"d313ac863a5d57cc082eeeb083c5041bd27563c3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":78,"id":"7fddf81f_9a598ba1","updated":"2022-03-16 10:16:21.000000000","message":"Since the introduction of Centos-9 jobs into master and creation of centos-9 specific patch[1] I\u0027m of the opinion that this review should be abandoned.\n\n[1]https://review.opendev.org/c/openstack/tripleo-ci/+/824479","commit_id":"337803ac0c0a6cf23c7415a36477cbb5fdf8cbda"},{"author":{"_account_id":32926,"name":"Jiri Podivin","display_name":"jpodivin","email":"jpodivin@redhat.com","username":"jpodivin"},"change_message_id":"a5711e7b25c45158772cafa603ee080d9a543b7d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":78,"id":"f8c05d45_926e762d","updated":"2022-03-02 10:53:22.000000000","message":"The centos-8 content provider was removed recently[n+1] this poses a new challenge as we have to make centos9 content provider FIPS compatible. Wallaby stable branch will see the same issues.\n\n[n+1]https://review.opendev.org/c/openstack/tripleo-ci/+/826317","commit_id":"337803ac0c0a6cf23c7415a36477cbb5fdf8cbda"}]}