)]}'
{"id":"openstack%2Fopenstack-helm~1014356","triplet_id":"openstack%2Fopenstack-helm~master~Ifec929bb3d1d88424e2dec50f91c4c1ce962c541","project":"openstack/openstack-helm","branch":"master","attention_set":{},"removed_from_attention_set":{"21040":{"account":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"last_update":"2026-11-06 15:34:38.000000000","reason":"\u003cGERRIT_ACCOUNT_21040\u003e replied on the change","reason_account":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"}},"34520":{"account":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"last_update":"2026-11-06 16:51:38.000000000","reason":"Change was submitted"}},"hashtags":[],"change_id":"Ifec929bb3d1d88424e2dec50f91c4c1ce962c541","subject":"fix(mariadb): Ride out Kubernetes API failures and stuck server pods","status":"MERGED","created":"2026-11-05 16:40:59.000000000","updated":"2026-11-06 16:52:59.000000000","submitted":"2026-11-06 16:51:38.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":2,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"1014356","meta_rev_id":"557c63b375e392fb2693f8b1e8ccefa8a2f6a8f6","_number":1014356,"virtual_id_number":1014356,"owner":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},{"value":0,"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},{"value":0,"_account_id":29974,"name":"Stephen Taylor","email":"staylor@mirantis.com","username":"st053q"},{"value":0,"_account_id":35943,"name":"Aleksei Chekunov","display_name":"Aleksei Chekunov","email":"chekunov87@gmail.com","username":"az7961"},{"tag":"autogenerated:zuul:gate","value":2,"date":"2026-11-06 16:51:38.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"recommended":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"all":[{"value":1,"date":"2026-11-06 15:34:38.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},{"value":2,"date":"2026-11-06 14:47:41.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},{"value":2,"date":"2026-11-06 13:12:02.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":29974,"name":"Stephen Taylor","email":"staylor@mirantis.com","username":"st053q"},{"value":1,"date":"2026-11-06 13:53:12.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":35943,"name":"Aleksei Chekunov","display_name":"Aleksei Chekunov","email":"chekunov87@gmail.com","username":"az7961"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","value":1,"default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"all":[{"value":0,"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},{"value":1,"date":"2026-11-06 14:47:41.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},{"value":0,"_account_id":29974,"name":"Stephen Taylor","email":"staylor@mirantis.com","username":"st053q"},{"value":0,"_account_id":35943,"name":"Aleksei Chekunov","display_name":"Aleksei Chekunov","email":"chekunov87@gmail.com","username":"az7961"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":29974,"name":"Stephen Taylor","email":"staylor@mirantis.com","username":"st053q"},{"_account_id":35943,"name":"Aleksei Chekunov","display_name":"Aleksei Chekunov","email":"chekunov87@gmail.com","username":"az7961"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2026-11-05 18:43:44.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2026-11-05 19:28:01.000000000","updated_by":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"reviewer":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"state":"CC"},{"updated":"2026-11-06 13:12:02.000000000","updated_by":{"_account_id":29974,"name":"Stephen Taylor","email":"staylor@mirantis.com","username":"st053q"},"reviewer":{"_account_id":29974,"name":"Stephen Taylor","email":"staylor@mirantis.com","username":"st053q"},"state":"REVIEWER"},{"updated":"2026-11-06 13:53:12.000000000","updated_by":{"_account_id":35943,"name":"Aleksei Chekunov","display_name":"Aleksei Chekunov","email":"chekunov87@gmail.com","username":"az7961"},"reviewer":{"_account_id":35943,"name":"Aleksei Chekunov","display_name":"Aleksei Chekunov","email":"chekunov87@gmail.com","username":"az7961"},"state":"REVIEWER"},{"updated":"2026-11-06 14:47:41.000000000","updated_by":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"reviewer":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"state":"REVIEWER"},{"updated":"2026-11-06 15:34:38.000000000","updated_by":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"reviewer":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"state":"REVIEWER"}],"messages":[{"id":"a554a782c39b4fdf2a4db219499ea48f0eb8cfa4","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"date":"2026-11-05 16:40:59.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"4b104aa0a3fa77c833d2c56b91a5a5a4fda3f7ef","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-11-05 18:43:44.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/3c8ce95dfeb6468a9ca88519eda90bb9\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/4b436b4efeb44e9fa9a747038b3534c4 : SUCCESS in 3m 56s\n- openstack-helm-linter https://zuul.opendev.org/t/openstack/build/30206cfc139b49378ba81264d9198236 : SUCCESS in 3m 16s\n- openstack-helm-pre-commit https://zuul.opendev.org/t/openstack/build/5acc0f90bfce4589877b06c7daddf272 : SUCCESS in 4m 40s\n- openstack-helm-release-notes https://zuul.opendev.org/t/openstack/build/a121f16513c147b997c775a955c53f67 : SUCCESS in 2m 21s\n- openstack-helm-bandit https://zuul.opendev.org/t/openstack/build/99627498df63420ab841319ff3cac172 : SUCCESS in 6m 12s\n- openstack-helm-build-charts https://zuul.opendev.org/t/openstack/build/0adb49b9e6434600b1406595d740e518 : SUCCESS in 1m 53s\n- openstack-helm-compute-kit-2025-2-ubuntu_noble https://zuul.opendev.org/t/openstack/build/450786eb07f343a1923b944b5f3162bb : SUCCESS in 1h 47m 22s\n- openstack-helm-compute-kit-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/e88ef16c75db45b7bf9d82abecdcdc74 : SUCCESS in 1h 12m 24s\n- openstack-helm-tls-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/b4286f06df734fa1a348b9ecf162c4b3 : SUCCESS in 1h 55m 16s\n- openstack-helm-crd-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/d5ec509761c74fe49b9a529c4a692a6e : SUCCESS in 1h 37m 24s\n- openstack-helm-operators-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/13242915386f4e2aac7f6f7fad037d55 : SUCCESS in 1h 14m 36s\n- openstack-helm-compute-kit-2026-2-ubuntu_resolute https://zuul.opendev.org/t/openstack/build/284ef34c59934f0e883118b52e40efe8 : SUCCESS in 1h 18m 32s\n- openstack-helm-monitoring https://zuul.opendev.org/t/openstack/build/5f3cf470077c497e81d092a29cd30055 : SUCCESS in 20m 31s","accounts_in_message":[],"_revision_number":1},{"id":"be2899166420005a6bb83cc01b135f70d18af463","author":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"date":"2026-11-05 19:28:01.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"810d5ed409932299aa2f18febc21a9e9ebb37cc5","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"date":"2026-11-05 20:42:59.000000000","message":"Uploaded patch set 2.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":2},{"id":"485afe6492d4d8491f2ccb963f361ab952851672","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"date":"2026-11-05 20:47:58.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"83cf416003d7aa9bc3a5d9f0531396a7c9019b45","author":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"date":"2026-11-05 20:48:38.000000000","message":"Patch Set 3:\n\n(1 comment)","accounts_in_message":[],"_revision_number":3},{"id":"502a46a5a4a9e599aee5fb1fd98b9eaa8a1ed44e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"date":"2026-11-05 21:40:48.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"76f95ed1f4ac43447aa5612357f561e2432612c6","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-11-05 23:25:04.000000000","message":"Patch Set 4: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/56866c7cfdc64b6cbb2ba9cfcd8b66b4\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/cd28aea1a4f0484787d3e1b2c693b960 : SUCCESS in 2m 14s\n- openstack-helm-linter https://zuul.opendev.org/t/openstack/build/4e900267d6e74dd192b45b55f287e473 : SUCCESS in 1m 51s\n- openstack-helm-pre-commit https://zuul.opendev.org/t/openstack/build/b5d6e8d9434a452c8f8f878ab691ff2d : SUCCESS in 1m 43s\n- openstack-helm-release-notes https://zuul.opendev.org/t/openstack/build/a3825e4c540d4a809eba9fea6cde60b8 : SUCCESS in 1m 27s\n- openstack-helm-bandit https://zuul.opendev.org/t/openstack/build/8a2e99ed944b46d2b6029dad3f497b5a : SUCCESS in 2m 50s\n- openstack-helm-build-charts https://zuul.opendev.org/t/openstack/build/a1464d9b0fc34b3aa167c6d521bb52d6 : SUCCESS in 2m 09s\n- openstack-helm-compute-kit-2025-2-ubuntu_noble https://zuul.opendev.org/t/openstack/build/2bd30e879d4c4a4ba8b0f889efa09db2 : SUCCESS in 1h 06m 45s\n- openstack-helm-compute-kit-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/06dff462862842f1845ab21cd1f4dae1 : SUCCESS in 1h 06m 53s\n- openstack-helm-tls-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/73c28841afac4e66a3e672f477ec6c2e : SUCCESS in 1h 41m 33s\n- openstack-helm-crd-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/f1b6f15a4fa143e0b070a8a85ec1b46e : SUCCESS in 1h 02m 55s\n- openstack-helm-operators-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/26502701c19b4b2d96afc9f79811e844 : SUCCESS in 1h 15m 19s\n- openstack-helm-compute-kit-2026-2-ubuntu_resolute https://zuul.opendev.org/t/openstack/build/50f598873b3f49b7ab3f8c1c7a852f82 : SUCCESS in 1h 17m 43s\n- openstack-helm-monitoring https://zuul.opendev.org/t/openstack/build/81ce2c7de4ef42a9a00887823d278c75 : SUCCESS in 22m 50s","accounts_in_message":[],"_revision_number":4},{"id":"ced0308b3c85572bfc80518b09b780f472131db1","author":{"_account_id":29974,"name":"Stephen Taylor","email":"staylor@mirantis.com","username":"st053q"},"date":"2026-11-06 13:12:02.000000000","message":"Patch Set 4: Code-Review+2","accounts_in_message":[],"_revision_number":4},{"id":"5d38f906fc81bbe7533dc4965e5d136e6b0e53c9","author":{"_account_id":35943,"name":"Aleksei Chekunov","display_name":"Aleksei Chekunov","email":"chekunov87@gmail.com","username":"az7961"},"date":"2026-11-06 13:53:12.000000000","message":"Patch Set 4: Code-Review+1","accounts_in_message":[],"_revision_number":4},{"id":"b603fee04f0f474f413de8ee84a96a4264c78b1e","author":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"date":"2026-11-06 14:47:41.000000000","message":"Patch Set 4: Code-Review+2 Workflow+1","accounts_in_message":[],"_revision_number":4},{"id":"7867832928ec64c79369081294e5ffbeeae79992","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-11-06 14:47:51.000000000","message":"Patch Set 4: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":4},{"id":"3b3811f2ddc28975b5576098d6c163335eb2ee7a","author":{"_account_id":21040,"name":"Alexey","email":"aodinokov@mirantis.com","username":"aodinokov"},"date":"2026-11-06 15:34:38.000000000","message":"Patch Set 4: Code-Review+1","accounts_in_message":[],"_revision_number":4},{"id":"26a5ce6f76a26738150a50989976e3302af9f329","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-11-06 16:51:38.000000000","message":"Patch Set 4: Verified+2\n\nBuild succeeded (gate pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/f3068aebb4ca40dd863c7f2064c2a414\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b15e256abb334c48b4a73cb551acdd95 : SUCCESS in 2m 52s\n- openstack-helm-linter https://zuul.opendev.org/t/openstack/build/0eb8bf819f4044cbb6cabbd661e3e02e : SUCCESS in 3m 54s\n- openstack-helm-release-notes https://zuul.opendev.org/t/openstack/build/05231702a8914c22ad3ce27774ad2837 : SUCCESS in 2m 40s\n- openstack-helm-compute-kit-2026-1-ubuntu_noble https://zuul.opendev.org/t/openstack/build/fd4c61ff22e9488da72f33f2b198bef1 : SUCCESS in 1h 49m 11s\n- openstack-helm-logging https://zuul.opendev.org/t/openstack/build/aedd855ab5994b33a26c572a1402419c : SUCCESS in 46m 04s\n- openstack-helm-monitoring https://zuul.opendev.org/t/openstack/build/913a51cdfe2f413896fbf621dfba164c : SUCCESS in 31m 23s","accounts_in_message":[],"_revision_number":4},{"id":"14e1a753f761ebe295e0826b3482c460a1f61171","tag":"autogenerated:gerrit:merged","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-11-06 16:51:38.000000000","message":"Change has been successfully merged","accounts_in_message":[],"_revision_number":4},{"id":"557c63b375e392fb2693f8b1e8ccefa8a2f6a8f6","tag":"autogenerated:zuul:promote","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-11-06 16:52:59.000000000","message":"Patch Set 4:\n\nBuild succeeded (promote pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/6f6b1b0f19ef4870ab4734c8e30d25bc\n\n- promote-openstack-tox-docs https://zuul.opendev.org/t/openstack/build/eb5d9940a6824849b225a0f6ad7f33ee : SUCCESS in 1m 02s","accounts_in_message":[],"_revision_number":4}],"current_revision_number":4,"current_revision":"c68d13e7676b1bf6f8eaec71ee202f4a66ccfc05","revisions":{"cb1e6fc472f2f855e9b206767f38e4cd9babc44d":{"kind":"REWORK","_number":1,"created":"2026-11-05 16:40:59.000000000","uploader":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"ref":"refs/changes/56/1014356/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/56/1014356/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/1"}}},"commit":{"parents":[{"commit":"9a6df85524ca1b1642ee38c2e30a2ca1dcee448b","subject":"refactor: Rename configmap-etc to secret-etc where it renders a Secret","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/9a6df85524ca1b1642ee38c2e30a2ca1dcee448b"}]}],"author":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 16:40:33.000000000","tz":0},"committer":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 16:40:33.000000000","tz":0},"subject":"fix(mariadb): Retry a failed check in mariadb-controller","message":"fix(mariadb): Retry a failed check in mariadb-controller\n\nmariadb-controller has no error handling around its loop, so a single\nfailed API call, such as a 504 from the apiserver while etcd is slow,\nends the process. Once the API is back the pod can still sit in\nCrashLoopBackOff for up to five minutes, with nothing to move the\nmaster service off a failed primary. Log the error and retry on the\nnext check, as mariadb-db-controller already does.\n\nstart.py also logs \"Declaring myself current cluster leader\" every half\nminute from whichever pod holds the lease that picks who bootstraps a\nnew cluster. That pod need not be the primary mariadb-controller points\nthe master service at, so the message reads as a split brain. Call it\nthe cluster bootstrap lease.\n\nChangelog: fixed: mariadb-controller recovers within one check after\n  a Kubernetes API failure instead of waiting out a CrashLoopBackOff\nChangelog: fixed: mariadb-server logs the bootstrap lease as such\n  rather than as the cluster leader, which is not the primary\nChange-Id: Ifec929bb3d1d88424e2dec50f91c4c1ce962c541\nSigned-off-by: Sergiy Markin \u003csmarkin@mirantis.com\u003e\nAssisted-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/cb1e6fc472f2f855e9b206767f38e4cd9babc44d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/cb1e6fc472f2f855e9b206767f38e4cd9babc44d"}]},"branch":"refs/heads/master"},"bd941e743ae7c728d98589c0c2c792fb1828ecb8":{"kind":"REWORK","_number":2,"created":"2026-11-05 20:42:59.000000000","uploader":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"ref":"refs/changes/56/1014356/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/56/1014356/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/2"}}},"commit":{"parents":[{"commit":"9a6df85524ca1b1642ee38c2e30a2ca1dcee448b","subject":"refactor: Rename configmap-etc to secret-etc where it renders a Secret","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/9a6df85524ca1b1642ee38c2e30a2ca1dcee448b"}]}],"author":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 16:40:33.000000000","tz":0},"committer":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 20:42:18.000000000","tz":0},"subject":"fix(mariadb): Ride out Kubernetes API failures and stuck server pods","message":"fix(mariadb): Ride out Kubernetes API failures and stuck server pods\n\nmariadb-controller has no error handling around its loop, so a single\nfailed API call, such as a 504 from the apiserver while etcd is slow,\nends the process. Once the API is back the pod can still sit in\nCrashLoopBackOff for up to five minutes, with nothing to move the\nmaster service off a failed primary. Log the error and retry on the\nnext check, as mariadb-db-controller already does.\n\nstart.py makes its Kubernetes API calls without a timeout, so a call\non a connection the apiserver no longer answers waits until TCP gives\nup. On SIGTERM it then joins the threads making those calls, and the\npod stays in Terminating for the whole grace period, ten minutes by\ndefault, after mysqld has stopped cleanly. Give every call a timeout,\nwake the threads as soon as the pod stops, and do not join them.\n\nAfter a full restart the servers also wait until every one of them\nhas reported its position in the state configmap within the last 50\nseconds. A server pod that stays in Terminating after its container\nhas exited, for example while its RBD volume is torn down, never\nreports again, so the whole cluster stays down. Its last report is no\nhelp either: the monitor stops before mysqld, so it is the -1 of a\nrunning server. On SIGTERM publish the position of the clean shutdown,\nand accept an old report with a seqno of 0 or more as final: seqno is\n-1 while mysqld runs, so such a server has stopped at that position\nand cannot move past it before it starts and reports again. A server\nthat did not stop cleanly still holds the recovery back.\n\nstart.py also logs \"Declaring myself current cluster leader\" every\nhalf minute from whichever pod holds the lease that picks who\nbootstraps a new cluster. That pod need not be the primary that\nmariadb-controller points the master service at, so the message reads\nas a split brain. Call it the cluster bootstrap lease.\n\nChangelog: fixed: mariadb-controller recovers within one check after\n  a Kubernetes API failure instead of waiting out a CrashLoopBackOff\nChangelog: fixed: mariadb-server exits as soon as mysqld stops on\n  SIGTERM, instead of waiting out the grace period when the\n  Kubernetes API does not respond\nChangelog: fixed: mariadb recovers from a full restart while a server\n  pod that stopped cleanly is still in Terminating, instead of\n  waiting for that pod to come back\nChangelog: fixed: mariadb-server logs the bootstrap lease as such\n  rather than as the cluster leader, which is not the primary\nChange-Id: Ifec929bb3d1d88424e2dec50f91c4c1ce962c541\nSigned-off-by: Sergiy Markin \u003csmarkin@mirantis.com\u003e\nAssisted-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/bd941e743ae7c728d98589c0c2c792fb1828ecb8"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/bd941e743ae7c728d98589c0c2c792fb1828ecb8"}]},"branch":"refs/heads/master"},"b7a0469f5367e334b396048262012ec34f9a2c40":{"kind":"REWORK","_number":3,"created":"2026-11-05 20:47:58.000000000","uploader":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"ref":"refs/changes/56/1014356/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/56/1014356/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/3"}}},"commit":{"parents":[{"commit":"9a6df85524ca1b1642ee38c2e30a2ca1dcee448b","subject":"refactor: Rename configmap-etc to secret-etc where it renders a Secret","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/9a6df85524ca1b1642ee38c2e30a2ca1dcee448b"}]}],"author":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 16:40:33.000000000","tz":0},"committer":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 20:47:45.000000000","tz":0},"subject":"fix(mariadb): Ride out Kubernetes API failures and stuck server pods","message":"fix(mariadb): Ride out Kubernetes API failures and stuck server pods\n\nmariadb-controller has no error handling around its loop, so a single\nfailed API call, such as a 504 from the apiserver while etcd is slow,\nends the process. Once the API is back the pod can still sit in\nCrashLoopBackOff for up to five minutes, with nothing to move the\nmaster service off a failed primary. Log the error and retry on the\nnext check, as mariadb-db-controller already does.\n\nstart.py makes its Kubernetes API calls without a timeout, so a call\non a connection the apiserver no longer answers waits until TCP gives\nup. On SIGTERM it then joins the threads making those calls, and the\npod stays in Terminating for the whole grace period, ten minutes by\ndefault, after mysqld has stopped cleanly. Give every call a timeout,\nwake the threads as soon as the pod stops, and do not join them.\n\nAfter a full restart the servers also wait until every one of them\nhas reported its position in the state configmap within the last 50\nseconds. A server pod that stays in Terminating after its container\nhas exited, for example while its RBD volume is torn down, never\nreports again, so the whole cluster stays down. Its last report is no\nhelp either: the monitor stops before mysqld, so it is the -1 of a\nrunning server. On SIGTERM publish the position of the clean shutdown,\nand accept an old report with a seqno of 0 or more as final: seqno is\n-1 while mysqld runs, so such a server has stopped at that position\nand cannot move past it before it starts and reports again. A server\nthat did not stop cleanly still holds the recovery back.\n\nstart.py also logs \"Declaring myself current cluster leader\" every\nhalf minute from whichever pod holds the lease that picks who\nbootstraps a new cluster. That pod need not be the primary that\nmariadb-controller points the master service at, so the message reads\nas a split brain. Call it the cluster bootstrap lease.\n\nChangelog: fixed: mariadb-controller recovers within one check after\n  a Kubernetes API failure instead of waiting out a CrashLoopBackOff\nChangelog: fixed: mariadb-server exits as soon as mysqld stops on\n  SIGTERM, instead of waiting out the grace period when the\n  Kubernetes API does not respond\nChangelog: fixed: mariadb recovers from a full restart while a server\n  pod that stopped cleanly is still in Terminating, instead of\n  waiting for that pod to come back\nChangelog: fixed: mariadb-server logs the bootstrap lease as such\n  rather than as the cluster leader, which is not the primary\nChange-Id: Ifec929bb3d1d88424e2dec50f91c4c1ce962c541\nSigned-off-by: Sergiy Markin \u003csmarkin@mirantis.com\u003e\nAssisted-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/b7a0469f5367e334b396048262012ec34f9a2c40"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/b7a0469f5367e334b396048262012ec34f9a2c40"}]},"branch":"refs/heads/master"},"c68d13e7676b1bf6f8eaec71ee202f4a66ccfc05":{"kind":"REWORK","_number":4,"created":"2026-11-05 21:40:48.000000000","uploader":{"_account_id":34520,"name":"Sergiy Markin","email":"smarkin@mirantis.com","username":"sm515x"},"ref":"refs/changes/56/1014356/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/56/1014356/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/56/1014356/4"}}},"commit":{"parents":[{"commit":"9a6df85524ca1b1642ee38c2e30a2ca1dcee448b","subject":"refactor: Rename configmap-etc to secret-etc where it renders a Secret","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/9a6df85524ca1b1642ee38c2e30a2ca1dcee448b"}]}],"author":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 16:40:33.000000000","tz":0},"committer":{"name":"Sergiy Markin","email":"smarkin@mirantis.com","date":"2026-11-05 21:40:28.000000000","tz":0},"subject":"fix(mariadb): Ride out Kubernetes API failures and stuck server pods","message":"fix(mariadb): Ride out Kubernetes API failures and stuck server pods\n\nmariadb-controller has no error handling around its loop, so a single\nfailed API call, such as a 504 from the apiserver while etcd is slow,\nends the process. Once the API is back the pod can still sit in\nCrashLoopBackOff for up to five minutes, with nothing to move the\nmaster service off a failed primary. Log the error and retry on the\nnext check, as mariadb-db-controller already does.\n\nstart.py makes its Kubernetes API calls without a timeout, so a call\non a connection the apiserver no longer answers waits until TCP gives\nup. On SIGTERM it then joins the threads making those calls, and the\npod stays in Terminating for the whole grace period, ten minutes by\ndefault, after mysqld has stopped cleanly. Give every call a timeout,\nwake the threads as soon as the pod stops, and do not join them.\n\nAfter a full restart the servers also wait until every one of them\nhas reported its position in the state configmap within the last 50\nseconds. A server pod that stays in Terminating after its container\nhas exited, for example while its RBD volume is torn down, never\nreports again, so the whole cluster stays down. Its last report is no\nhelp either: the monitor stops before mysqld, so it is the -1 of a\nrunning server. On SIGTERM publish the position of the clean shutdown,\nand accept an old report with a seqno of 0 or more as final: seqno is\n-1 while mysqld runs, so such a server has stopped at that position\nand cannot move past it before it starts and reports again. A server\nthat did not stop cleanly still holds the recovery back.\n\nstart.py also logs \"Declaring myself current cluster leader\" every\nhalf minute from whichever pod holds the lease that picks who\nbootstraps a new cluster. That pod need not be the primary that\nmariadb-controller points the master service at, so the message reads\nas a split brain. Call it the cluster bootstrap lease.\n\nTake the current UTC time from datetime.now(timezone.utc) rather than\nthe deprecated datetime.utcnow(), which Python 3.12 warns about in the\nmariadb-server log.\n\nChangelog: fixed: mariadb-controller recovers within one check after\n  a Kubernetes API failure instead of waiting out a CrashLoopBackOff\nChangelog: fixed: mariadb-server exits as soon as mysqld stops on\n  SIGTERM, instead of waiting out the grace period when the\n  Kubernetes API does not respond\nChangelog: fixed: mariadb recovers from a full restart while a server\n  pod that stopped cleanly is still in Terminating, instead of\n  waiting for that pod to come back\nChangelog: fixed: mariadb-server logs the bootstrap lease as such\n  rather than as the cluster leader, which is not the primary\nChange-Id: Ifec929bb3d1d88424e2dec50f91c4c1ce962c541\nSigned-off-by: Sergiy Markin \u003csmarkin@mirantis.com\u003e\nAssisted-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/c68d13e7676b1bf6f8eaec71ee202f4a66ccfc05"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/c68d13e7676b1bf6f8eaec71ee202f4a66ccfc05"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"CLOSED","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY","applied_by":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"}},{"label":"Workflow","status":"MAY","applied_by":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"}}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"SATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Verified\u003dMAX"],"failing_atoms":["label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"SATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Code-Review\u003dMAX"],"failing_atoms":["label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"SATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Workflow\u003dMAX"],"failing_atoms":["label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}
