)]}'
{"id":"openstack%2Fopenstack-helm~1005157","triplet_id":"openstack%2Fopenstack-helm~master~If982d5e9191502a345dbd9d0fbcfc12c07f2090f","project":"openstack/openstack-helm","branch":"master","hashtags":[],"change_id":"If982d5e9191502a345dbd9d0fbcfc12c07f2090f","subject":"[WIP] deploy: retry kubeadm join once more","status":"ABANDONED","created":"2026-09-11 01:09:36.000000000","updated":"2026-09-11 02:37:29.000000000","total_comment_count":0,"unresolved_comment_count":0,"work_in_progress":true,"has_review_started":false,"meta_rev_id":"c4afab730f867ceb8321752152f3587dbce01790","_number":1005157,"virtual_id_number":1005157,"owner":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"actions":{},"labels":{"Verified":{"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{},"pending_reviewers":{},"reviewer_updates":[],"messages":[{"id":"7620ade90aaec99a4a5cfd204619fdb7f9072f86","tag":"autogenerated:gerrit:newWipPatchSet","author":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"date":"2026-09-11 01:09:36.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"ffd90f89459120f9dff1a77054ea04bf37cbd947","tag":"autogenerated:gerrit:newWipPatchSet","author":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"date":"2026-09-11 02:29:29.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"c4afab730f867ceb8321752152f3587dbce01790","tag":"autogenerated:gerrit:abandon","author":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"date":"2026-09-11 02:37:29.000000000","message":"Abandoned\n\nSquashed into I2c42202b135915bbe714c210dbcfa505f770c92e (change 1005140).","accounts_in_message":[],"_revision_number":2}],"current_revision_number":2,"current_revision":"29c2dbb4143fc216614bfcbd2c83cfa2c97f4d4c","revisions":{"50ceb5d59a524558c822e83e91744ced3e63d851":{"kind":"REWORK","_number":1,"created":"2026-09-11 01:09:36.000000000","uploader":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"ref":"refs/changes/57/1005157/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/57/1005157/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/1"}}},"commit":{"parents":[{"commit":"dd5656a9c213151d93f3dfa3c8b5a4b344607fb3","subject":"Fix rabbit_quorum_queues typo to rabbit_quorum_queue","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/dd5656a9c213151d93f3dfa3c8b5a4b344607fb3"}]}],"author":{"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","date":"2026-09-11 01:09:21.000000000","tz":-300},"committer":{"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","date":"2026-09-11 01:09:35.000000000","tz":-300},"subject":"[WIP] deploy: retry kubeadm join once more","message":"[WIP] deploy: retry kubeadm join once more\n\nA node that cannot complete discovery currently kills the buildset. It\naccounts for 8 of the last 150 check failures across six different jobs,\nall of them before a single chart is installed:\n\n  error execution phase preflight: couldn\u0027t validate the identity of the\n  API Server: could not find a JWS signature in the cluster-info\n  ConfigMap for token ID \"0sv3z6\"\n\nThe existing mitigation does not cover this. On build 49b90fa7 the\ncontrol plane\u0027s own \"Wait for the bootstrap token signature to be\npublished\" check passed on its first try, node-2, node-3 and node-4\njoined with that token 0.3s after starting, and the collected\ncluster-info carries jws-kubeconfig-0sv3z6 at resourceVersion 419 -\nwritten right after the token Secret at 418 and never written again.\nnode-1 nevertheless polled for the full five minutes and its last\ncompleted read returned a cluster-info without that key. So this is not\nthe publish race the wait task closes: node-1\u0027s view of the cluster was\nsimply wrong, and kubeadm re-reads the same view every five seconds\nuntil the discovery timeout, so it can never recover on its own.\n\nRetry the join instead. Reset first so an attempt that got further than\ndiscovery does not leave state behind, and skip the work entirely once\nkubelet.conf exists so the task stays idempotent. This costs up to one\nmore discovery timeout on a job that was going to fail anyway.\n\nAlso print, on a failed attempt, the token IDs the cluster-info this\nnode can read is signed for. kubeadm\u0027s discovery client sets\nInsecureSkipTLSVerify, so an API server that is not ours answers without\ncomplaint - if the IDs that come back are not ours, the node reached\nsomebody else\u0027s cluster, which would explain everything above. Every job\nbuilds its overlay with the same VNI 42 on the same 10.248.0.0/24, so\nthat is worth being able to confirm or rule out. Collect ip neigh and\nbridge fdb with the other host logs for the same reason.\n\nThis reverts the reasoning of Id373f8f5c5fe2f07f6149008128983f9aeba0ab7\n(\"deploy: do not retry kubeadm join\"), which assumed kubeadm\u0027s own\ndiscovery retry made an outer retry pointless.\n\nSigned-off-by: Vladimir Kozhukalov \u003ckozhukalov@gmail.com\u003e\nChange-Id: If982d5e9191502a345dbd9d0fbcfc12c07f2090f\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/50ceb5d59a524558c822e83e91744ced3e63d851"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/50ceb5d59a524558c822e83e91744ced3e63d851"}]},"branch":"refs/heads/master"},"29c2dbb4143fc216614bfcbd2c83cfa2c97f4d4c":{"kind":"REWORK","_number":2,"created":"2026-09-11 02:29:29.000000000","uploader":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"ref":"refs/changes/57/1005157/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/57/1005157/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/57/1005157/2"}}},"commit":{"parents":[{"commit":"dd5656a9c213151d93f3dfa3c8b5a4b344607fb3","subject":"Fix rabbit_quorum_queues typo to rabbit_quorum_queue","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/dd5656a9c213151d93f3dfa3c8b5a4b344607fb3"}]}],"author":{"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","date":"2026-09-11 01:09:21.000000000","tz":-300},"committer":{"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","date":"2026-09-11 02:29:22.000000000","tz":-300},"subject":"[WIP] deploy: retry kubeadm join once more","message":"[WIP] deploy: retry kubeadm join once more\n\nA node that cannot complete discovery currently kills the buildset. It\naccounts for 8 of the last 150 check failures across six different jobs,\nall of them before a single chart is installed:\n\n  error execution phase preflight: couldn\u0027t validate the identity of the\n  API Server: could not find a JWS signature in the cluster-info\n  ConfigMap for token ID \"0sv3z6\"\n\nThe existing mitigation does not cover this. On build 49b90fa7 the\ncontrol plane\u0027s own \"Wait for the bootstrap token signature to be\npublished\" check passed on its first try, node-2, node-3 and node-4\njoined with that token 0.3s after starting, and the collected\ncluster-info carries jws-kubeconfig-0sv3z6 at resourceVersion 419 -\nwritten right after the token Secret at 418 and never written again.\nnode-1 nevertheless polled for the full five minutes and its last\ncompleted read returned a cluster-info without that key. So this is not\nthe publish race the wait task closes: node-1\u0027s view of the cluster was\nsimply wrong, and kubeadm re-reads the same view every five seconds\nuntil the discovery timeout, so it can never recover on its own.\n\nRetry the join instead. Reset first so an attempt that got further than\ndiscovery does not leave state behind, and skip the work entirely once\nkubelet.conf exists so the task stays idempotent. This costs up to one\nmore discovery timeout on a job that was going to fail anyway.\n\nAlso print, on a failed attempt, the token IDs the cluster-info this\nnode can read is signed for. kubeadm\u0027s discovery client sets\nInsecureSkipTLSVerify, so an API server that is not ours answers without\ncomplaint - if the IDs that come back are not ours, the node reached\nsomebody else\u0027s cluster, which would explain everything above. Every job\nbuilds its overlay with the same VNI 42 on the same 10.248.0.0/24, so\nthat is worth being able to confirm or rule out. Collect ip neigh and\nbridge fdb with the other host logs for the same reason.\n\nThis reverts the reasoning of Id373f8f5c5fe2f07f6149008128983f9aeba0ab7\n(\"deploy: do not retry kubeadm join\"), which assumed kubeadm\u0027s own\ndiscovery retry made an outer retry pointless.\n\nSigned-off-by: Vladimir Kozhukalov \u003ckozhukalov@gmail.com\u003e\nChange-Id: If982d5e9191502a345dbd9d0fbcfc12c07f2090f\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/29c2dbb4143fc216614bfcbd2c83cfa2c97f4d4c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/29c2dbb4143fc216614bfcbd2c83cfa2c97f4d4c"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}