)]}'
{"id":"openstack%2Fopenstack-helm~1001002","triplet_id":"openstack%2Fopenstack-helm~master~I2316c1dfd14d54ca49586adedba8fa77a9a5799e","project":"openstack/openstack-helm","branch":"master","hashtags":[],"change_id":"I2316c1dfd14d54ca49586adedba8fa77a9a5799e","subject":"[WIP] DNM: reproduce the intermittent kubeadm join failure","status":"ABANDONED","created":"2026-08-14 19:21:58.000000000","updated":"2026-08-18 13:55:05.000000000","total_comment_count":0,"unresolved_comment_count":0,"work_in_progress":true,"has_review_started":false,"meta_rev_id":"3dc9ba71caa97919c57f41430809b70345b40ad1","_number":1001002,"virtual_id_number":1001002,"owner":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:zuul:check","value":1,"date":"2026-08-14 19:51:30.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{"REVIEWER":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"reviewer_updates":[{"updated":"2026-08-14 19:51:30.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"f72e2d1231bb21d6e79fec2ed478b3901f78bff3","tag":"autogenerated:gerrit:newWipPatchSet","author":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"date":"2026-08-14 19:21:58.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"5c22dcf2e9ef4eacfc6cd6dc60f1f4ac72da1898","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-08-14 19:51:30.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/8057685398ab41bcbfa1663005fdc7b9\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/2976585daf5047a3905cfae6fa0d77c1 : SUCCESS in 2m 16s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/0ed0bb1fd6ca480699130d4f201e13ef : SUCCESS in 1m 59s\n- openstack-helm-deploy-env-debug-1 https://zuul.opendev.org/t/openstack/build/ab495ce640504f95ac49edf7d5e0d768 : SUCCESS in 10m 49s\n- openstack-helm-deploy-env-debug-2 https://zuul.opendev.org/t/openstack/build/0b418e6f5fcb4d86b8548550a9f7649c : SUCCESS in 11m 54s\n- openstack-helm-deploy-env-debug-3 https://zuul.opendev.org/t/openstack/build/fed8412b77da4bc99890a919b7dee375 : SUCCESS in 19m 05s\n- openstack-helm-deploy-env-debug-4 https://zuul.opendev.org/t/openstack/build/235f9e3dd3b443b3882a6d4e1943d25b : SUCCESS in 18m 59s\n- openstack-helm-deploy-env-debug-5 https://zuul.opendev.org/t/openstack/build/f1825b7b43eb4996b4f461a8e68d7556 : SUCCESS in 10m 50s","accounts_in_message":[],"_revision_number":1},{"id":"3dc9ba71caa97919c57f41430809b70345b40ad1","tag":"autogenerated:gerrit:abandon","author":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"date":"2026-08-18 13:55:05.000000000","message":"Abandoned","accounts_in_message":[],"_revision_number":1}],"current_revision_number":1,"current_revision":"d05d129e5b5d2c6c435bd5937ecea1642f276d31","revisions":{"d05d129e5b5d2c6c435bd5937ecea1642f276d31":{"kind":"REWORK","_number":1,"created":"2026-08-14 19:21:58.000000000","uploader":{"_account_id":3009,"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","username":"kozhukalov"},"ref":"refs/changes/02/1001002/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/openstack-helm","ref":"refs/changes/02/1001002/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/02/1001002/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/02/1001002/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/openstack-helm refs/changes/02/1001002/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/openstack-helm refs/changes/02/1001002/1"}}},"commit":{"parents":[{"commit":"716a34de3df7f582dd9d7c985437be0d01a6b6bb","subject":"deploy: wait for the bootstrap token signature before joining","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/716a34de3df7f582dd9d7c985437be0d01a6b6bb"}]}],"author":{"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","date":"2026-08-14 19:21:49.000000000","tz":-300},"committer":{"name":"Vladimir Kozhukalov","email":"kozhukalov@gmail.com","date":"2026-08-14 19:21:57.000000000","tz":-300},"subject":"[WIP] DNM: reproduce the intermittent kubeadm join failure","message":"[WIP] DNM: reproduce the intermittent kubeadm join failure\n\nWorkers intermittently fail to join with \"could not find a JWS\nsignature in the cluster-info ConfigMap for token ID\". Waiting for the\nsignature to be published, which merged as I3558527c, closed the race\nbetween kubeadm token create and the bootstrap-signer, but did not fix\nthis: in the run that prompted this change the wait succeeded, two of\nfour workers then joined in about a second, and the other two could\nnot find the signature for that same token across five attempts and\nthirty minutes.\n\nSo the object was signed and the token was valid, yet two nodes could\nnot see it. Either they read a different or stale copy, or they never\ncompleted the read and kubeadm reported the last error of its\ndiscovery loop, which would make the message misleading. The endpoint\nthey join is a VXLAN overlay address that deploy-env builds itself, so\na per-peer forwarding problem would produce exactly this split.\n\nFive identical jobs deploy only the test environment on the same\nnodeset, giving five independent chances to reproduce something that\nhappens perhaps once in a buildset. Nothing but deploy-env runs, so\neach is cheap and the failure is not buried under an OpenStack\ndeployment.\n\nplaybooks/debug-join-state.yaml runs as a post-run on every job, so it\nreports whether the join succeeded or not. It records which tokens the\ncontrol plane has signed, which nodes joined, the overlay and\nneighbour state on each host, whether each host can reach the endpoint\nat all, what each host sees when it reads cluster-info anonymously the\nway kubeadm does, and the fingerprint of the certificate served on\nthat address. A key set that differs from the control plane\u0027s means\nthe object really is inconsistent; a fingerprint that differs means\nthe node is reaching a different server; a read that hangs means the\nmessage was hiding a connectivity problem.\n\nDNM: the check pipeline is commented out entirely and replaced by the\nfive jobs. Restore that block, delete the DNM markers and delete\nzuul.d/debug-join.yaml before anything here merges: `grep -rn \u0027DNM(\u0027\nzuul.d/` must print nothing. The gate pipeline is untouched.\n\nSigned-off-by: Vladimir Kozhukalov \u003ckozhukalov@gmail.com\u003e\nChange-Id: I2316c1dfd14d54ca49586adedba8fa77a9a5799e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/d05d129e5b5d2c6c435bd5937ecea1642f276d31"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/openstack-helm/commit/d05d129e5b5d2c6c435bd5937ecea1642f276d31"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}