)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2f198badefa0f4ce838291365198cdbbb5691619","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"73499345_f992b654","updated":"2026-06-02 12:11:58.000000000","message":"I\u0027m OK to try this","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9dad6721a027abe6251f49d52e27388b4049609e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"33563e6c_db3a954b","updated":"2026-06-03 17:54:02.000000000","message":"on one hand none of the recent failre appare to be realted to the image at all\n\ni has been a mix of slow nodes where the vm boot to reach the promt took longer then tempst waited or other failure in other services like cinder that are again unrealted to the iamge.\n\nos i do wonder what fraction of show up as common recheck reasons","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"edcf66978ce42c2f4c05a21d0ecbaa1b535c710c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"ae949377_584664fb","updated":"2026-06-03 21:12:51.000000000","message":"recheck  vtpm jobs landed on an amd host that was detected as incompatable with the request cpu.\n\nlol every possibe ci job issue \n```\nJun 03 18:07:52.748136 npa5a9814153f44 nova-compute[80813]: DEBUG nova.virt.libvirt.driver [None req-2db7c913-1c53-4211-939c-fd89de01b4fa None None] cpu compare xml: \u003ccpu match\u003d\"exact\"\u003e\nJun 03 18:07:52.748136 npa5a9814153f44 nova-compute[80813]:   \u003cmodel\u003eNehalem\u003c/model\u003e\nJun 03 18:07:52.748136 npa5a9814153f44 nova-compute[80813]:   \u003cfeature name\u003d\"mmx\" policy\u003d\"disable\"/\u003e\nJun 03 18:07:52.748136 npa5a9814153f44 nova-compute[80813]:   \u003cfeature name\u003d\"ssse3\" policy\u003d\"require\"/\u003e\nJun 03 18:07:52.748136 npa5a9814153f44 nova-compute[80813]:   \u003cfeature name\u003d\"vme\" policy\u003d\"require\"/\u003e\nJun 03 18:07:52.748136 npa5a9814153f44 nova-compute[80813]: \u003c/cpu\u003e\nJun 03 18:07:52.748136 npa5a9814153f44 nova-compute[80813]:  {{(pid\u003d80813) _compare_cpu /opt/stack/nova/nova/virt/libvirt/driver.py:11097}}\nJun 03 18:07:52.748962 npa5a9814153f44 nova-compute[80813]: ERROR nova.virt.libvirt.driver [None req-2db7c913-1c53-4211-939c-fd89de01b4fa None None] CPU doesn\u0027t have compatibility.\nJun 03 18:07:52.748962 npa5a9814153f44 nova-compute[80813]: 0\nJun 03 18:07:52.748962 npa5a9814153f44 nova-compute[80813]: Refer to http://libvirt.org/html/libvirt-libvirt-host.html#virCPUCompareResult\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: CRITICAL nova [None req-2db7c913-1c53-4211-939c-fd89de01b4fa None None] Unhandled error: nova.exception.InvalidCPUInfo: Configured CPU model: Nehalem and CPU Flags [\u0027vme\u0027, \u0027+ssse3\u0027, \u0027-mmx\u0027] ar not compatible with host CPU. Please correct your config and try again. Unacceptable CPU info: CPU doesn\u0027t have compatibility.\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: 0\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: Refer to http://libvirt.org/html/libvirt-libvirt-host.html#virCPUCompareResult\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova Traceback (most recent call last):\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/virt/libvirt/driver.py\", line 1139, in _check_cpu_compatibility\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova     self._compare_cpu(cpu, self._get_cpu_info(), None)\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/virt/libvirt/driver.py\", line 11113, in _compare_cpu\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova     raise exception.InvalidCPUInfo(reason\u003dm % {\u0027ret\u0027: ret, \u0027u\u0027: u})\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova nova.exception.InvalidCPUInfo: Unacceptable CPU info: CPU doesn\u0027t have compatibility.\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova \nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova 0\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova \nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova Refer to http://libvirt.org/html/libvirt-libvirt-host.html#virCPUCompareResult\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova \nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova During handling of the above exception, another exception occurred:\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova \nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova Traceback (most recent call last):\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/data/venv/bin/nova-compute\", line 6, in \u003cmodule\u003e\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova     sys.exit(main())\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova              ^^^^^^\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/cmd/compute.py\", line 67, in main\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova     service.serve(server, workers\u003d1, no_fork\u003dTrue)\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/service.py\", line 384, in serve\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova     _launcher \u003d service.launch(CONF, server, workers\u003dworkers,\nJun 03 18:07:52.757809 npa5a9814153f44 nova-compute[80813]: ERROR nova                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/data/venv/lib/python3.12/site-packages/oslo_service/backend/_threading/service.py\", line 500, in launch\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova     launcher.launch_service(service, workers\u003dworkers)\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/data/venv/lib/python3.12/site-packages/oslo_service/backend/_threading/service.py\", line 374, in launch_service\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova     self.service.start()\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/service.py\", line 165, in start\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova     self.manager.init_host(self.service_ref)\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/compute/manager.py\", line 1762, in init_host\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova     self.driver.init_host(host\u003dself.host)\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/virt/libvirt/driver.py\", line 906, in init_host\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova     self._check_cpu_compatibility()\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova   File \"/opt/stack/nova/nova/virt/libvirt/driver.py\", line 1147, in _check_cpu_compatibility\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova     raise exception.InvalidCPUInfo(msg)\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova nova.exception.InvalidCPUInfo: Configured CPU model: Nehalem and CPU Flags [\u0027vme\u0027, \u0027+ssse3\u0027, \u0027-mmx\u0027] ar not compatible with host CPU. Please correct your config and try again. Unacceptable CPU info: CPU doesn\u0027t have compatibility.\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova \nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova 0\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova \nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova Refer to http://libvirt.org/html/libvirt-libvirt-host.html#virCPUCompareResult\nJun 03 18:07:52.758199 npa5a9814153f44 nova-compute[80813]: ERROR nova \n```","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"409371eecd2b49bd48251c853cafb93fc6383122","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"3b3990a1_50c33328","updated":"2026-06-04 00:45:30.000000000","message":"recheck live migration with the manager role failed\n\njsonschema.exceptions.ValidationError: Additional properties are not allowed (\u0027project_id\u0027, \u0027user_id\u0027 were unexpected","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6bea107814c4f7539332c5020417874dfb965389","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"35b8a374_530b1d7e","updated":"2026-06-03 17:47:48.000000000","message":"recheck multi cells slow boot\n\n\u003d\u003d\u003d cirros: current\u003dd260603 uptime\u003d50.99 \u003d\u003d\u003d","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"a65d86bba60b34a0352a1fbd35be90e38260903d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"5fd27a79_c2bfc123","updated":"2026-06-03 01:55:17.000000000","message":"recheck new image","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"1236268d8c58a01eb2457926a24fb54a903d94e0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"786de2c5_576bb62f","updated":"2026-06-03 10:29:33.000000000","message":"recheck slow vm, cirrors took 11 second to load the initram and 58 second to boot to the login promt but tempst only waits 15 seconds. we cant fix slow rax nodes with a tweaked image","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"41efeaa487b6a3c075dfe66c71be1a8eef0f7edd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"7f754674_52c7855b","updated":"2026-06-02 20:02:08.000000000","message":"recheck so looking at the logs if the first boot was interupted it appares that we can then fail on ssh because the autorsize keys may not be populated.\n\ni am investingating making the first-boot intialista more robost to that also in cirros now but for now lets proceed with this to gater more data.","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"77d62e256124c8b64c2c292427ea7ecd0fc475f2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"461dbebe_457364c3","updated":"2026-06-03 00:32:09.000000000","message":"so the ssh failure shoudl be fixed by https://github.com/cirros-dev/cirros/pull/139\n\ni just rebuilt the test image and uploaded a new one to the same location with that fix included.\n\nthe current ci run needs to complete before i can retrigger it with the fix to retry injecting the ssh key if that is unterupted so we will see if that helps.\n\nso far the image seams to helping or at least making forward progress as its uncovering addtional failure that can happen when we have interupted boots like this which is good.\n\n\nthe other thing to note is we coudl work aroudn this specific issue by either makign test that reboot instance wait for SSHable like we did for volume detach\nor by setting the fallback ssh password to gocubsgo so that even if the ssh key pair si not properly injected into the iamge by cirros-init it will still be able to log in but that maskign teh problem to some degree.\n\n\nwaiting for sshabel it proably valid long term for tests like this that are doign reboot but if we can just make cirros more resiltent in genreal thats a good thing.","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"d8966193736aef7820d4e3bf3d1cb204a1f1d7cc","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"02b5ab25_7d1d99bf","updated":"2026-06-04 07:26:43.000000000","message":"the frequency with which we see CI issues on this change sure looks suspicious to me, other changes do not seem to be affected as much. so I think it would be better to go through some more iterations of test runs before actually merging this. maybe also do a second change to compare against the \"minimal\" image from the same build?","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"69d38a1dc3d1e342ba160faea44bd259c739e9d9","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"b4f55753_27cbf082","in_reply_to":"02b5ab25_7d1d99bf","updated":"2026-06-04 09:38:15.000000000","message":"we could ha have been goign through each failure and tryign to see if they were know issues or not\n\nthe last failure \n```\njsonschema.exceptions.ValidationError: Additional properties are not allowed (\u0027project_id\u0027, \u0027user_id\u0027 were unexpected\n```\n\nthey should be there in 2.80 \n\n\nhttps://github.com/openstack/nova/blob/master/nova/api/openstack/compute/schemas/migrations.py#L131-L137\nhttps://github.com/openstack/tempest/blob/master/tempest/lib/api_schema/response/compute/v2_80/migrations.py\nhttps://github.com/openstack/tempest/blob/master/tempest/lib/api_schema/response/compute/v2_80/servers.py\n\n\n\n```\n    Response - Headers: {\u0027date\u0027: \u0027Wed, 03 Jun 2026 22:25:24 GMT\u0027, \u0027server\u0027: \u0027Apache/2.4.58 (Ubuntu)\u0027, \u0027content-length\u0027: \u0027618\u0027, \u0027content-type\u0027: \u0027application/json\u0027, \u0027openstack-api-version\u0027: \u0027compute 2.80\u0027, \u0027x-openstack-nova-api-version\u0027: \u00272.80\u0027, \u0027vary\u0027: \u0027OpenStack-API-Version,X-OpenStack-Nova-API-Version\u0027, \u0027x-openstack-request-id\u0027: \u0027req-c1712a90-6edf-4bed-85d7-06b5398be0e7\u0027, \u0027x-compute-request-id\u0027: \u0027req-c1712a90-6edf-4bed-85d7-06b5398be0e7\u0027, \u0027connection\u0027: \u0027close\u0027, \u0027status\u0027: \u0027200\u0027, \u0027content-location\u0027: \u0027http://10.209.128.192/compute/v2.1/servers/857c70f6-014f-45f0-bbbf-f720f7011f6e/migrations\u0027}\n        Body: b\u0027{\"migrations\": [{\"created_at\": \"2026-06-03T22:25:25.000000\", \"dest_compute\": null, \"dest_host\": null, \"dest_node\": null, \"disk_processed_bytes\": null, \"disk_remaining_bytes\": null, \"disk_total_bytes\": null, \"id\": 2, \"memory_processed_bytes\": null, \"memory_remaining_bytes\": null, \"memory_total_bytes\": null, \"server_uuid\": \"857c70f6-014f-45f0-bbbf-f720f7011f6e\", \"source_compute\": null, \"source_node\": null, \"status\": \"queued\", \"updated_at\": \"2026-06-03T22:25:32.000000\", \"uuid\": \"eec8026a-77bd-4d0d-9ec1-4aa54080f26a\", \"user_id\": \"43abe2c3d2dc4d7a98eff7445cc2caf0\", \"project_id\": \"3d40924ff3134d22a99feb8949ca18c2\"}]}\u0027\n}}}\n```\n\nlooks like a real bug in the responce schema validation in tempest to me.\n\nthe odd part is that nova and tempst appear to have the correct checks but it obviously failed\nhttps://1891d9588d40ea5325f3-1988f1bc3d637497f7692396b58d77ce.ssl.cf5.rackcdn.com/openstack/099abcc112ac4e5c85e99755024e110e/testr_results.html\nhttps://github.com/openstack/tempest/blob/master/tempest/api/compute/admin/test_live_migration.py#L497\n\ni think this is a bug here \n\nhttps://github.com/openstack/tempest/blob/master/tempest/lib/api_schema/response/compute/v2_80/servers.py#L62\nintoduced by \nhttps://github.com/openstack/tempest/commit/97162f7eea17b6d71b5a273ca02004f16d8d59fd\nas its overrieding the updated version with the 2.79 version\nwhat make 0 sense to me is how this ever passes in that case.\n\nother then the slow boots which seam to be rackspace xen nodes\nthe ohter failures do seam ot be intermetint failures we see\n\nhttps://review.opendev.org/c/openstack/nova/+/991220 for example which merged yesterday took 4 rechecks due to similar failures\n\nthis has already provided a tone of data such as the fact we have not seen a singel kernel panic wiht this change so i think that by its self has confirmed your orginal theory.\n\nanyway im happy to rebuild both images again and do addtioan ltesting if we have new thoeroes or bug fixes.","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ac9a19abec96478363bf579d7501b9af950cfe4b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"3352036e_861bb24f","in_reply_to":"461dbebe_457364c3","updated":"2026-06-03 00:37:12.000000000","message":"the smokign gun in this case is this\n```\ninfo: initramfs loading root from /dev/vda1\ninfo: /etc/init.d/rc.sysinit: up at 4.22\ninfo: container: none\nStarting syslogd: OK\ncurrently loaded modules: 8021q 8139cp 8390 9pnet 9pnet_virtio ahci cec dca drm drm_kms_helper e1000 e1000e failover fb_sys_fops garp hid hid_generic i2c_algo_bit igb igbvf ip6_udp_tunnel ip_tables isofs libahci libcrc32c llc mii mrp ne2k_pci net_failover nls_ascii nls_iso8859_1 nls_utf8 pcnet32 qemu_fw_cfg rc_core sctp stp syscopyarea sysfillrect sysimgblt udp_tunnel usbhid virtio_blk virtio_dma_buf virtio_gpu virtio_input virtio_net virtio_rng virtio_scsi virtiofs x_tables \nInitializing random number generator: OK\nSaving random seed: OK\nStarting acpid: OK\ncirros-apply-local already run per instance\nStarting network: dhcpcd-9.4.1 starting\nDUID 00:04:37:96:d1:67:19:bc:4d:4d:84:28:14:1b:2e:3d:7e:1c\neth0: waiting for carrier\neth0: carrier acquired\neth0: IAID 3e:69:cf:88\neth0: adding address fe80::f816:3eff:fe69:cf88\neth0: soliciting a DHCP lease\neth0: offered 10.1.0.13 from 10.1.0.1\neth0: probing address 10.1.0.13/28\neth0: soliciting an IPv6 router\neth0: leased 10.1.0.13 for 43200 seconds\neth0: adding route to 10.1.0.0/28\neth0: adding host route to 169.254.169.254 via 10.1.0.2\neth0: adding default route via 10.1.0.1\nforked to background, child pid 292\nOK\nTop of dropbear init script\nStarting dropbear sshd: OK\nGROWROOT: CHANGED: partition\u003d1 start\u003d18432 old: size\u003d210911 end\u003d229343 new: size\u003d2078687,end\u003d2097119\n/dev/root resized successfully [took 0.16s]\nPrinting cirros user authorized keys\n\u003d\u003d\u003d system information \u003d\u003d\u003d\n```\n\nso we properly loaded the root partion \n\ninfo: initramfs loading root from /dev/vda1\n\nbut skiped runing cirros-apply-local for the per instance (first boot) target\n\n```cirros-apply-local already run per instance```\n\nand stgart the ssh server\n\n```Starting dropbear sshd: OK```\n\nbut when we preting the user authrized keys it was empty.\n\n```\nPrinting cirros user authorized keys\n\u003d\u003d\u003d system information \u003d\u003d\u003d\n```\n\nso because the marker was writen even fi the write were not sync the command failed before it was never retired which is what is fixed by\n\nhttps://github.com/cirros-dev/cirros/pull/139/changes/d1266a2d73b23fea2d30bb00ef21b645953039e1","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"53afdd5afef77ec15d9b0e7177ed4b6e0e980e21","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"16c0d7f3_021752bf","in_reply_to":"b4f55753_27cbf082","updated":"2026-06-04 16:05:19.000000000","message":"https://review.opendev.org/c/openstack/tempest/+/991556\n\nshould fix the schema issue","commit_id":"c60425cf93cd6c6e8c302751df853038487c39cc"}]}