)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"84fb004f5e6a68915acdc88d19e9fa1aa0d9cb40","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"0865ff59_f22a5e60","updated":"2026-07-08 13:46:02.000000000","message":"Looks good, just waiting for CI","commit_id":"f07f732139e03adf7cd0de4251a1bb7004b2ba36"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"c27484087f168b8183c0d6a0de96e8b55392a081","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"64d5fdb1_79cd8daf","updated":"2026-07-07 17:30:12.000000000","message":"recheck ansible galaxy connection failure","commit_id":"f07f732139e03adf7cd0de4251a1bb7004b2ba36"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"637afd88515beccc719a440b5586cec0c3d91f7c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"beb4f7af_26dd1f00","updated":"2026-07-08 15:27:33.000000000","message":"recheck dependency changed","commit_id":"f07f732139e03adf7cd0de4251a1bb7004b2ba36"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"1af42ec2241e4da508b1fcffc8ae99363196fa8f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"5f5de7c3_1f3c4d72","updated":"2026-07-08 09:50:10.000000000","message":"recheck yum download error (on a different job, that passed the first time)","commit_id":"f07f732139e03adf7cd0de4251a1bb7004b2ba36"},{"author":{"_account_id":28048,"name":"Will Szumski","email":"will@stackhpc.com","username":"jovial"},"change_message_id":"f340335f43674a2fef2a1b4cc8b58c2c8d5c6594","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"8402d6c2_62d392ed","updated":"2026-07-09 14:42:15.000000000","message":"Very nice. I noticed ipmi inspection rules still get applied to the redfish node, but that is more the fact that they are configured on global basis and don\u0027t consider the driver.","commit_id":"e620d2273079937b7c499bb456a69bbbc01e11db"},{"author":{"_account_id":28048,"name":"Will Szumski","email":"will@stackhpc.com","username":"jovial"},"change_message_id":"c68c12b9ef828fc985588e36411b06e7638bf198","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"2192b767_a01dee73","updated":"2026-07-10 18:11:41.000000000","message":"Hit the same issue again","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"8802222efa114f083b713f905527b6589a567c27","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"3f575793_47566158","updated":"2026-07-09 16:19:06.000000000","message":"Many thanks Jed,\nI see it running: https://zuul.opendev.org/t/openstack/build/7dc6300d5cb644b597dc30af1ed80c9a/log/primary/kolla/tenks/red0-console.txt\n\nI don\u0027t see any harm in backporting.","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":35263,"name":"Matt Crees","email":"mattc@stackhpc.com","username":"mattcrees"},"change_message_id":"aa54d41fba26bccc6c64b44db06a6c204449033f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"eb929abd_4b5e152f","updated":"2026-07-09 15:10:54.000000000","message":"Thanks!","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":14200,"name":"Maksim Malchuk","email":"maksim.malchuk@gmail.com","username":"mmalchuk"},"change_message_id":"3e85021fa5c266b07006f4b0c26eb4df8eb47f55","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"7c21096b_ebd9e16b","updated":"2026-07-13 09:53:20.000000000","message":"its random","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":28048,"name":"Will Szumski","email":"will@stackhpc.com","username":"jovial"},"change_message_id":"81e2b37510528fe422dcfa3ebcb5f64771106858","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"e0c4358a_a373942c","updated":"2026-07-10 09:15:06.000000000","message":"recheck: Need to keep an eye on how reliable this is and improve if necessary: \"Node red0 failed to reach state manageable\"","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":28048,"name":"Will Szumski","email":"will@stackhpc.com","username":"jovial"},"change_message_id":"7d4d6501b22c8030b516a6e13374047712e196b0","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"ddfa895d_7958ecca","in_reply_to":"0a9fc410_f8791d34","updated":"2026-07-15 09:47:27.000000000","message":"Maybe we should add some retries to inspect? We can\u0027t really merge if its gunna make CI really unreliable as it will block us merging anything.","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"f71048a43e61e3a568e4a8d0eb99392ca7a20db5","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"0a9fc410_f8791d34","in_reply_to":"29dd6d8b_85f092dc","updated":"2026-07-15 09:39:10.000000000","message":"@will@stackhpc.com I\u0027ve just seen that you\u0027ve hit the same error on an ipmi node, without the redfish change: https://zuul.opendev.org/t/openstack/build/826d1c84eb5c43b18925f662fb9c9b77","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":28048,"name":"Will Szumski","email":"will@stackhpc.com","username":"jovial"},"change_message_id":"6291de185cb720b46965f6911f70a8a1bb0e0723","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"c8701487_b15f00cf","in_reply_to":"9f7fcfde_523518f4","updated":"2026-07-10 10:13:00.000000000","message":"Cool, it might just be that we have another machine so higher probability of hitting random failure. What I was thinking is that we might not want to backport this if it reduces CI reliability. If we run it in master for a while and there are no issues, I\u0027m all for it.","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":28048,"name":"Will Szumski","email":"will@stackhpc.com","username":"jovial"},"change_message_id":"c68c12b9ef828fc985588e36411b06e7638bf198","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"fdeb4a4e_6a3b8bc9","in_reply_to":"c8701487_b15f00cf","updated":"2026-07-10 18:11:41.000000000","message":"Hit the same issue again on the gate 🤔","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"6ef0d2e21ff0cd67096c2ed2e7f0033bf1d60904","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"e2e01e40_bd8961ca","in_reply_to":"d7a93813_17ebc60a","updated":"2026-07-17 09:53:44.000000000","message":"This seems much more stable running the redfish inspection first. I\u0027m going to call it resolved unless you have any objections.","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"9e7d09b2a4ec7c5621f96f7e4d3e4d1c00bba526","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"d7a93813_17ebc60a","in_reply_to":"ddfa895d_7958ecca","updated":"2026-07-15 14:54:07.000000000","message":"Maybe. I\u0027d like to do a little more testing. I\u0027m wondering if it\u0027s something to do with running the inspections back to back, and if adding a delay between them would help.","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"3b4e7337a9e47321fdba38bbda88eee404cf259f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"9f7fcfde_523518f4","in_reply_to":"e0c4358a_a373942c","updated":"2026-07-10 09:54:00.000000000","message":"There doesn\u0027t seem to be many errors related to it in the logs. Inspection time out from ironic, nothing obvious from libvirt or sushy. The inspection image is running, and the only error from the logs for that is \u0027clocksource: timekeeping watchdog on CPU0: acpi_pm retried 1 times before success\u0027","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"1ac2c2e1f69487a0dc562b904034403f5a9f57f5","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"29dd6d8b_85f092dc","in_reply_to":"fdeb4a4e_6a3b8bc9","updated":"2026-07-13 10:56:20.000000000","message":"I\u0027ve found an error might be relevant in the ironic API logs: \n2026-07-10 12:22:46.891 27 DEBUG ironic.api.method [None req-6c6cb314-0fb8-45e2-94c1-784644fb4739 - - - - - -] Client-side error: Schema error for data: [] should be non-empty format_exception /var/lib/kolla/venv/lib/python3.12/site-packages/ironic/api/method.py:124\n2026-07-10 12:22:46.892 27 INFO ironic.api [None req-6c6cb314-0fb8-45e2-94c1-784644fb4739 - - - - - -] 192.168.33.137 - POST /v1/continue_inspection - 400 (28.66ms)\n\nNow we just need to work out why the inspection is occasionally not returning any data\n\nAlso, I think its happening often enough that it will be a pain if we merge without fixing it","commit_id":"feb71c267432d42b9dcd2e16ffbe51dcd8724c8e"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"8f687beaca5a9c35fa402ca74f1edbafcdb7fda7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"09da70d0_61b6dea8","updated":"2026-07-13 16:20:01.000000000","message":"I see this for the failed Ubuntu job where we don\u0027y say anything on the console past iPXE:\n\n```\n2026-07-10 12:16:20.335 32 WARNING sushy.connector [None req-0e73a2aa-f364-4c5c-bfff-d5acaf6da391 ab808d0bf827446daaad95fc41878226 ffb8999cf2634882bea9913797497556 - - default default] Initial request with eTag failed: HTTP PATCH http://0.0.0.0:8000/redfish/v1/Systems/46e99de6-549c-4ef8-9d23-6a35595bdc71 returned code 500. Base.1.0.GeneralError: Error changing boot mode at libvirt URI \"qemu:///system\": no loader path specified and firmware auto selection disabled Extended information: [{\u0027@odata.type\u0027: \u0027/redfish/v1/$metadata#Message.1.0.0.Message\u0027, \u0027MessageId\u0027: \u0027Base.1.0.GeneralError\u0027}]: sushy.exceptions.ServerSideError: HTTP PATCH http://0.0.0.0:8000/redfish/v1/Systems/46e99de6-549c-4ef8-9d23-6a35595bdc71 returned code 500. Base.1.0.GeneralError: Error changing boot mode at libvirt URI \"qemu:///system\": no loader path specified and firmware auto selection disabled Extended information: [{\u0027@odata.type\u0027: \u0027/redfish/v1/$metadata#Message.1.0.0.Message\u0027, \u0027MessageId\u0027: \u0027Base.1.0.GeneralError\u0027}]\n2026-07-10 12:16:20.336 32 ERROR ironic.drivers.modules.redfish.management [None req-0e73a2aa-f364-4c5c-bfff-d5acaf6da391 ab808d0bf827446daaad95fc41878226 ffb8999cf2634882bea9913797497556 - - default default] Setting boot mode to bios failed for node 313eb96e-a571-44d4-b0db-58eed2a4ea37. Error: HTTP PATCH http://0.0.0.0:8000/redfish/v1/Systems/46e99de6-549c-4ef8-9d23-6a35595bdc71 returned code 500. Base.1.0.GeneralError: Error changing boot mode at libvirt URI \"qemu:///system\": no loader path specified and firmware auto selection disabled Extended information: [{\u0027@odata.type\u0027: \u0027/redfish/v1/$metadata#Message.1.0.0.Message\u0027, \u0027MessageId\u0027: \u0027Base.1.0.GeneralError\u0027}]: sushy.exceptions.ServerSideError: HTTP PATCH http://0.0.0.0:8000/redfish/v1/Systems/46e99de6-549c-4ef8-9d23-6a35595bdc71 returned code 500. Base.1.0.GeneralError: Error changing boot mode at libvirt URI \"qemu:///system\": no loader path specified and firmware auto selection disabled Extended information: [{\u0027@odata.type\u0027: \u0027/redfish/v1/$metadata#Message.1.0.0.Message\u0027, \u0027MessageId\u0027: \u0027Base.1.0.GeneralError\u0027}]\n2026-07-10 12:16:20.337 32 INFO ironic.drivers.modules.redfish.management [None req-0e73a2aa-f364-4c5c-bfff-d5acaf6da391 ab808d0bf827446daaad95fc41878226 ffb8999cf2634882bea9913797497556 - - default default] Attempt to set boot mode on node 313eb96e-a571-44d4-b0db-58eed2a4ea37 failed to set boot mode as the node does not appear to support overriding the boot mode. Possibly partial Redfish implementation?\n```","commit_id":"afc63496573830337938fdc9ea4052bd56e108cb"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"e5ef04a2b0d0bfb042d5565485f4dafd0e1a263a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"42c59dd1_20e58c0e","in_reply_to":"09da70d0_61b6dea8","updated":"2026-07-13 16:22:36.000000000","message":"s/say/see ^\n\nIt\u0027s odd how that is sporadic - just another data point.","commit_id":"afc63496573830337938fdc9ea4052bd56e108cb"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"713e27802cbdadfd8a1ef7c6bb88e4842a33f606","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"b8d44760_8aabf58d","in_reply_to":"42c59dd1_20e58c0e","updated":"2026-07-13 16:29:27.000000000","message":"That seems to happen on the other jobs without causing issues.","commit_id":"afc63496573830337938fdc9ea4052bd56e108cb"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"085931061ebec22a3cd5f9396c5f9a17092da68f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"452c8014_84dcb766","updated":"2026-07-15 15:33:22.000000000","message":"recheck prev recheck was ignored","commit_id":"6f75f468ef2bcb1a08dd682744f5a62b2bfa86d9"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"9e7d09b2a4ec7c5621f96f7e4d3e4d1c00bba526","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"22d27799_ac259313","updated":"2026-07-15 14:54:07.000000000","message":"recheck: CI fixed","commit_id":"6f75f468ef2bcb1a08dd682744f5a62b2bfa86d9"},{"author":{"_account_id":14200,"name":"Maksim Malchuk","email":"maksim.malchuk@gmail.com","username":"mmalchuk"},"change_message_id":"fa199ec3b97f8d906d913cc2d1fdaa3edfcff7d7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"6fae2712_b4d09f64","updated":"2026-07-22 20:33:00.000000000","message":"recheck POST_FAILURE","commit_id":"b8bb880f769025e60fa2a794b13ef58b38a940db"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"c73e9a1737a866211c2a3fc348547a7800204810","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"d892d0ea_8650c538","updated":"2026-07-17 07:46:04.000000000","message":"recheck ansible galaxy connection error. It seems much more stable inspecting the redfish node first.","commit_id":"b8bb880f769025e60fa2a794b13ef58b38a940db"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"2b4ca3dfc0639f30b6166d7a5770c2d969fe6603","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"05657b26_58192b2f","updated":"2026-07-16 15:07:37.000000000","message":"recheck checking reliability","commit_id":"b8bb880f769025e60fa2a794b13ef58b38a940db"},{"author":{"_account_id":39213,"name":"Jed Preist","display_name":"Jed Preist","email":"jed@stackhpc.com","username":"jed-preist"},"change_message_id":"8538badb8a76c4222772a1e9d2097197bc964471","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"8247ce1b_36433b5e","updated":"2026-07-16 12:28:10.000000000","message":"recheck hit different error, looking for inspection failure","commit_id":"b8bb880f769025e60fa2a794b13ef58b38a940db"}]}