)]}'
{".zuul.yaml":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"85ee84b0e07760d75fdfc32e2e6d193417a4d086","unresolved":true,"context_lines":[{"line_number":381,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":382,"context_line":"    required-projects:"},{"line_number":383,"context_line":"        - novnc/novnc"},{"line_number":384,"context_line":"        - openstack/oslo.service"},{"line_number":385,"context_line":"    nodeset: openstack-two-node-noble"},{"line_number":386,"context_line":"    vars:"},{"line_number":387,"context_line":"      # We use the \"all\" environment for tempest_test_regex and"}],"source_content_type":"text/x-yaml","patch_set":35,"id":"4f48449f_07c7b320","line":384,"updated":"2025-07-30 00:25:06.000000000","message":"this might not be needed if oslo has done a release with the requried functionalty now.\nwith that said ti will allow use to continue testign as they make improvment so in general i think this makes sense to pull in direct form source.","commit_id":"1467aacca2dc34a462cecac10e6fd9c29b04bd13"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"4ad763806463cc4ccf4c7eb035b1eb768978cf10","unresolved":false,"context_lines":[{"line_number":381,"context_line":"    pre-run: playbooks/nova-next/pre.yaml"},{"line_number":382,"context_line":"    required-projects:"},{"line_number":383,"context_line":"        - novnc/novnc"},{"line_number":384,"context_line":"        - openstack/oslo.service"},{"line_number":385,"context_line":"    nodeset: openstack-two-node-noble"},{"line_number":386,"context_line":"    vars:"},{"line_number":387,"context_line":"      # We use the \"all\" environment for tempest_test_regex and"}],"source_content_type":"text/x-yaml","patch_set":35,"id":"fec78e16_6683415e","line":384,"in_reply_to":"4f48449f_07c7b320","updated":"2025-07-30 19:29:05.000000000","message":"Correct we don\u0027t need it any more. I dropped it from here and I will add back if we need further testing on unreleased testing.","commit_id":"1467aacca2dc34a462cecac10e6fd9c29b04bd13"}],"/COMMIT_MSG":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"df6e9d3db5636bf9ed8019ca9e2ea28190be45f3","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Run nova-next with n-sch in threading mode"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Depends-On: https://review.opendev.org/c/openstack/devstack/+/948436"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Change-Id: I36c68740fae3e3a9bd3286a1b66d86fd3341aff5"},{"line_number":12,"context_line":"Signed-off-by: Balazs Gibizer \u003cgibi@redhat.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":37,"id":"4febd333_ffd14a6b","line":9,"updated":"2025-07-30 19:47:04.000000000","message":"this si now merged which is good timing.","commit_id":"12a70387bbe67ccd1717f2272647bbc4135623d3"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"23e8940351253719d23ad30498aa5c7ebf04074d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"dd3e2572_a90c5f5a","updated":"2025-05-06 08:31:24.000000000","message":"recheck rerun the test","commit_id":"9d63f436d394150708ad4ac9565d080bfad53f78"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b6818de9b9bfb7f0d62df3430738b80aa6974d76","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"9ff461df_ec6381b2","updated":"2025-05-07 12:04:00.000000000","message":"Couple of things about the last nova-next run:\n* Log is now visible at startup:\n```May 06 17:18:05.226443 np0040666732 nova-scheduler[84526]: Service is starting with native threading. This is currently experimental. Do not use it in production.```\nhttps://zuul.opendev.org/t/openstack/build/6b0a4e3a7b0845f3a48cc222efbaea23/log/controller/logs/screen-n-sch.txt#2\nIt is very early logging hence the log formatting is not applied yet.\n\n* ThreadPoolExecutors works OK in the worker processes\nworker1:\n```\nMay 06 18:48:22.147251 np0040666732 nova-scheduler[85618]: INFO nova.utils [None req-6488536f-19a0-4e8b-b334-2e004c7f6b06 admin admin] Stats of Process-2.scatter-gather executor: \u003cExecutorStatistics object at 0x7cec540a2440 (failures\u003d0, executed\u003d140, runtime\u003d1.71, cancelled\u003d0)\u003e\nMay 06 18:48:22.147456 np0040666732 nova-scheduler[85618]: INFO nova.utils [None req-6488536f-19a0-4e8b-b334-2e004c7f6b06 admin admin] State of Process-2.scatter-gather ThreadPoolExecutor: max_workers: 5, workers: 5, idle workers: 5\n```\nworker2:\n```\nMay 06 18:47:57.793854 np0040666732 nova-scheduler[85612]: INFO nova.utils [None req-1fe46eea-d0a4-4ba6-9093-d2c26f0f8daf admin admin] Stats of Process-1.scatter-gather executor: \u003cExecutorStatistics object at 0x7cec54366040 (failures\u003d0, executed\u003d143, runtime\u003d1.76, cancelled\u003d0)\u003e\nMay 06 18:47:57.794060 np0040666732 nova-scheduler[85612]: INFO nova.utils [None req-1fe46eea-d0a4-4ba6-9093-d2c26f0f8daf admin admin] State of Process-1.scatter-gather ThreadPoolExecutor: max_workers: 5, workers: 5, idle workers: 5\n```\n\n* We have the executor initialized also in the main process due to the pre-fork serice version check uses scatter-gather:\n```\nMay 06 17:18:07.568977 np0040666732 nova-scheduler[84526]: INFO nova.utils [None req-27b8ad7b-192d-4532-91a8-bd865c1b89b7 None None] Stats of MainProcess.scatter-gather executor: \u003cExecutorStatistics object at 0x7cec576c2180 (failures\u003d0, executed\u003d0, runtime\u003d0.00, cancelled\u003d0)\u003e\nMay 06 17:18:07.568977 np0040666732 nova-scheduler[84526]: INFO nova.utils [None req-27b8ad7b-192d-4532-91a8-bd865c1b89b7 None None] State of MainProcess.scatter-gather ThreadPoolExecutor: max_workers: 5, workers: 0, idle workers: 0\n```\n\n* with the reduced pool size the nova-scheduler\u0027s RSS is back to 85MB from 122MB, (eventlet based scheduler uses 56MB)\n```\nMay 06 17:40:43.944943 np0040666732 memory_tracker.sh[118522]:      85612    1.4          118156      84526   00:00:04       62 do_select                 nova-scheduler: ServiceWrapper worker(0)\n4464\tMay 06 17:40:43.944943 np0040666732 memory_tracker.sh[118522]:      79370    1.4          118056      78979   00:01:44        1 ep_poll                   neutron-server: rpc worker (/opt/stack/data/venv/bin/python3.12 /opt/stack/data/venv/bin/neutron-server --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini)\n4465\tMay 06 17:40:43.944943 np0040666732 memory_tracker.sh[118522]:      85618    1.4          118028      84526   00:00:04       62 do_select                 nova-scheduler: ServiceWrapper worker(1)\n```","commit_id":"f783a3acc289b804a0d7b945f813558688267015"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"855e8c2a77bd825a8e026f545e6e8500a0f39f91","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"f977cedd_f98c4c9c","updated":"2025-05-20 11:56:40.000000000","message":"nova-next is green and n-sch is happy in threading mode:\nworker1\n```\nMay 19 15:51:31.851906 np0040808937 nova-scheduler[83424]: INFO nova.utils [None req-85f39ec9-4520-489b-94ed-f27d1dbc826a admin admin] Stats of Process-1.cell_worker executor: \u003cExecutorStatistics object at 0x70ab5c176700 (failures\u003d0, executed\u003d140, runtime\u003d1.95, cancelled\u003d0)\u003e\n25989\tMay 19 15:51:31.852125 np0040808937 nova-scheduler[83424]: INFO nova.utils [None req-85f39ec9-4520-489b-94ed-f27d1dbc826a admin admin] State of Process-1.cell_worker ThreadPoolExecutor: max_workers: 5, workers: 5, idle workers: 5, queued work: 0\n```\nworker2\n```\nMay 19 15:51:56.441666 np0040808937 nova-scheduler[83428]: INFO nova.utils [None req-e42b85fd-db78-41c0-8740-32a0f1f2d382 admin admin] Stats of Process-2.cell_worker executor: \u003cExecutorStatistics object at 0x70ab544ddd00 (failures\u003d0, executed\u003d143, runtime\u003d2.00, cancelled\u003d0)\u003e\nMay 19 15:51:56.441876 np0040808937 nova-scheduler[83428]: INFO nova.utils [None req-e42b85fd-db78-41c0-8740-32a0f1f2d382 admin admin] State of Process-2.cell_worker ThreadPoolExecutor: max_workers: 5, workers: 5, idle workers: 5, queued work: 0","commit_id":"0b475df2d3dc5d2c1b2da6f379fd3d8ae52723e5"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"bff973ec95a35a6c9b1c163d5bd3a4ca2d56dec5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"3ef2a1e1_fb63c0c9","updated":"2025-05-20 11:53:37.000000000","message":"recheck grenade-skip-level-always timed out while dumping the databases. Seen twice now:\n\n```\n2025-05-19 14:44:28.838839 | controller | 2025-05-19 14:44:28.838 | Dumping new databases\n2025-05-19 16:56:18.302462 | RUN END RESULT_TIMED_OUT: [untrusted : opendev.org/openstack/grenade/playbooks/grenade.yaml@master]\n```\n\nhttps://zuul.opendev.org/t/openstack/build/18677894d0cd403194ebbd27cebd416d/log/job-output.txt#21769-21770\n\n\n---\n\nrecheck nova-lvm tempest run timed out. It has a bunch of timeout related test failures from various services (nova, neutron) so it is probably resource exhaustion in the runner. I don\u0027t see neutron api logs collected. Am I blind?","commit_id":"0b475df2d3dc5d2c1b2da6f379fd3d8ae52723e5"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"6daedb77f032cb6fd75df49d54a7874a476680dd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":18,"id":"09c1b5f5_077c3c67","updated":"2025-05-26 14:59:22.000000000","message":"recheck devstack dependency has been updated","commit_id":"973cab5de67578c1ddd04d4e63e482f46df783af"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"7cf1a02cd6228761423de3c8458f84e6f269e00a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":21,"id":"75489574_f1bc0bf8","updated":"2025-06-03 15:42:25.000000000","message":"recheck kernel panic in the guest in the grenade job","commit_id":"0e66ff3d2aa0f611006f90757b65528d787a3f54"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"48103c47d652064566c882fb3fa608163a95691c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":25,"id":"700d2622_a515161f","updated":"2025-06-16 12:31:35.000000000","message":"recheck grenade-skip-level-always timed out dumping database","commit_id":"72f06c79cd1e04cb84c676db1f1f93f9d32e6a3f"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"57c9cf7f98ca70f4f4978e783848a32540df4bb4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":35,"id":"6a1f3a6a_e980f3e1","updated":"2025-07-28 14:20:02.000000000","message":"let\u0027s give it a shot, sooner than later.","commit_id":"1467aacca2dc34a462cecac10e6fd9c29b04bd13"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"fa230fb3d55eeb08ef36be82c9a28fe0f81bb363","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":35,"id":"bf240797_6f790aa4","updated":"2025-07-28 12:38:21.000000000","message":"recheck","commit_id":"1467aacca2dc34a462cecac10e6fd9c29b04bd13"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"10cca4a4ee273c0875a7e9b404b2743582673097","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":35,"id":"072fce1e_432b2073","updated":"2025-07-29 08:11:52.000000000","message":"recheck ssh to the guest failed","commit_id":"1467aacca2dc34a462cecac10e6fd9c29b04bd13"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e7b7bb22d54b9cf0b699510f97cbbaf33ab1fc50","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":37,"id":"58a6a39c_8ec2a55c","updated":"2025-07-31 07:08:29.000000000","message":"recheck sshing to the guest timed out","commit_id":"12a70387bbe67ccd1717f2272647bbc4135623d3"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"8784e9618e016c4a6e0308e40567b2ba7316b06b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"a1796a04_f3728953","updated":"2025-08-02 18:38:35.000000000","message":"recheck\n```\n+ functions-common:apt_get:1214            :   sudo DEBIAN_FRONTEND\u003dnoninteractive http_proxy\u003d https_proxy\u003d no_proxy\u003d apt-get --option Dpkg::Options::\u003d--force-confold --assume-yes install ksmtuned\nReading package lists...\nBuilding dependency tree...\nReading state information...\nE: Unable to locate package ksmtuned\n~~~","commit_id":"ccd03cb710d2815a445aa3f75e2f780233bdea2d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"91094e78dfaa7cdb92aa33f83302b49cfea6d3f0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"81dcad9b_bb7c1f38","updated":"2025-08-04 14:23:29.000000000","message":"recheck guest kernel crash, obviously not related because not even the affected job","commit_id":"ccd03cb710d2815a445aa3f75e2f780233bdea2d"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"3af2602026dba4e466219baa974201de16a71058","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"8141eeef_f278ae61","updated":"2025-08-01 12:41:59.000000000","message":"recheck guest kernel panic","commit_id":"ccd03cb710d2815a445aa3f75e2f780233bdea2d"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"782eff2fa452a29ee3d581593d12ed4b7cd738fc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"208220f7_2a93e86a","updated":"2025-08-02 07:52:36.000000000","message":"recheck guest kernel panic\n```\n[    6.793404] random: crng init done\ncurrently loaded modules: 8021q 8139cp 8390 9pnet 9pnet_virtio ahci cec dca drm drm_kms_helper e1000 e1000e failover fb_sys_fops garp hid hid_generic i2c_algo_bit igb igbvf ip6_udp_tunnel ip_tables isofs libahci libcrc32c llc mii mrp ne2k_pci net_failover nls_ascii nls_iso8859_1 nls_utf8 pcnet32 qemu_fw_cfg rc_core sctp stp syscopyarea sysfillrect sysimgblt udp_tunnel usbhid virtio_blk virtio_dma_buf virtio_gpu virtio_input virtio_net virtio_rng virtio_scsi virtiofs x_tables \ninfo: initramfs loading root from /dev/vda1\n/sbin/init: can\u0027t load library \u0027libtirpc.so.3\u0027\n[    8.160178] Kernel panic - not syncing: Attempted to kill init! exitcode\u003d0x00001000\n[    8.160870] CPU: 0 PID: 1 Comm: init Not tainted 5.15.0-117-generic #127-Ubuntu\n[    8.161703] Hardware name: OpenStack Foundation OpenStack Nova, BIOS 1.16.3-debian-1.16.3-2 04/01/2014\n[    8.162250] Call Trace:\n[    8.162844]  \u003cTASK\u003e\n[    8.163104]  show_stack+0x52/0x5c\n[    8.163626]  dump_stack_lvl+0x4a/0x63\n[    8.163820]  dump_stack+0x10/0x16\n[    8.164026]  panic+0x15c/0x33b\n[    8.164238]  do_exit.cold+0x15/0xa0\n[    8.164433]  __x64_sys_exit+0x1b/0x20\n[    8.164621]  x64_sys_call+0x1f30/0x1fa0\n[    8.164843]  do_syscall_64+0x56/0xb0\n[    8.165188]  ? __vm_munmap+0x96/0x130\n[    8.165447]  ? exit_to_user_mode_prepare+0x37/0xb0\n[    8.165667]  ? syscall_exit_to_user_mode+0x2c/0x50\n[    8.165878]  ? x64_sys_call+0x1e07/0x1fa0\n[    8.166082]  ? do_syscall_64+0x63/0xb0\n[    8.166268]  ? new_sync_write+0x114/0x1a0\n[    8.166493]  ? vfs_write+0x1d5/0x270\n[    8.166675]  ? ksys_write+0x67/0xf0\n[    8.166849]  ? exit_to_user_mode_prepare+0x37/0xb0\n[    8.167057]  ? syscall_exit_to_user_mode+0x2c/0x50\n[    8.167280]  ? x64_sys_call+0x47c/0x1fa0\n[    8.170779]  ? do_syscall_64+0x63/0xb0\n[    8.172940]  ? syscall_exit_to_user_mode+0x2c/0x50\n[    8.174957]  ? x64_sys_call+0x47c/0x1fa0\n[    8.176840]  ? do_syscall_64+0x63/0xb0\n[    8.178631]  ? handle_mm_fault+0xd8/0x2c0\n[    8.180499]  ? do_user_addr_fault+0x1e7/0x670\n[    8.182353]  ? exit_to_user_mode_loop+0x7e/0x160\n[    8.184085]  ? exit_to_user_mode_prepare+0x37/0xb0\n[    8.185876]  ? irqentry_exit_to_user_mode+0xe/0x20\n[    8.187576]  ? irqentry_exit+0x1d/0x30\n[    8.189341]  ? exc_page_fault+0x89/0x170\n[    8.191098]  entry_SYSCALL_64_after_hwframe+0x6c/0xd6\n[    8.193246] RIP: 0033:0x7f266731e55e\n[    8.195291] Code: 05 d7 2a 00 00 4c 89 f9 bf 02 00 00 00 48 8d 35 fb 0d 00 00 48 8b 10 31 c0 e8 50 d2 ff ff bf 10 00 00 00 b8 3c 00 00 00 0f 05 \u003c48\u003e 8d 15 f3 2a 00 00 f7 d8 89 02 48 83 ec 20 49 8b 8c 24 b8 00 00\n[    8.199257] RSP: 002b:00007ffde0632a10 EFLAGS: 00000207 ORIG_RAX: 000000000000003c\n[    8.201273] RAX: ffffffffffffffda RBX: 00007ffde0633cc0 RCX: 00007f266731e55e\n[    8.203254] RDX: 0000000000000002 RSI: 0000000000001000 RDI: 0000000000000010\n[    8.205192] RBP: 00007ffde0633ca0 R08: 00007f2667317000 R09: 00007f266731701a\n[    8.207144] R10: 0000000000000001 R11: 0000000000000207 R12: 00007f2667318040\n[    8.208985] R13: 00000000004bae50 R14: 0000000000000000 R15: 0000000000403d66\n[    8.210935]  \u003c/TASK\u003e\n[    8.216386] Kernel Offset: 0x15a00000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)\n[    8.218706] ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode\u003d0x00001000 ]---\n```","commit_id":"ccd03cb710d2815a445aa3f75e2f780233bdea2d"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"f2cbe9b17c784772f22ffecca739d0307e323a20","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"64a43366_96c64174","updated":"2025-08-01 17:34:26.000000000","message":"recheck https://review.opendev.org/c/openstack/openstacksdk/+/956369 landed unblocking the gate","commit_id":"ccd03cb710d2815a445aa3f75e2f780233bdea2d"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"187774eaed2c9fae1427e8e907bbf3cfb4cb999b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"928a6ce0_d9ab5da1","updated":"2025-08-01 15:31:07.000000000","message":"sdk job fail depends on https://review.opendev.org/c/openstack/openstacksdk/+/956369 which is still in the gate","commit_id":"ccd03cb710d2815a445aa3f75e2f780233bdea2d"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a8744ec0b5ecbea4d7123997e1b85cd8f3e3632e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":39,"id":"ef65d396_240378e0","updated":"2025-08-14 11:32:50.000000000","message":"readding +W","commit_id":"3e33b5809226cda067871d45572bfe6bbb61dbd2"}]}