)]}'
{".zuul.yaml":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"49deb4b6514e68e919f7ad87a8d0a555fa231afd","unresolved":true,"context_lines":[{"line_number":837,"context_line":"        - barbican-tempest-plugin-simple-crypto:"},{"line_number":838,"context_line":"            irrelevant-files: *nova-base-irrelevant-files"},{"line_number":839,"context_line":"            voting: false"},{"line_number":840,"context_line":"        - nova-tox-py312-threading"},{"line_number":841,"context_line":"    gate:"},{"line_number":842,"context_line":"      jobs:"},{"line_number":843,"context_line":"        - nova-live-migration"}],"source_content_type":"text/x-yaml","patch_set":8,"id":"24ac34dc_59905590","side":"PARENT","line":840,"updated":"2025-09-09 14:26:53.000000000","message":"Just temporary removed everything to save CI bandwidth while I\u0027m iterating on the new job","commit_id":"0dd7cb1fb0996c4298cc970dab9230f598d2d246"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"768477b705ed2c1d3b2ef5685eec82ac2b090620","unresolved":true,"context_lines":[{"line_number":821,"context_line":"        neutron-qos: false"},{"line_number":822,"context_line":"        neutron-segments: false"},{"line_number":823,"context_line":"        q-svc: false"},{"line_number":824,"context_line":"        neutron-api: false"},{"line_number":825,"context_line":"        # atop is used to measure proc cpu time"},{"line_number":826,"context_line":"        atop: true"},{"line_number":827,"context_line":"      devstack_localrc:"}],"source_content_type":"text/x-yaml","patch_set":18,"id":"c4a987d3_dbee85f2","line":824,"updated":"2025-09-12 13:29:01.000000000","message":"bah I cannot turn off neutron nova hard depends on it even if the VM is not using ports\n\n```\nSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi [None req-3afeb441-53e3-45aa-b6d2-7fe66e26797f c_rally_cddd1b32_PMytA74d c_rally_cddd1b32_hJ0GfwYx] Unexpected exception in API method: keystoneauth1.exceptions.catalog.EndpointNotFound: [\u0027internal\u0027, \u0027public\u0027] endpoint for network service in RegionOne region not found\n2990\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi Traceback (most recent call last):\n2991\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi   File \"/opt/stack/nova/nova/api/openstack/wsgi.py\", line 767, in wrapped\n2992\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi     return f(*args, **kwargs)\n2993\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi            ^^^^^^^^^^^^^^^^^^\n2994\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi   File \"/opt/stack/nova/nova/api/validation/__init__.py\", line 384, in wrapper\n2995\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi     return func(*args, **kwargs)\n2996\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi            ^^^^^^^^^^^^^^^^^^^^^\n2997\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi   File \"/opt/stack/nova/nova/api/openstack/compute/servers.py\", line 474, in show\n2998\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi     return self._view_builder.show(\n2999\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi            ^^^^^^^^^^^^^^^^^^^^^^^^\n3000\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi   File \"/opt/stack/nova/nova/api/openstack/compute/views/servers.py\", line 360, in show\n3001\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi     self._add_security_grps(request, [server[\"server\"]], [instance])\n3002\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi   File \"/opt/stack/nova/nova/api/openstack/compute/views/servers.py\", line 770, in _add_security_grps\n3003\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi     security_group_api.get_instances_security_groups_bindings(\n3004\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi   File \"/opt/stack/nova/nova/network/security_group_api.py\", line 528, in get_instances_security_groups_bindings\n3005\tSep 12 12:58:23.788805 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi     ports \u003d _get_ports_from_server_list(servers, neutron)\n3006\tSep 12 12:58:23.789978 np72c9fa2160774 devstack@n-api.service[53196]: ERROR nova.api.openstack.wsgi             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n```","commit_id":"bedafb3efad8ab4cdfa109e8c2a3457033ec0e31"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1217ca4f3ba3c910b6fac4d3363e69d62a1c8f51","unresolved":false,"context_lines":[{"line_number":821,"context_line":"        neutron-qos: false"},{"line_number":822,"context_line":"        neutron-segments: false"},{"line_number":823,"context_line":"        q-svc: false"},{"line_number":824,"context_line":"        neutron-api: false"},{"line_number":825,"context_line":"        # atop is used to measure proc cpu time"},{"line_number":826,"context_line":"        atop: true"},{"line_number":827,"context_line":"      devstack_localrc:"}],"source_content_type":"text/x-yaml","patch_set":18,"id":"d627d3ec_af46d929","line":824,"in_reply_to":"c4a987d3_dbee85f2","updated":"2025-09-28 14:19:13.000000000","message":"Done","commit_id":"bedafb3efad8ab4cdfa109e8c2a3457033ec0e31"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1217ca4f3ba3c910b6fac4d3363e69d62a1c8f51","unresolved":true,"context_lines":[{"line_number":806,"context_line":"        # NOTE(gibi): We canno disable neutron as nova depends on it even"},{"line_number":807,"context_line":"        # if the VM has no ports"},{"line_number":808,"context_line":"        # atop is used to measure proc cpu time"},{"line_number":809,"context_line":"        atop: true"},{"line_number":810,"context_line":"      devstack_localrc:"},{"line_number":811,"context_line":"        KEYSTONE_ADMIN_ENDPOINT: true"},{"line_number":812,"context_line":"        VIRT_DRIVER: fake"}],"source_content_type":"text/x-yaml","patch_set":19,"id":"e423f662_adf9add2","line":809,"updated":"2025-09-28 14:19:13.000000000","message":"cpu time is also logged by the memory_tacker so I will just pull RSS and TIME out of that log instead.","commit_id":"1e3732a7d92e15991077ee9ec1377d4446cfd6fa"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2efdce6760b6dc84e0abc9b6321af08b6d1ecac8","unresolved":false,"context_lines":[{"line_number":806,"context_line":"        # NOTE(gibi): We canno disable neutron as nova depends on it even"},{"line_number":807,"context_line":"        # if the VM has no ports"},{"line_number":808,"context_line":"        # atop is used to measure proc cpu time"},{"line_number":809,"context_line":"        atop: true"},{"line_number":810,"context_line":"      devstack_localrc:"},{"line_number":811,"context_line":"        KEYSTONE_ADMIN_ENDPOINT: true"},{"line_number":812,"context_line":"        VIRT_DRIVER: fake"}],"source_content_type":"text/x-yaml","patch_set":19,"id":"6c44fec9_07aef21b","line":809,"in_reply_to":"e423f662_adf9add2","updated":"2025-10-21 14:44:48.000000000","message":"Done","commit_id":"1e3732a7d92e15991077ee9ec1377d4446cfd6fa"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"4a123e350a6791fcf723d063ab81e79cc80f5993","unresolved":true,"context_lines":[{"line_number":819,"context_line":"              # Default is 64 and with native threading"},{"line_number":820,"context_line":"              # and high parallelism that pushes the conductor"},{"line_number":821,"context_line":"              # and scheduler peek memory up pretty much"},{"line_number":822,"context_line":"              executor_thread_pool_size: 32"},{"line_number":823,"context_line":"      rally_task: rally-jobs/task-nova-threading-scale.yaml"},{"line_number":824,"context_line":""},{"line_number":825,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":39,"id":"005576af_2b66a178","line":822,"updated":"2025-10-28 08:38:27.000000000","message":"It does not seem to affect the memory consumption. Strange lets limit it further","commit_id":"4ad7f16763a1aa411b1e95a62ae11f08b8f1d4aa"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"646b9933d5f9c5b5d2c12bb5a96fe6bcb54e0844","unresolved":false,"context_lines":[{"line_number":819,"context_line":"              # Default is 64 and with native threading"},{"line_number":820,"context_line":"              # and high parallelism that pushes the conductor"},{"line_number":821,"context_line":"              # and scheduler peek memory up pretty much"},{"line_number":822,"context_line":"              executor_thread_pool_size: 32"},{"line_number":823,"context_line":"      rally_task: rally-jobs/task-nova-threading-scale.yaml"},{"line_number":824,"context_line":""},{"line_number":825,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":39,"id":"1840ba23_8badc6df","line":822,"in_reply_to":"005576af_2b66a178","updated":"2025-10-30 13:57:13.000000000","message":"Done","commit_id":"4ad7f16763a1aa411b1e95a62ae11f08b8f1d4aa"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"c9222c2ae44fda41c9f19ac937e3837566961883","unresolved":true,"context_lines":[{"line_number":819,"context_line":"              # Default is 64 and with native threading"},{"line_number":820,"context_line":"              # and high parallelism that pushes the conductor"},{"line_number":821,"context_line":"              # and scheduler peek memory up pretty much"},{"line_number":822,"context_line":"              executor_thread_pool_size: 4"},{"line_number":823,"context_line":"      rally_task: rally-jobs/task-nova-threading-scale.yaml"},{"line_number":824,"context_line":""},{"line_number":825,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":40,"id":"fe4bd878_f71b25c3","line":822,"updated":"2025-10-30 08:55:56.000000000","message":"bah this is not transferred to the cell conf files :/","commit_id":"e8476cc2fddea2b1449e08c8f339953a6f0c048e"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ce43a7771d727fb2f76e22c8568d27b2fbf3d1ce","unresolved":false,"context_lines":[{"line_number":819,"context_line":"              # Default is 64 and with native threading"},{"line_number":820,"context_line":"              # and high parallelism that pushes the conductor"},{"line_number":821,"context_line":"              # and scheduler peek memory up pretty much"},{"line_number":822,"context_line":"              executor_thread_pool_size: 4"},{"line_number":823,"context_line":"      rally_task: rally-jobs/task-nova-threading-scale.yaml"},{"line_number":824,"context_line":""},{"line_number":825,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":40,"id":"46c61ba7_ff217568","line":822,"in_reply_to":"fe4bd878_f71b25c3","updated":"2025-10-30 13:56:59.000000000","message":"Done","commit_id":"e8476cc2fddea2b1449e08c8f339953a6f0c048e"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1ae7c46b48aac708166bb6bab789db0d1929e1b9","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"8ca0047f_8ac3a6b8","updated":"2025-09-08 19:08:55.000000000","message":"The basic rally setup works. The new job runs a simple rally scenario of booting multiple VMs on a single node devstack with 5 fake computes and measures the timing. Now we need extra steps to reconfigure the nova services to from eventlet to threading and re-run the same scenarios. Then we probably want this be multi cell to allow more test scenarios. \n\nhttps://b2793fc951b6cda052a8-c455553d95560f1580667e93cc59b7bd.ssl.cf2.rackcdn.com/openstack/af4b867ad2d149ff8d7722c69ce53f2c/results/report.html#/NovaServers.boot_and_delete_server","commit_id":"c7b895ba4fb2071ce62516651ece8b41c219325f"},{"author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"change_message_id":"ffc19458ae9ae64f66bea3613d8032189d574fb9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"4f5404f8_660ff122","updated":"2025-09-09 11:48:13.000000000","message":"Hey, do you mind if we tag this patch (and similar patches): `eventlet-removal-perfscale`?\n\nThis would allow us to know the current status of performance scaling tests, while also allowing us to integrate them with the `eventlet-removal` prefix.","commit_id":"59bab7c234ad0299d19a5e4e3a725cfeabe62f9b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"4522689f66af621ad9768479979e724e974cbd86","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"9f2ca270_2a400d7c","in_reply_to":"4f5404f8_660ff122","updated":"2025-09-11 16:20:27.000000000","message":"That works for me","commit_id":"59bab7c234ad0299d19a5e4e3a725cfeabe62f9b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e6a461d92172aea570e8f18405d47ee00b9c300a","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":16,"id":"7f4fc037_6940f711","updated":"2025-09-12 08:58:46.000000000","message":"We have the first comparable results. 200 VM boot/delete with 20 concurrency.\n\nrun with eventlet: https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_0a7/openstack/0a726a4277b64223b159a4e81279ff2e/results/report.html#/NovaServers.boot_and_delete_server\n\nscheduler pool stats:\n```\nSep 11 20:38:13.081605 np63e0c85633014 nova-scheduler[63073]: DEBUG nova.utils [None req-9d744297-ac2f-40ef-82f1-28524e609db1 c_rally_35634a4e_aZXGmD8s c_rally_35634a4e_kupEUzGW] State of MainProcess.cell_worker GreenThreadPoolExecutor when submitting a new task: workers: 0, max_workers: 1000, work queued length: 0, stats: \u003cExecutorStatistics object at 0x7f03cd221980 (failures\u003d0, executed\u003d99, runtime\u003d1.56, cancelled\u003d0)\u003e {{(pid\u003d63073) _log_executor_stats /opt/stack/nova/nova/utils.py:1244}}\nSep 11 20:38:14.853219 np63e0c85633014 nova-scheduler[63074]: DEBUG nova.utils [None req-96e1fe1e-b272-4147-abdf-0ee9df2db249 c_rally_35634a4e_aZXGmD8s c_rally_35634a4e_kupEUzGW] State of MainProcess.cell_worker GreenThreadPoolExecutor when submitting a new task: workers: 0, max_workers: 1000, work queued length: 0, stats: \u003cExecutorStatistics object at 0x7f03ccec0b80 (failures\u003d0, executed\u003d99, runtime\u003d1.43, cancelled\u003d0)\u003e {{(pid\u003d63074) _log_executor_stats /opt/stack/nova/nova/utils.py:1244}}\n```\n\nscheduler memory stats:\n```\nSep 11 20:28:54.883759 np63e0c85633014 memory_tracker.sh[36624]:        PID   %MEM             RSS       PPID       TIME     NLWP WCHAN                     COMMAND\n71\t\nSep 11 20:38:16.412908 np63e0c85633014 memory_tracker.sh[71201]:      62156    1.5          129852          1   00:00:06        1 hrtimer_nanosleep         /opt/stack/data/venv/bin/python3.12 /opt/stack/data/venv/bin/nova-scheduler --config-file /etc/nova/nova.conf\nSep 11 20:38:16.413744 np63e0c85633014 memory_tracker.sh[71201]:      63073    1.5          126340      62156   00:00:10        1 ep_poll                   /opt/stack/data/venv/bin/python3.12 /opt/stack/data/venv/bin/nova-scheduler --config-file /etc/nova/nova.conf\nSep 11 20:38:16.413744 np63e0c85633014 memory_tracker.sh[71201]:      63074    1.5          126212      62156   00:00:10        1 ep_poll                   /opt/stack/data/venv/bin/python3.12 /opt/stack/data/venv/bin/nova-scheduler --config-file /etc/nova/nova.conf\n```\n\n---\n\nrun with native threading: https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_0a7/openstack/0a726a4277b64223b159a4e81279ff2e/results/treading-report.html#/NovaServers.boot_and_delete_server\n\nscheduler pool stats\n```\nSep 11 20:42:48.472352 np63e0c85633014 nova-scheduler[71751]: DEBUG nova.utils [None req-88cb95f5-8b6f-4c74-b1a0-a6f2e86d5755 c_rally_699565c4_8i77mJ3C c_rally_699565c4_JDLZSyf7] State of Process-1.cell_worker ThreadPoolExecutor when submitting a new task: max_workers: 5, workers: 5, idle workers: 5, queued work: 0, stats: \u003cExecutorStatistics object at 0x77fa66bfee40 (failures\u003d0, executed\u003d99, runtime\u003d1.47, cancelled\u003d0)\u003e {{(pid\u003d71751) _log_executor_stats /opt/stack/nova/nova/utils.py:1235}}\nSep 11 20:42:50.185192 np63e0c85633014 nova-scheduler[71753]: DEBUG nova.utils [None req-c7c01858-0a5c-4b1d-9004-a1f9bc300971 c_rally_699565c4_8i77mJ3C c_rally_699565c4_JDLZSyf7] State of Process-2.cell_worker ThreadPoolExecutor when submitting a new task: max_workers: 5, workers: 5, idle workers: 5, queued work: 0, stats: \u003cExecutorStatistics object at 0x77fa64190fc0 (failures\u003d0, executed\u003d99, runtime\u003d1.73, cancelled\u003d0)\u003e {{(pid\u003d71753) _log_executor_stats /opt/stack/nova/nova/utils.py:1235}}\n\n```\nscheduler memory stats\n```\nSep 11 20:28:54.883759 np63e0c85633014 memory_tracker.sh[36624]:        PID   %MEM             RSS       PPID       TIME     NLWP WCHAN                     COMMAND\nSep 11 20:40:56.557842 np63e0c85633014 memory_tracker.sh[72387]:      71751    1.5          124984      71603   00:00:03       55 do_select                 nova-scheduler: ServiceWrapper worker(0)\nSep 11 20:40:56.557842 np63e0c85633014 memory_tracker.sh[72387]:      71753    1.5          124856      71603   00:00:03       53 do_select                 nova-scheduler: ServiceWrapper worker(1)\nSep 11 20:41:16.603404 np63e0c85633014 memory_tracker.sh[72465]:      71753    1.5          126392      71603   00:00:04       63 do_select                 nova-scheduler: ServiceWrapper worker(1)\nSep 11 20:41:16.603404 np63e0c85633014 memory_tracker.sh[72465]:      71751    1.5          126008      71603   00:00:04       65 do_select                 nova-scheduler: ServiceWrapper worker(0)\n\n```\n\nNeither the runtime nor the memory consumption of the scheduler is different significantly between eventlet and threading.","commit_id":"cc185f9ba96a536e861b90f1bf832bf89f2f208d"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1f12ccedc335aa0e59d469a478012a41d76fd67c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":30,"id":"b22b4569_7ef02812","updated":"2025-09-29 12:12:56.000000000","message":"Now the recent rally run shows that we are able to catch threading slowdowns. In PS 30 I injected a 100ms sleep in each task scheduled to a native thread pool. This obviously results in a slowdown on the native threaded execution of rally. The new compare-rally-runs role detects the 56% slowdown and fails the job as expected.\nSee\n* https://zuul.opendev.org/t/openstack/build/eb1c16166a9e42b5be0987f5d8ca0d01/log/job-output.txt#24270-24286\n```\n2025-09-29 11:51:46.284173 | TASK [compare-rally-runs : Compare rally runs and fail if any scenario is at least 20% slower with threading]\n24271\t2025-09-29 11:51:46.544584 | controller | {\n24272\t2025-09-29 11:51:46.544619 | controller |   \"Boot and delete a server.\": 1.5580588769456634\n24273\t2025-09-29 11:51:46.544624 | controller | }\n```\n\nHere are the rally reports: \n* eventlet: https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_eb1/openstack/eb1c16166a9e42b5be0987f5d8ca0d01/results/report.html\n* native threading: https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_eb1/openstack/eb1c16166a9e42b5be0987f5d8ca0d01/results/treading-report.html\n\nHere are the processed outputs\n* here is the data showing the load time of both execution https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_eb1/openstack/eb1c16166a9e42b5be0987f5d8ca0d01/results/load_durations.json\n* and here is the result that forced the run to fail: https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_eb1/openstack/eb1c16166a9e42b5be0987f5d8ca0d01/results/load_durations_trend.json","commit_id":"638041f6fa69f9806d6ed6f34295427067751f16"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2566c03c77909ba9d603e3f443faec3374810af1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":31,"id":"cfcb0ea8_ff174e24","updated":"2025-10-21 11:23:28.000000000","message":"now that we landed the conductor threading support this job is also reconfiguring all the conductors","commit_id":"8bdde50b9ff43cdc94b6ba04d624ded3df8fa355"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"7bc8e5e5fd3f62a29e8a58b7134dccc68d898b3a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":35,"id":"2d257364_0ccac0b7","updated":"2025-10-22 07:50:49.000000000","message":"current failure is due to a VM not booting up\n2025-10-21 15:42:45.573807 | controller | rally.exceptions.TimeoutException: Rally tired waiting 300.00 seconds for Server s_rally_0adf9c23_83frtnWz:7fdb389f-2678-47a5-97ad-746071b9fb2f to become (\u0027ACTIVE\u0027) current status BUILD\n\nAt least we know that it is automatically triggering a failure.","commit_id":"775863fcdf4e91f98ea9587102ad1f73b65d1e19"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2efdce6760b6dc84e0abc9b6321af08b6d1ecac8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":35,"id":"9f7899d3_34214f56","updated":"2025-10-21 14:44:48.000000000","message":"no we have automated memory and cpu time comparison as well.","commit_id":"775863fcdf4e91f98ea9587102ad1f73b65d1e19"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2284b9b935b65d08c578e4402f6885bcf28f8238","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":37,"id":"cc41f91d_ba226145","updated":"2025-10-22 15:38:15.000000000","message":"decreased the number of cells from 5 to 3 as the extra nova cell conductors using extra memory and we are running out of memory in threading mode.","commit_id":"222319cb1993c3a6b032acbdf0c2ec712d0b7a40"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"4bfb32f5c6b6946bc22b86a4d8bf44dcbf4bb40f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":46,"id":"1649203f_475e3998","updated":"2025-12-15 14:10:59.000000000","message":"recheck","commit_id":"2fa3a29ff69ce3773dd0f248218dcbb71b10c4e1"}],"nova/utils.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"98608ab877a4d2ed2b8bea7416fb3a149c7e0a3d","unresolved":true,"context_lines":[{"line_number":622,"context_line":"            \"queued. If this happens repeatedly then the size of the pool is \""},{"line_number":623,"context_line":"            \"too small for the load or there are stuck threads filling the \""},{"line_number":624,"context_line":"            \"pool.\", executor.name, func)"},{"line_number":625,"context_line":""},{"line_number":626,"context_line":"    return _pass_context(executor.submit, func, *args, **kwargs)"},{"line_number":627,"context_line":""},{"line_number":628,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"9d95a7ae_1c496cdb","line":625,"updated":"2025-10-21 14:40:56.000000000","message":"whitespace damage","commit_id":"ac5024a2b22b137a6d0945a28714cbcd5f4a7283"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1035c10348e613443e568d70a9d5632afc42a7ce","unresolved":false,"context_lines":[{"line_number":622,"context_line":"            \"queued. If this happens repeatedly then the size of the pool is \""},{"line_number":623,"context_line":"            \"too small for the load or there are stuck threads filling the \""},{"line_number":624,"context_line":"            \"pool.\", executor.name, func)"},{"line_number":625,"context_line":""},{"line_number":626,"context_line":"    return _pass_context(executor.submit, func, *args, **kwargs)"},{"line_number":627,"context_line":""},{"line_number":628,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"534779dd_bbcfb418","line":625,"in_reply_to":"9d95a7ae_1c496cdb","updated":"2025-10-22 07:51:45.000000000","message":"Done","commit_id":"ac5024a2b22b137a6d0945a28714cbcd5f4a7283"}],"rally-jobs/task-nova-threading-scale.yaml":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"dd5972bf17c0f46f761fa5f3987ffa4620cfbbe5","unresolved":true,"context_lines":[{"line_number":21,"context_line":"          runner:"},{"line_number":22,"context_line":"            constant:"},{"line_number":23,"context_line":"              times: 200"},{"line_number":24,"context_line":"              concurrency: 20"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"bb316d19_10e48a7a","line":24,"updated":"2025-09-10 10:05:31.000000000","message":"https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_62d/openstack/62db48c2f72a4f0b9ee666b5348a89aa/results/report.html#/NovaServers.boot_and_delete_server\n\nOK I think this is enough load for now to see clear patterns with eventlet. Now I need to build up the tooling to run the same with threading and then compare.","commit_id":"238b92c326bb9389929a23aba7afa409bc4e24ca"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e6a461d92172aea570e8f18405d47ee00b9c300a","unresolved":true,"context_lines":[{"line_number":21,"context_line":"          runner:"},{"line_number":22,"context_line":"            constant:"},{"line_number":23,"context_line":"              times: 200"},{"line_number":24,"context_line":"              concurrency: 20"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"d8ee679d_bdcc990b","line":24,"in_reply_to":"59597128_3d050104","updated":"2025-09-12 08:58:46.000000000","message":"yes, your interpretation is correct. So this creates ~ 20 parallel boot / delete requests 10 times.","commit_id":"238b92c326bb9389929a23aba7afa409bc4e24ca"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8c9907fd6807f5a6263c81f5882f7ab8e5dcb370","unresolved":true,"context_lines":[{"line_number":21,"context_line":"          runner:"},{"line_number":22,"context_line":"            constant:"},{"line_number":23,"context_line":"              times: 200"},{"line_number":24,"context_line":"              concurrency: 20"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"59597128_3d050104","line":24,"in_reply_to":"bb316d19_10e48a7a","updated":"2025-09-11 18:48:22.000000000","message":"im not really familar with rallly be am i correct in sayign this will boot 200 servers with 20 parallel then as each boots it will delete it and then move on to the next\n\nefectivly doign boot and deletle in a singel tread of exection then moving on to the next boot and delete until 200 instance have been created and destored?","commit_id":"238b92c326bb9389929a23aba7afa409bc4e24ca"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"bb14eff5c77689dc0dc8c98fff8bb62453f287a7","unresolved":false,"context_lines":[{"line_number":21,"context_line":"          runner:"},{"line_number":22,"context_line":"            constant:"},{"line_number":23,"context_line":"              times: 200"},{"line_number":24,"context_line":"              concurrency: 20"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"77854c28_18159338","line":24,"in_reply_to":"d8ee679d_bdcc990b","updated":"2025-12-18 11:46:47.000000000","message":"Acknowledged","commit_id":"238b92c326bb9389929a23aba7afa409bc4e24ca"}],"rally-jobs/task-nova.yaml":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1ae7c46b48aac708166bb6bab789db0d1929e1b9","unresolved":true,"context_lines":[{"line_number":1,"context_line":"{"},{"line_number":2,"context_line":"    \"NovaServers.boot_and_delete_server\": ["},{"line_number":3,"context_line":"        {"},{"line_number":4,"context_line":"            \"args\": {"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"870105e2_e3ed4c87","line":1,"updated":"2025-09-08 19:08:55.000000000","message":"Translate it to more human readable yaml","commit_id":"c7b895ba4fb2071ce62516651ece8b41c219325f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"c4cba8393301d13daf1f733bb5959e71065ce6de","unresolved":false,"context_lines":[{"line_number":1,"context_line":"{"},{"line_number":2,"context_line":"    \"NovaServers.boot_and_delete_server\": ["},{"line_number":3,"context_line":"        {"},{"line_number":4,"context_line":"            \"args\": {"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"4e9387b7_5c7cf343","line":1,"in_reply_to":"870105e2_e3ed4c87","updated":"2025-09-11 16:20:41.000000000","message":"Done","commit_id":"c7b895ba4fb2071ce62516651ece8b41c219325f"}],"roles/configure-native-threading/tasks/main.yaml":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8c9907fd6807f5a6263c81f5882f7ab8e5dcb370","unresolved":true,"context_lines":[{"line_number":10,"context_line":"  loop:"},{"line_number":11,"context_line":"    - /etc/systemd/system/devstack@n-sch.service"},{"line_number":12,"context_line":"    - /etc/systemd/system/devstack@n-api.service"},{"line_number":13,"context_line":"    - /etc/systemd/system/devstack@n-api-meta.service"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"- name: Restart service"},{"line_number":16,"context_line":"  ansible.builtin.systemd_service:"}],"source_content_type":"text/x-yaml","patch_set":13,"id":"8de90955_6c9b0b2c","line":13,"updated":"2025-09-11 18:48:22.000000000","message":"is there a reason you did not do this vai devstack?","commit_id":"b33833abb63c6f8edbd1585cd572e75e66dd9ff1"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"bb14eff5c77689dc0dc8c98fff8bb62453f287a7","unresolved":false,"context_lines":[{"line_number":10,"context_line":"  loop:"},{"line_number":11,"context_line":"    - /etc/systemd/system/devstack@n-sch.service"},{"line_number":12,"context_line":"    - /etc/systemd/system/devstack@n-api.service"},{"line_number":13,"context_line":"    - /etc/systemd/system/devstack@n-api-meta.service"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"- name: Restart service"},{"line_number":16,"context_line":"  ansible.builtin.systemd_service:"}],"source_content_type":"text/x-yaml","patch_set":13,"id":"69ba42f5_f28e5956","line":13,"in_reply_to":"456ad79e_7ea2272e","updated":"2025-12-18 11:46:47.000000000","message":"ah ok that makes sense thanks.\nno no need to restack i was just wondering why you were doing this btu i was not aware you were doing two runs in this change. i just breilfy looked at it and was trying ot understrand the change in general.","commit_id":"b33833abb63c6f8edbd1585cd572e75e66dd9ff1"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e6a461d92172aea570e8f18405d47ee00b9c300a","unresolved":true,"context_lines":[{"line_number":10,"context_line":"  loop:"},{"line_number":11,"context_line":"    - /etc/systemd/system/devstack@n-sch.service"},{"line_number":12,"context_line":"    - /etc/systemd/system/devstack@n-api.service"},{"line_number":13,"context_line":"    - /etc/systemd/system/devstack@n-api-meta.service"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"- name: Restart service"},{"line_number":16,"context_line":"  ansible.builtin.systemd_service:"}],"source_content_type":"text/x-yaml","patch_set":13,"id":"456ad79e_7ea2272e","line":13,"in_reply_to":"8de90955_6c9b0b2c","updated":"2025-09-12 08:58:46.000000000","message":"I do this after the first rally run. So the first rally run is with eventlet, then I reconfigure to threading and then run the same rally task again. So the two rally result are comparable as it was run on the same CI worker node.\n\nDo you suggest to re-stack the whole devstack between the two rally runs? That seems a lot more expensive than a bit of ansible in the post-run phase.","commit_id":"b33833abb63c6f8edbd1585cd572e75e66dd9ff1"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"4a1ec71e41dda30ac8f70ac6a69982c5388759e0","unresolved":true,"context_lines":[{"line_number":7,"context_line":"    section: Service"},{"line_number":8,"context_line":"    option: Environment"},{"line_number":9,"context_line":"    value: \"OS_NOVA_DISABLE_EVENTLET_PATCHING\u003dtrue\""},{"line_number":10,"context_line":"  loop:"},{"line_number":11,"context_line":"    - /etc/systemd/system/devstack@n-sch.service"},{"line_number":12,"context_line":"    - /etc/systemd/system/devstack@n-api.service"},{"line_number":13,"context_line":"    - /etc/systemd/system/devstack@n-api-meta.service"}],"source_content_type":"text/x-yaml","patch_set":30,"id":"655d7dea_32167344","line":10,"updated":"2025-10-20 16:31:09.000000000","message":"Add all conductors here","commit_id":"638041f6fa69f9806d6ed6f34295427067751f16"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"acc7e3f4c450a76c2ed2fc2bf4bb77a7deea9404","unresolved":true,"context_lines":[{"line_number":7,"context_line":"    section: Service"},{"line_number":8,"context_line":"    option: Environment"},{"line_number":9,"context_line":"    value: \"OS_NOVA_DISABLE_EVENTLET_PATCHING\u003dtrue\""},{"line_number":10,"context_line":"  loop:"},{"line_number":11,"context_line":"    - /etc/systemd/system/devstack@n-sch.service"},{"line_number":12,"context_line":"    - /etc/systemd/system/devstack@n-api.service"},{"line_number":13,"context_line":"    - /etc/systemd/system/devstack@n-api-meta.service"}],"source_content_type":"text/x-yaml","patch_set":30,"id":"94a55da0_5530c22e","line":10,"in_reply_to":"655d7dea_32167344","updated":"2025-10-20 16:33:44.000000000","message":"As globbing is not supported neither in ini_file nor in systemd_service we need to read what conductor services are on the node to catch them all in a multi cell setuo","commit_id":"638041f6fa69f9806d6ed6f34295427067751f16"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"6fd144d6fd4d3221b1b485b9e8f4168b68e39135","unresolved":false,"context_lines":[{"line_number":7,"context_line":"    section: Service"},{"line_number":8,"context_line":"    option: Environment"},{"line_number":9,"context_line":"    value: \"OS_NOVA_DISABLE_EVENTLET_PATCHING\u003dtrue\""},{"line_number":10,"context_line":"  loop:"},{"line_number":11,"context_line":"    - /etc/systemd/system/devstack@n-sch.service"},{"line_number":12,"context_line":"    - /etc/systemd/system/devstack@n-api.service"},{"line_number":13,"context_line":"    - /etc/systemd/system/devstack@n-api-meta.service"}],"source_content_type":"text/x-yaml","patch_set":30,"id":"85790645_031ac488","line":10,"in_reply_to":"94a55da0_5530c22e","updated":"2025-10-21 11:27:44.000000000","message":"Done","commit_id":"638041f6fa69f9806d6ed6f34295427067751f16"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"4a1ec71e41dda30ac8f70ac6a69982c5388759e0","unresolved":true,"context_lines":[{"line_number":20,"context_line":"  loop:"},{"line_number":21,"context_line":"    - devstack@n-sch.service"},{"line_number":22,"context_line":"    - devstack@n-api.service"},{"line_number":23,"context_line":"    - devstack@n-api-meta.service"},{"line_number":24,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":30,"id":"ba89d8e9_e76607ec","line":23,"updated":"2025-10-20 16:31:09.000000000","message":"Add all conductors here","commit_id":"638041f6fa69f9806d6ed6f34295427067751f16"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"6fd144d6fd4d3221b1b485b9e8f4168b68e39135","unresolved":false,"context_lines":[{"line_number":20,"context_line":"  loop:"},{"line_number":21,"context_line":"    - devstack@n-sch.service"},{"line_number":22,"context_line":"    - devstack@n-api.service"},{"line_number":23,"context_line":"    - devstack@n-api-meta.service"},{"line_number":24,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":30,"id":"24485622_ecd0f058","line":23,"in_reply_to":"ba89d8e9_e76607ec","updated":"2025-10-21 11:27:44.000000000","message":"Done","commit_id":"638041f6fa69f9806d6ed6f34295427067751f16"}]}