)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"060871ff7eba9e9694a2075c9b6ef4096c123a75","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"fe6b4264_3905f8ba","updated":"2026-09-23 13:12:59.000000000","message":"1. I will going to test this locally... \n2. I don\u0027t have the full understanding what is happening here so I\u0027m flying a bit blind here.\n3. I have some suggestion in the test case to make it more stable.","commit_id":"bc416fbcd8b749add2fb203b81c9101e3511a540"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e6d33587fa46ee6de9b0fd9d91946c8279ce6019","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"285df962_db1b37ba","in_reply_to":"fe6b4264_3905f8ba","updated":"2026-09-23 13:44:56.000000000","message":"\u003e 1. I will going to test this locally... \n\nDone. Indeed this resolves the hang I can reproduce locally.","commit_id":"bc416fbcd8b749add2fb203b81c9101e3511a540"}],"nova/tests/fixtures/nova.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"060871ff7eba9e9694a2075c9b6ef4096c123a75","unresolved":true,"context_lines":[{"line_number":654,"context_line":""},{"line_number":655,"context_line":"        # NOTE(melwitt): This is a hack to try to deal with"},{"line_number":656,"context_line":"        # local accesses i.e. non target_cell accesses."},{"line_number":657,"context_line":"        with self._cell_lock.read_lock():"},{"line_number":658,"context_line":"            # FIXME(mriedem): This is actually misleading and means we don\u0027t"},{"line_number":659,"context_line":"            # catch things like bug 1717000 where a context should be targeted"},{"line_number":660,"context_line":"            # to a cell but it\u0027s not, and the fixture here just returns the"}],"source_content_type":"text/x-python","patch_set":1,"id":"3ee5af58_7511f610","side":"PARENT","line":657,"updated":"2026-09-23 13:12:59.000000000","message":"this part of the fixture is so obscure that I cannot even judge that it is the right move or not. @melwittt@gmail.com I know that this was 10 years ago but maybe you still have some context about this lock. Could you please look at this and see if removing this read lock could cause unexpected side effects?\n\nWe need to remove the deadlocks so I\u0027m tempted to test this locally to see if it resolves my reproduction of the hang and if so I will simply suggest to do this and handle the surprises later as I have no better idea at the moment but the hangs are baaad.","commit_id":"94de0576ca02e0668233bedfbe7e18b87316c708"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"5e32db8bcaef8c063ba9f31378a4f4ff12721543","unresolved":true,"context_lines":[{"line_number":654,"context_line":""},{"line_number":655,"context_line":"        # NOTE(melwitt): This is a hack to try to deal with"},{"line_number":656,"context_line":"        # local accesses i.e. non target_cell accesses."},{"line_number":657,"context_line":"        with self._cell_lock.read_lock():"},{"line_number":658,"context_line":"            # FIXME(mriedem): This is actually misleading and means we don\u0027t"},{"line_number":659,"context_line":"            # catch things like bug 1717000 where a context should be targeted"},{"line_number":660,"context_line":"            # to a cell but it\u0027s not, and the fixture here just returns the"}],"source_content_type":"text/x-python","patch_set":1,"id":"0fa0a87a_8e4be2d0","side":"PARENT","line":657,"in_reply_to":"03081ed1_5f99cc5c","updated":"2026-09-24 07:30:40.000000000","message":"Yeah ... this stuff is a confusing mess 🫠 TBH I\u0027m not sure if removing this read lock could cause unexpected side effects but if it does, it\u0027s likely less bad than the deadlock we are currently dealing with. So I would not be opposed to removing it to see how it goes.\n\nI did some AI-assisted investigation into what could have caused this \"recent\" deadlock behavior it suspected this recent change to parallelize per-node resource updates [1]. This change added another thread spawn point which might have created the conditions for this deadlock to happen (a new thread that will also try to acquire the lock when previously there was not). This seemed a plausible explanation to me.\n\nWhat I would like to do longer term though is move away from this shared locking stuff and use thread local variables instead, which is [a change I proposed months ago](https://review.opendev.org/c/openstack/nova/+/984480/1/nova/tests/fixtures/nova.py) but did not follow up on because things got busy and a lot of other eventlet removal work was going on at the same time.\n\nI have just rebased and made some tweaks to the patch ^, so perhaps maybe we could reconsider it soon.\n\n[1] https://review.opendev.org/c/openstack/nova/+/980679","commit_id":"94de0576ca02e0668233bedfbe7e18b87316c708"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a2c66921f488be6c1ed30cf81848fa751a36edba","unresolved":true,"context_lines":[{"line_number":654,"context_line":""},{"line_number":655,"context_line":"        # NOTE(melwitt): This is a hack to try to deal with"},{"line_number":656,"context_line":"        # local accesses i.e. non target_cell accesses."},{"line_number":657,"context_line":"        with self._cell_lock.read_lock():"},{"line_number":658,"context_line":"            # FIXME(mriedem): This is actually misleading and means we don\u0027t"},{"line_number":659,"context_line":"            # catch things like bug 1717000 where a context should be targeted"},{"line_number":660,"context_line":"            # to a cell but it\u0027s not, and the fixture here just returns the"}],"source_content_type":"text/x-python","patch_set":1,"id":"e2a9503e_da21d6d8","side":"PARENT","line":657,"in_reply_to":"0fa0a87a_8e4be2d0","updated":"2026-09-24 11:29:43.000000000","message":"@melwittt@gmail.com thanks. I run your change and that also seems to fix the hang of this functional tests. I feel like you change is a good step forward and a bit better understood than this current change from @rene.ribaud@gmail.com. So I rather land yours at the moment. \n\nI still think that running the periodics on multiple computes within a computes RPC handler is a super unrealistic test and I will try to fix this test case independently as I think it could cause other problems too.","commit_id":"94de0576ca02e0668233bedfbe7e18b87316c708"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"7c23329ee44fa3739ac8c56240c9d123a763da1d","unresolved":true,"context_lines":[{"line_number":654,"context_line":""},{"line_number":655,"context_line":"        # NOTE(melwitt): This is a hack to try to deal with"},{"line_number":656,"context_line":"        # local accesses i.e. non target_cell accesses."},{"line_number":657,"context_line":"        with self._cell_lock.read_lock():"},{"line_number":658,"context_line":"            # FIXME(mriedem): This is actually misleading and means we don\u0027t"},{"line_number":659,"context_line":"            # catch things like bug 1717000 where a context should be targeted"},{"line_number":660,"context_line":"            # to a cell but it\u0027s not, and the fixture here just returns the"}],"source_content_type":"text/x-python","patch_set":1,"id":"03081ed1_5f99cc5c","side":"PARENT","line":657,"in_reply_to":"3ee5af58_7511f610","updated":"2026-09-23 13:33:04.000000000","message":"OK I deduced something form the hanging test case. It does something unrealistic. It mocks a compute method and calls run_periodics() inside it. That means a method on compute A runs now a method that does things with compute A and compute B as well. Morovere to do that it targets the context to do so. So there is a cell targeting call embedded within a compute function. That never happens in a normal case as each context on the compute side is expected to be already targeted to the cell the compute is in. Probably this contributes to the deadlock on that read lock Rene identified.\n\nhttps://github.com/openstack/nova/blob/94de0576ca02e0668233bedfbe7e18b87316c708/nova/tests/functional/libvirt/test_numa_live_migration.py#L414\nhttps://github.com/openstack/nova/blob/94de0576ca02e0668233bedfbe7e18b87316c708/nova/tests/functional/libvirt/test_numa_live_migration.py#L382\nhttps://github.com/openstack/nova/blob/94de0576ca02e0668233bedfbe7e18b87316c708/nova/test.py#L551\n\nBased on this anther alternative would be to run the periodics not within the compute method but from the a separate thread in the test and syncronize with events across the computes and the test thread. I will try to implement that to see if it is possible.","commit_id":"94de0576ca02e0668233bedfbe7e18b87316c708"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"8f9a0df7a5a9ee9955c196f1fa636c7d81179ad4","unresolved":true,"context_lines":[{"line_number":654,"context_line":""},{"line_number":655,"context_line":"        # NOTE(melwitt): This is a hack to try to deal with"},{"line_number":656,"context_line":"        # local accesses i.e. non target_cell accesses."},{"line_number":657,"context_line":"        with self._cell_lock.read_lock():"},{"line_number":658,"context_line":"            # FIXME(mriedem): This is actually misleading and means we don\u0027t"},{"line_number":659,"context_line":"            # catch things like bug 1717000 where a context should be targeted"},{"line_number":660,"context_line":"            # to a cell but it\u0027s not, and the fixture here just returns the"}],"source_content_type":"text/x-python","patch_set":1,"id":"96a86659_ee810ac8","side":"PARENT","line":657,"in_reply_to":"e2a9503e_da21d6d8","updated":"2026-09-24 13:02:28.000000000","message":"https://review.opendev.org/c/openstack/nova/+/1007150 this is the refactor of the test case to move the run_periodics call out from the compute RPC handler during the test.","commit_id":"94de0576ca02e0668233bedfbe7e18b87316c708"}],"nova/tests/unit/test_fixtures.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"060871ff7eba9e9694a2075c9b6ef4096c123a75","unresolved":true,"context_lines":[{"line_number":278,"context_line":"        cells \u003d fixtures.CellDatabases()"},{"line_number":279,"context_line":"        cells._last_ctxt_mgr \u003d mock.sentinel.current_cell"},{"line_number":280,"context_line":"        ctxt \u003d context.get_admin_context()"},{"line_number":281,"context_line":"        writer_waiting \u003d threading.Event()"},{"line_number":282,"context_line":"        lookup_done \u003d threading.Event()"},{"line_number":283,"context_line":"        results \u003d []"},{"line_number":284,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"6402ce3f_a1f450fe","line":281,"updated":"2026-09-23 13:12:59.000000000","message":"add self.addCleanup(writer_waiting.set) and similar to all events to allow a failing test to terminate","commit_id":"bc416fbcd8b749add2fb203b81c9101e3511a540"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"060871ff7eba9e9694a2075c9b6ef4096c123a75","unresolved":true,"context_lines":[{"line_number":299,"context_line":"            return wait(*args, **kwargs)"},{"line_number":300,"context_line":""},{"line_number":301,"context_line":"        writer \u003d threading.Thread(target\u003dchange_cell)"},{"line_number":302,"context_line":"        reader \u003d threading.Thread(target\u003dlookup_cell)"},{"line_number":303,"context_line":"        with mock.patch.object(cells._cell_lock._cond, \u0027wait\u0027,"},{"line_number":304,"context_line":"                               side_effect\u003dwait_for_readers):"},{"line_number":305,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":1,"id":"10e863f5_a36fa037","line":302,"updated":"2026-09-23 13:12:59.000000000","message":"ensure that these threads are joined during the cleanup of the test. We don\u0027t want to leak running threads across test cases within the same test executor.","commit_id":"bc416fbcd8b749add2fb203b81c9101e3511a540"}]}