)]}'
{"id":"openstack%2Fnova~815017","triplet_id":"openstack%2Fnova~master~If5a0f2daace83cb85030cb3e253608a90f733ce4","project":"openstack/nova","branch":"master","topic":"bug/1946339","attention_set":{},"removed_from_attention_set":{"9708":{"account":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"last_update":"2021-10-25 17:39:04.000000000","reason":"Change was abandoned"},"11604":{"account":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"last_update":"2021-10-25 17:39:04.000000000","reason":"Change was abandoned"},"4690":{"account":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"last_update":"2021-10-25 17:39:04.000000000","reason":"Change was abandoned"}},"hashtags":[],"change_id":"If5a0f2daace83cb85030cb3e253608a90f733ce4","subject":"Prevent leaked greenlets to interact with later tests","status":"ABANDONED","created":"2021-10-21 15:05:04.000000000","updated":"2021-10-25 17:39:04.000000000","total_comment_count":1,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"679f5c9cb38faa8287de89783eb6e94784106653","_number":815017,"virtual_id_number":815017,"owner":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"actions":{},"labels":{"Verified":{"disliked":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"tag":"autogenerated:zuul:check","value":-1,"date":"2021-10-21 16:23:19.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":-1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"rejected":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"all":[{"value":-1,"date":"2021-10-21 17:31:43.000000000","permitted_voting_range":{"min":-1,"max":1},"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Review-Priority":{"all":[{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"value":0,"permitted_voting_range":{"min":0,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"}],"values":{" 0":"Default Priority","+1":"Contributor Review Promise","+2":"Core Review Promise"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"CC":[{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"}],"REVIEWER":[{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2021-10-21 15:10:17.000000000","updated_by":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"reviewer":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"state":"REVIEWER"},{"updated":"2021-10-21 15:10:17.000000000","updated_by":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"reviewer":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"state":"REVIEWER"},{"updated":"2021-10-21 15:51:56.000000000","updated_by":{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},"reviewer":{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2021-10-21 16:16:54.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2021-10-21 16:23:19.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2021-10-21 17:38:14.000000000","updated_by":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"reviewer":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"state":"CC"}],"messages":[{"id":"a390b5a1144ae5ab438674e17332f948a996926c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2021-10-21 15:05:04.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"6319ff122c20e1ef2e36c495e0900ad1825822b5","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2021-10-21 15:10:14.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"983105ca99bcc2724722513ff4dbf9cbb4e86b15","author":{"_account_id":10118,"name":"IBM PowerKVM CI","email":"kvmpower@linux.vnet.ibm.com","username":"powerkvm","tags":["SERVICE_USER"]},"date":"2021-10-21 15:51:56.000000000","message":"Patch Set 2:\n\nBuild failed. Test completed on IBM PowerKVM platform. For rechecking only on the IBM PowerKVM CI, add a review comment with pkvm- recheck. For contact and more information, see https://wiki.openstack.org/wiki/PowerKVM\n\n- tempest-dsvm-full-focal-py3 https://oplab9.parqtec.unicamp.br/pub/ppc64el/openstack/nova/17/815017/2/check/tempest-dsvm-full-focal-py3/3e3815b/ : FAILURE in 35m 21s","accounts_in_message":[],"_revision_number":2},{"id":"46ccaa10529daee4b8962c8684260b46f6789132","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-21 16:16:54.000000000","message":"Patch Set 2:\n\nBuild succeeded (ARM64 pipeline).\n\n- openstack-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/6906b7b27e40468da9b2009a82d5a613 : TIMED_OUT in 1h 01m 19s (non-voting)\n- openstack-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/0e0fb2c371394a1eaa20183278cbeaf4 : TIMED_OUT in 1h 01m 18s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"dcd38a7008ad36bc54392e040387abb9990c509e","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-21 16:23:19.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/5f6c76030a3d4554b4b678d75faa6607 : TIMED_OUT in 51m 31s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/0a6b6e728430428aa80c325db9cc39c0 : TIMED_OUT in 41m 35s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/d0a99be3889642c5ba666f43c3222194 : SUCCESS in 10m 44s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/01b55439f60b49efbcaa44bbd52509f2 : TIMED_OUT in 41m 45s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/18ea1b3f28474149b5da2367afc5e21d : TIMED_OUT in 41m 17s\n- openstack-tox-py39 https://zuul.opendev.org/t/openstack/build/b17759d460434590af8723a8493f40ec : TIMED_OUT in 41m 49s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/85fbe870ad274ef0bc7e07a29e9322b1 : SUCCESS in 11m 41s\n- nova-tox-validate-backport https://zuul.opendev.org/t/openstack/build/4be7164418d54a9497bbcfe39a765484 : SUCCESS in 8m 17s (non-voting)\n- nova-tox-functional-centos8-py36 https://zuul.opendev.org/t/openstack/build/5499fcbb1a4e424ea8085903aae69238 : FAILURE in 48m 36s\n- nova-tox-functional-py38 https://zuul.opendev.org/t/openstack/build/1c64c4d198f7435e82d3c44ce755fb91 : FAILURE in 41m 17s\n- nova-tox-functional-py39 https://zuul.opendev.org/t/openstack/build/0c5af30ee95540e1a460ada5296b6544 : TIMED_OUT in 1h 01m 54s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"4eaad470fad52e758f905189d9b709f6642ea974","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2021-10-21 17:31:43.000000000","message":"Patch Set 2: Workflow-1\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"921e13210cbd903c237782fc11eb211aa022fd62","tag":"autogenerated:cloudbase-ci","author":{"_account_id":26515,"name":"Cloudbase Nova Hyper-V CI","email":"nova_hyperv_ci@cloudbasesolutions.com","username":"nova_hyperv_ci"},"date":"2021-10-21 17:38:14.000000000","message":"Patch Set 2:\n\nBuild succeeded. For checking Nova with Ceph, add a comment with run-Cloudbase Nova Ceph Hyper-V CI\n\n- nova http://cloudbase-ci.com/nova/815017/2/iscsi : SUCCESS in 2h 26m 56s","accounts_in_message":[],"_revision_number":2},{"id":"679f5c9cb38faa8287de89783eb6e94784106653","tag":"autogenerated:gerrit:abandon","author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"date":"2021-10-25 17:39:04.000000000","message":"Abandoned\n\nThis is a wack-a-mole. Our functional tests especially the ones  starting a libvirt driver leak many strange eventlet proxy objects and greenlets. E.g. the test run are full of unmocked libvirt.host._connect() calls. If I kill those at the end of the test case then later tests hangs as I probably broke some eventlet global state by killing eventlet managed greenlets. I gave up. Most of these leaked stuff eventually terminate and does not interfere with later test executions. But it does not allow us to make a proper cleanup between test cases.","accounts_in_message":[],"_revision_number":2}],"current_revision_number":2,"current_revision":"3b10f00b66743dca6bbed378aff1c4146565f5c3","revisions":{"9dadda15d7f9444fa7ff5b93208fca4ee50dfe51":{"kind":"REWORK","_number":1,"created":"2021-10-21 15:05:04.000000000","uploader":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"ref":"refs/changes/17/815017/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/nova","ref":"refs/changes/17/815017/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/nova refs/changes/17/815017/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/nova refs/changes/17/815017/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/nova refs/changes/17/815017/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/nova refs/changes/17/815017/1"}}},"commit":{"parents":[{"commit":"e14eef0719eceef35e7e96b3e3d242ec79a80969","subject":"Merge \"Fix the wrong exception used to retry detach API calls\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/e14eef0719eceef35e7e96b3e3d242ec79a80969"}]}],"author":{"name":"Balazs Gibizer","email":"balazs.gibizer@est.tech","date":"2021-10-21 14:48:10.000000000","tz":120},"committer":{"name":"Balazs Gibizer","email":"balazs.gibizer@est.tech","date":"2021-10-21 15:02:34.000000000","tz":120},"subject":"Prevent leaked greenlets to interact with later tests","message":"Prevent leaked greenlets to interact with later tests\n\nIn out functional tests we run nova services as greenlets. Also those\nservices can spawn there own greenlets for RPC or other parallel\nprocessing. The test case executor only sees and tracks the main\ngreenlet where the code of the test case is running. When that is\nfinishes the test executor considers the test case to be finished\nregardless of the other spawned greenlets. This could lead to leaked\ngreenlets that are running in parallel with later test cases.\n\nOne way that it can cause trouble is via the global variables in\nnova.rpc module. Those globals are re-initialized for each test case so\nthey are not directly leaking information between test cases. However if\na late greenlet calls nova.rpc.get_versioned_notifier() it will get a\ntotally usable FakeVersionedNotifier object regardless of which test\ncase this notifier is belongs to or which test case the greenlet belongs\nto. This way the late greenelt can send a notification to the currently\nrunning test case and therefore can make it fail.\n\nThe current case we saw is the following:\n\n1) The test case\n  nova.tests.functional.test_servers.ServersTestV219.test_description_errors\n  creates a server but don\u0027t wait for it to reach terminal state (ACTIVE\n  / ERROR). This test case finishes quickly but leaks running greenlets\n  in the background waiting for some RPC call to return.\n2) As the test case finished the cleanup code deletes the test case\n   specific setup, including the DB.\n3) The test executor moves forward and starts running another test case\n4) 60 seconds later the leaked greenlet times out waiting for the RPC\n   call to return and tries doing things, but fails as the DB is already\n   gone. Then it tries to  report this as an error notification. It calls\n   nova.rpc.get_versioned_notifier() and gets a fresh notifier that is\n   connected to the currently running test case. Then emits the error\n   notification there.\n5) The currently running test case also waits for an error notification\n   to be triggered by the currently running test code. But it gets the\n   notification form the late greenlet first. As the content of the\n   notification does not match with the expectations the currently\n   running test case fails. The late greenlet prints a lot of\n   error about the DB being gone making the troubleshooting pretty hard.\n\nThis patch proposes a way to kill every leaking greenlets at the end of\neach test case.\n\nChange-Id: If5a0f2daace83cb85030cb3e253608a90f733ce4\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/9dadda15d7f9444fa7ff5b93208fca4ee50dfe51"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/9dadda15d7f9444fa7ff5b93208fca4ee50dfe51"}]},"branch":"refs/heads/master"},"3b10f00b66743dca6bbed378aff1c4146565f5c3":{"kind":"REWORK","_number":2,"created":"2021-10-21 15:10:14.000000000","uploader":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"ref":"refs/changes/17/815017/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/nova","ref":"refs/changes/17/815017/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/nova refs/changes/17/815017/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/nova refs/changes/17/815017/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/nova refs/changes/17/815017/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/nova refs/changes/17/815017/2"}}},"commit":{"parents":[{"commit":"e14eef0719eceef35e7e96b3e3d242ec79a80969","subject":"Merge \"Fix the wrong exception used to retry detach API calls\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/e14eef0719eceef35e7e96b3e3d242ec79a80969"}]}],"author":{"name":"Balazs Gibizer","email":"balazs.gibizer@est.tech","date":"2021-10-21 14:48:10.000000000","tz":120},"committer":{"name":"Balazs Gibizer","email":"balazs.gibizer@est.tech","date":"2021-10-21 15:08:47.000000000","tz":120},"subject":"Prevent leaked greenlets to interact with later tests","message":"Prevent leaked greenlets to interact with later tests\n\nIn out functional tests we run nova services as greenlets. Also those\nservices can spawn there own greenlets for RPC or other parallel\nprocessing. The test case executor only sees and tracks the main\ngreenlet where the code of the test case is running. When that is\nfinishes the test executor considers the test case to be finished\nregardless of the other spawned greenlets. This could lead to leaked\ngreenlets that are running in parallel with later test cases.\n\nOne way that it can cause trouble is via the global variables in\nnova.rpc module. Those globals are re-initialized for each test case so\nthey are not directly leaking information between test cases. However if\na late greenlet calls nova.rpc.get_versioned_notifier() it will get a\ntotally usable FakeVersionedNotifier object regardless of which test\ncase this notifier is belongs to or which test case the greenlet belongs\nto. This way the late greenelt can send a notification to the currently\nrunning test case and therefore can make it fail.\n\nThe current case we saw is the following:\n\n1) The test case\n  nova.tests.functional.test_servers.ServersTestV219.test_description_errors\n  creates a server but don\u0027t wait for it to reach terminal state (ACTIVE\n  / ERROR). This test case finishes quickly but leaks running greenlets\n  in the background waiting for some RPC call to return.\n2) As the test case finished the cleanup code deletes the test case\n   specific setup, including the DB.\n3) The test executor moves forward and starts running another test case\n4) 60 seconds later the leaked greenlet times out waiting for the RPC\n   call to return and tries doing things, but fails as the DB is already\n   gone. Then it tries to  report this as an error notification. It calls\n   nova.rpc.get_versioned_notifier() and gets a fresh notifier that is\n   connected to the currently running test case. Then emits the error\n   notification there.\n5) The currently running test case also waits for an error notification\n   to be triggered by the currently running test code. But it gets the\n   notification form the late greenlet first. As the content of the\n   notification does not match with the expectations the currently\n   running test case fails. The late greenlet prints a lot of\n   error about the DB being gone making the troubleshooting pretty hard.\n\nThis patch proposes a way to kill every leaking greenlets at the end of\neach test case.\n\nChange-Id: If5a0f2daace83cb85030cb3e253608a90f733ce4\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/3b10f00b66743dca6bbed378aff1c4146565f5c3"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/nova/commit/3b10f00b66743dca6bbed378aff1c4146565f5c3"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}