)]}'
{"tempest/api/object_storage/base.py":[{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"779817e83d03bacf16869c0ea8c30292224851e3","unresolved":true,"context_lines":[{"line_number":52,"context_line":"            # Verify resource deletion"},{"line_number":53,"context_line":"            container_client.delete_container(cont)"},{"line_number":54,"context_line":"            # Remove the container name from the list"},{"line_number":55,"context_line":"            containers.remove(cont)"},{"line_number":56,"context_line":"            container_client.wait_for_resource_deletion(cont)"},{"line_number":57,"context_line":"        except lib_exc.NotFound:"},{"line_number":58,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":2,"id":"660f2e66_6d4306d4","line":55,"range":{"start_line":55,"start_character":12,"end_line":55,"end_character":35},"updated":"2021-07-27 19:57:41.000000000","message":"This may cause issues with removing an item from a collection while iterating over it.","commit_id":"12e52a792536bd098817560c87b29809ecfc60cc"},{"author":{"_account_id":33282,"name":"Huy Tran","email":"huy.q.tran@ericsson.com","username":"ht095u"},"change_message_id":"1ca81067817b9bd7ccf9995685189ddb09a831bb","unresolved":false,"context_lines":[{"line_number":52,"context_line":"            # Verify resource deletion"},{"line_number":53,"context_line":"            container_client.delete_container(cont)"},{"line_number":54,"context_line":"            # Remove the container name from the list"},{"line_number":55,"context_line":"            containers.remove(cont)"},{"line_number":56,"context_line":"            container_client.wait_for_resource_deletion(cont)"},{"line_number":57,"context_line":"        except lib_exc.NotFound:"},{"line_number":58,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":2,"id":"a7a64476_fa56ef98","line":55,"range":{"start_line":55,"start_character":12,"end_line":55,"end_character":35},"in_reply_to":"660f2e66_6d4306d4","updated":"2021-07-28 11:41:49.000000000","message":"Ah yes! some items would not be removed for testcases that created more than one containers. Code has been updated. Thanks!","commit_id":"12e52a792536bd098817560c87b29809ecfc60cc"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"e7ec69b9262c62653ce0dd31c4b537df4146f32b","unresolved":true,"context_lines":[{"line_number":51,"context_line":"                object_client.wait_for_resource_deletion(obj[\u0027name\u0027], cont)"},{"line_number":52,"context_line":"            # Verify resource deletion"},{"line_number":53,"context_line":"            container_client.delete_container(cont)"},{"line_number":54,"context_line":"            # Remove the container name from the list"},{"line_number":55,"context_line":"            containers.remove(cont)"},{"line_number":56,"context_line":"            container_client.wait_for_resource_deletion(cont)"},{"line_number":57,"context_line":"        except lib_exc.NotFound:"},{"line_number":58,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":4,"id":"9828a264_e15f24c6","line":55,"range":{"start_line":54,"start_character":0,"end_line":55,"end_character":35},"updated":"2021-07-30 15:43:37.000000000","message":"this can move after wait after L56. wait call is sync call so until it is deleted we will not process next.","commit_id":"3fe3cef7b7967bfb53abc1fc4a4a1ded32d68347"},{"author":{"_account_id":33282,"name":"Huy Tran","email":"huy.q.tran@ericsson.com","username":"ht095u"},"change_message_id":"c8feb3868fbd9c34f8ddaa1db4a063d140fa3dd3","unresolved":true,"context_lines":[{"line_number":51,"context_line":"                object_client.wait_for_resource_deletion(obj[\u0027name\u0027], cont)"},{"line_number":52,"context_line":"            # Verify resource deletion"},{"line_number":53,"context_line":"            container_client.delete_container(cont)"},{"line_number":54,"context_line":"            # Remove the container name from the list"},{"line_number":55,"context_line":"            containers.remove(cont)"},{"line_number":56,"context_line":"            container_client.wait_for_resource_deletion(cont)"},{"line_number":57,"context_line":"        except lib_exc.NotFound:"},{"line_number":58,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":4,"id":"b6c7b70e_36d75c4a","line":55,"range":{"start_line":54,"start_character":0,"end_line":55,"end_character":35},"in_reply_to":"033252ad_fd6585f6","updated":"2021-08-02 13:54:31.000000000","message":"Bug https://bugs.launchpad.net/tempest/+bug/1938593 has been created with logs per discussion.","commit_id":"3fe3cef7b7967bfb53abc1fc4a4a1ded32d68347"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"f34abfa17dd2c4e81f20acf6d48359f81c3918a7","unresolved":true,"context_lines":[{"line_number":51,"context_line":"                object_client.wait_for_resource_deletion(obj[\u0027name\u0027], cont)"},{"line_number":52,"context_line":"            # Verify resource deletion"},{"line_number":53,"context_line":"            container_client.delete_container(cont)"},{"line_number":54,"context_line":"            # Remove the container name from the list"},{"line_number":55,"context_line":"            containers.remove(cont)"},{"line_number":56,"context_line":"            container_client.wait_for_resource_deletion(cont)"},{"line_number":57,"context_line":"        except lib_exc.NotFound:"},{"line_number":58,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":4,"id":"033252ad_fd6585f6","line":55,"range":{"start_line":54,"start_character":0,"end_line":55,"end_character":35},"in_reply_to":"9477d34f_75778b8c","updated":"2021-07-30 16:19:02.000000000","message":"tearDown are run per test case and within one test class, all the test cases run in serial means every test tearDown will be called in serial. Yes, test case tearDown of other test class can be run in parallel but they should not delete the container of other test class.\n\nSo basically delete_containers() request will be coming from a test case with either containers created within the test case or created at test class level. There cannot be delete_containers() call which try to delete the containers created in other test class. if so then it is bug in test.","commit_id":"3fe3cef7b7967bfb53abc1fc4a4a1ded32d68347"},{"author":{"_account_id":33282,"name":"Huy Tran","email":"huy.q.tran@ericsson.com","username":"ht095u"},"change_message_id":"97d72fd82bd3760e34b88e0b2064df6a8b886954","unresolved":true,"context_lines":[{"line_number":51,"context_line":"                object_client.wait_for_resource_deletion(obj[\u0027name\u0027], cont)"},{"line_number":52,"context_line":"            # Verify resource deletion"},{"line_number":53,"context_line":"            container_client.delete_container(cont)"},{"line_number":54,"context_line":"            # Remove the container name from the list"},{"line_number":55,"context_line":"            containers.remove(cont)"},{"line_number":56,"context_line":"            container_client.wait_for_resource_deletion(cont)"},{"line_number":57,"context_line":"        except lib_exc.NotFound:"},{"line_number":58,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":4,"id":"9477d34f_75778b8c","line":55,"range":{"start_line":54,"start_character":0,"end_line":55,"end_character":35},"in_reply_to":"9828a264_e15f24c6","updated":"2021-07-30 16:00:19.000000000","message":"Thanks for the comments. Per observation in the lab, the tearDown for each method is not being called in sync. The tearDown of current method can be called while tearDown of previous method is waiting due to wait_for_resource_deletion. If moving the line as suggested, the subsequent method will likely send extra request to delete same container while is being deleted per request of previous method.","commit_id":"3fe3cef7b7967bfb53abc1fc4a4a1ded32d68347"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"ca2b3c1f76577a8fef45b8121aff92d80c9065bc","unresolved":true,"context_lines":[{"line_number":43,"context_line":""},{"line_number":44,"context_line":"    for cont in list(containers):"},{"line_number":45,"context_line":"        try:"},{"line_number":46,"context_line":"            # Remove the container name from the list before"},{"line_number":47,"context_line":"            # any condenses to reduce possible race condition"},{"line_number":48,"context_line":"            # that can occur when tearDown is invoked by"},{"line_number":49,"context_line":"            # subsequent method/test while message is being sent"},{"line_number":50,"context_line":"            # and waiting for resource deletion"},{"line_number":51,"context_line":"            containers.remove(cont)"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"            params \u003d {\u0027limit\u0027: 9999, \u0027format\u0027: \u0027json\u0027}"},{"line_number":54,"context_line":"            _, objlist \u003d container_client.list_container_objects(cont, params)"},{"line_number":55,"context_line":"            # delete every object in the container"}],"source_content_type":"text/x-python","patch_set":5,"id":"46e88215_7994cd3f","line":52,"range":{"start_line":46,"start_character":0,"end_line":52,"end_character":0},"updated":"2021-08-03 21:05:13.000000000","message":"thanks for explaining in bug report, now it is clear to me.\n\nInstead of here deleting, we should delete in class method delete_containers()at L137","commit_id":"c026cf10849f8b6ff66f3f8b88659c8d07a2d156"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"ca2b3c1f76577a8fef45b8121aff92d80c9065bc","unresolved":true,"context_lines":[{"line_number":140,"context_line":"        if object_client is None:"},{"line_number":141,"context_line":"            object_client \u003d cls.object_client"},{"line_number":142,"context_line":"        delete_containers(cls.containers, container_client, object_client)"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"    def assertHeaders(self, resp, target, method):"},{"line_number":145,"context_line":"        \"\"\"Check the existence and the format of response headers\"\"\""},{"line_number":146,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"796849a8_26c29887","line":143,"range":{"start_line":143,"start_character":0,"end_line":143,"end_character":0},"updated":"2021-08-03 21:05:13.000000000","message":"here after L142 we can clear the cls.containers list.\n\n     cls.containers \u003d []","commit_id":"c026cf10849f8b6ff66f3f8b88659c8d07a2d156"},{"author":{"_account_id":33282,"name":"Huy Tran","email":"huy.q.tran@ericsson.com","username":"ht095u"},"change_message_id":"5024aebe9ff9478a4d0cb64daf2c3b9fff0597cd","unresolved":false,"context_lines":[{"line_number":140,"context_line":"        if object_client is None:"},{"line_number":141,"context_line":"            object_client \u003d cls.object_client"},{"line_number":142,"context_line":"        delete_containers(cls.containers, container_client, object_client)"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"    def assertHeaders(self, resp, target, method):"},{"line_number":145,"context_line":"        \"\"\"Check the existence and the format of response headers\"\"\""},{"line_number":146,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"7d3508f4_7cb69c6e","line":143,"range":{"start_line":143,"start_character":0,"end_line":143,"end_character":0},"in_reply_to":"1d1416b3_cbe2b4f5","updated":"2021-08-04 18:12:30.000000000","message":"Updated PS as suggested.","commit_id":"c026cf10849f8b6ff66f3f8b88659c8d07a2d156"},{"author":{"_account_id":8556,"name":"Ghanshyam Maan","display_name":"Ghanshyam Maan","email":"gmaan.os14@gmail.com","username":"ghanshyam"},"change_message_id":"bd0f6b9647e58710ce1a4c508c85154f8894cac7","unresolved":true,"context_lines":[{"line_number":140,"context_line":"        if object_client is None:"},{"line_number":141,"context_line":"            object_client \u003d cls.object_client"},{"line_number":142,"context_line":"        delete_containers(cls.containers, container_client, object_client)"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"    def assertHeaders(self, resp, target, method):"},{"line_number":145,"context_line":"        \"\"\"Check the existence and the format of response headers\"\"\""},{"line_number":146,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"1d1416b3_cbe2b4f5","line":143,"range":{"start_line":143,"start_character":0,"end_line":143,"end_character":0},"in_reply_to":"59534166_1bcd4167","updated":"2021-08-03 22:46:59.000000000","message":"ok, so actual issue is how classes are cleaning up the container. The delete operation on container (this method) is supposed to delete all the crated containers at the test class level tearDownClass not at the test level tearDown() which is what few test classes doing and you are seeing the issue\n\n1 https://github.com/openstack/tempest/blob/7e96c8e854386f43604ad098a6ec7606ee676145/tempest/api/object_storage/test_container_services.py#L24\n\n2 https://github.com/openstack/tempest/blob/7e96c8e854386f43604ad098a6ec7606ee676145/tempest/api/object_storage/test_container_acl.py#L34\n\n3 https://github.com/openstack/tempest/blob/7e96c8e854386f43604ad098a6ec7606ee676145/tempest/api/object_storage/test_container_quotas.py#L49\n\n4 https://github.com/openstack/tempest/blob/7e96c8e854386f43604ad098a6ec7606ee676145/tempest/api/object_storage/test_container_acl_negative.py#L45\n\nWe need to change the cleanup from tearDown to resource_cleanup(this is called in tearDownClass), like below\n\n    def resource_cleanup(cls):\n        cls.delete_containers()\n        super(ContainerTest, cls).resource_cleanup()\n\nlike done in https://github.com/openstack/tempest/blob/7e96c8e854386f43604ad098a6ec7606ee676145/tempest/api/object_storage/test_account_quotas_negative.py#L74\n\nIf you change it in above 4 test class then it will fix the issue with cleanup at class level.","commit_id":"c026cf10849f8b6ff66f3f8b88659c8d07a2d156"},{"author":{"_account_id":33282,"name":"Huy Tran","email":"huy.q.tran@ericsson.com","username":"ht095u"},"change_message_id":"148fcae311b80ef3f76517c77f74a026e2749736","unresolved":true,"context_lines":[{"line_number":140,"context_line":"        if object_client is None:"},{"line_number":141,"context_line":"            object_client \u003d cls.object_client"},{"line_number":142,"context_line":"        delete_containers(cls.containers, container_client, object_client)"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"    def assertHeaders(self, resp, target, method):"},{"line_number":145,"context_line":"        \"\"\"Check the existence and the format of response headers\"\"\""},{"line_number":146,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"59534166_1bcd4167","line":143,"range":{"start_line":143,"start_character":0,"end_line":143,"end_character":0},"in_reply_to":"796849a8_26c29887","updated":"2021-08-03 21:35:49.000000000","message":"Thanks for the comments and suggestion. As mentioned in IRC channel, clearing the list of container names here won\u0027t solve the race condition, and the change would become unnecessary as the list is being re-initialized. \nCan we make a copy of the list, clear the list, and pass the copy of the list to delete_containers()?\n\nLogs have been provided in the bug to show the race condition where second test/method delete the container which was created by the first test/method.","commit_id":"c026cf10849f8b6ff66f3f8b88659c8d07a2d156"}]}