)]}'
{"tempest/common/dynamic_creds.py":[{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"8552523216c4f4892c8327e4dbb3043fd998e41f","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_27f24107","line":164,"updated":"2016-06-27 15:06:22.000000000","message":"Personally I\u0027d prefer to see a check done upfront for the presence of the role to make sure we don\u0027t even run this block if we don\u0027t need to.","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"866c30341936b112a7911395de8ce960d50f6d26","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_9dbd4405","line":164,"in_reply_to":"3aaa91ec_27f24107","updated":"2016-06-27 15:32:03.000000000","message":"Unfortunately I\u0027m not sure that is going to fully resolve the issue we keep seeing in the puppet run of tempest. I believe it to be a race condition and the under lying issue is the fact that this is not test specific as keystone will insert this as Member-\u003c\u003cnull\u003e\u003e because there is no domain ID provided.  So even if we did a check, it could still be possible to hit this conflict based on timing.  I can look into adding a check condition here as well.","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"9bd2bb3991c8f2a5a9eaa92f32279052f2115936","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_d8107aae","line":164,"in_reply_to":"3aaa91ec_58f80ac5","updated":"2016-06-27 15:49:17.000000000","message":"yes it started late last week as it started showing up on the setUpClass for tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON in our puppet ci.  I only ever saw it on that class and I tracked down the code in keystone and that hasn\u0027t changed in forever. So something else in the ordering is causing it. That being said, this creation of a role can only occur once due to the lack of a domain id and if it happens it will return a conflict.","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"dd760b93758ac8128ebb63d0e44d34a59aae434e","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_247238ae","line":164,"in_reply_to":"3aaa91ec_7e016f39","updated":"2016-06-27 16:20:16.000000000","message":"If you look at the tempest log for one of the failures you can see the two threads creating the role 2ms apart from each other.\n\nhttp://logs.openstack.org/28/332928/5/check/gate-puppet-openstack-integration-3-scenario001-tempest-centos-7-nv/84f6841/console.html.gz#_2016-06-24_00_49_21_334964\n\nhttp://logs.openstack.org/28/332928/5/check/gate-puppet-openstack-integration-3-scenario001-tempest-centos-7-nv/84f6841/logs/tempest.txt.gz#_2016-06-24_00_49_21_320\n\nhttp://logs.openstack.org/28/332928/5/check/gate-puppet-openstack-integration-3-scenario001-tempest-centos-7-nv/84f6841/logs/tempest.txt.gz#_2016-06-24_00_49_21_322\n\nSo something has sped up the initial setup class so they a run at almost the exact same time. Perhaps that\u0027s what changed where previously the threads were off by enough to not conflict with each other. It does appear to be happening on the first test through the tempest run.","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"0dc5e22a292afe9c790b841c417ba7d1eebbf4d3","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_58dc8a74","line":164,"in_reply_to":"3aaa91ec_9dbd4405","updated":"2016-06-27 15:42:40.000000000","message":"Actually I checked, and this call does do a check about the role existing. So we\u0027re hitting a race condition.  http://git.openstack.org/cgit/openstack/tempest/tree/tempest/common/cred_client.py#n65\n\nSo I believe this to be the correct fix.","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"22489c3599244cc7ef2b7b570aaf0240e55dd2ac","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_58f80ac5","line":164,"in_reply_to":"3aaa91ec_9dbd4405","updated":"2016-06-27 15:44:36.000000000","message":"wait so you\u0027re saying this is a race between tests\u0027 setupclasses. So that when 2 workers run setupclass at the same time there is a potential conflict. If that\u0027s the case then how did this ever work correctly? That race should be very apparent when running with \u003e1 worker, it would have started showing up almost immediately. I\u0027m pretty sure the puppet job failure rates don\u0027t support that being the case.\nSomething had to have changed to cause this to start showing up now.","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"02fef97796161d5cd8385cb0f7c0192a0b9e1eef","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_7e016f39","line":164,"in_reply_to":"3aaa91ec_d8107aae","updated":"2016-06-27 16:03:45.000000000","message":"It was happening on the puppet ci in the tempest reviews last week on some other tests but i guess since it\u0027s non-voting no one noticed.  Looks like some of the other classes had similar failures as well.\n\nhttps://review.openstack.org/#/c/333553/\nhttps://review.openstack.org/#/c/332928/\nhttps://review.openstack.org/#/c/333103/\nhttps://review.openstack.org/#/c/333658/\nhttps://review.openstack.org/#/c/333793/\nhttps://review.openstack.org/#/c/310013/\nhttps://review.openstack.org/#/c/333074/","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"63c0f168c276c7890086a655c3dde10f64f452a5","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        if self.identity_version \u003d\u003d \u0027v3\u0027 and not role_assigned:"},{"line_number":162,"context_line":"            try:"},{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_de77c39d","line":164,"in_reply_to":"3aaa91ec_d8107aae","updated":"2016-06-27 15:56:04.000000000","message":"ok, so the existence check will cover the rarity aspect here. (if that wasn\u0027t there we\u0027d hit this all the time) I\u0027m still not sure what would have to peak recently. You can basically assume with \u003e1 worker setupclass will always be running at the same time. The race between You can look at a tempest log for lines like:\n\nhttp://logs.openstack.org/09/334209/4/check/gate-puppet-openstack-integration-3-scenario001-tempest-centos-7-nv/2176015/logs/tempest.txt.gz#_2016-06-27_01_24_55_767\n\nto see the rate at which dynamic_creds is being called. I would have expected a race like this should have been more obvious","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":5196,"name":"Matthew Treinish","email":"mtreinish@kortar.org","username":"treinish"},"change_message_id":"8552523216c4f4892c8327e4dbb3043fd998e41f","unresolved":false,"context_lines":[{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"        creds \u003d self.creds_client.get_credentials(user, project, user_password)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_e7ce79b6","line":166,"updated":"2016-06-27 15:06:22.000000000","message":"We probably should log something (just like we do on delete checks when we get a 404)","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"4af706dc732b9eb3fe7c579dca9ac2ec0c58be03","unresolved":false,"context_lines":[{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"        creds \u003d self.creds_client.get_credentials(user, project, user_password)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_529f4250","line":166,"in_reply_to":"3aaa91ec_3847fe1e","updated":"2016-06-27 16:59:35.000000000","message":"Done","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"0dc5e22a292afe9c790b841c417ba7d1eebbf4d3","unresolved":false,"context_lines":[{"line_number":163,"context_line":"                self.creds_client.create_user_role(\u0027Member\u0027)"},{"line_number":164,"context_line":"            except lib_exc.Conflict:"},{"line_number":165,"context_line":"                # if the role is already created, ignore the exception"},{"line_number":166,"context_line":"                pass"},{"line_number":167,"context_line":"            self.creds_client.assign_user_role(user, project, \u0027Member\u0027)"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"        creds \u003d self.creds_client.get_credentials(user, project, user_password)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3aaa91ec_3847fe1e","line":166,"in_reply_to":"3aaa91ec_e7ce79b6","updated":"2016-06-27 15:42:40.000000000","message":"I\u0027ll add a log line","commit_id":"5c8819d7e036119345407ed69f9213e79edf20d6"}]}