)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"5e2225962c589d044e57f2d123714350c4a622cb","unresolved":false,"context_lines":[{"line_number":12,"context_line":"deleted."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I2179a1e6e27a636bfca448fa18481eb11e64e6da"},{"line_number":15,"context_line":"Closes-Bug: #1518431"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"7a740942_2f0a9d20","line":15,"updated":"2015-12-14 15:57:05.000000000","message":"I tried to debug the issue. It seems there is no race condition at all.\nThe problem is in CGI framework used. By default mod_fast_cgi doesn\u0027t support chunked requests, so Apache rejects the requests because there is no length specified.\nDo you have such cases reproduced for RadosGW under others CGI (for example mod_proxy_fcgi)?","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"93167da7e7a7da6e25c6346ee18a212a2d4d31d2","unresolved":false,"context_lines":[{"line_number":12,"context_line":"deleted."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I2179a1e6e27a636bfca448fa18481eb11e64e6da"},{"line_number":15,"context_line":"Closes-Bug: #1518431"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"5a710552_55f2ee9b","line":15,"updated":"2015-12-15 01:55:21.000000000","message":"This issue has been reported by a couple of users at least. There\u0027s a report in launchpad. It\u0027s hard to reproduce manually and even harder to test.","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":8122,"name":"Cyril Roelandt","email":"cyril@redhat.com","username":"cyril.roelandt.enovance"},"change_message_id":"3fee8b60796d69893506266c0c547bbcc173fe4b","unresolved":false,"context_lines":[{"line_number":12,"context_line":"deleted."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I2179a1e6e27a636bfca448fa18481eb11e64e6da"},{"line_number":15,"context_line":"Closes-Bug: #1518431"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"7a740942_84a37342","line":15,"in_reply_to":"7a740942_2f0a9d20","updated":"2015-12-14 16:39:58.000000000","message":"No, I could not reproduce the issue, but that is what race conditions are about :) \n\nIn the code I fixed, I believe it makes sense to make sure that Swift is done creating the object before deleting it. WDYT?","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"78c062aa6931b7f794b1108c5068f0b748ccb5b7","unresolved":false,"context_lines":[{"line_number":12,"context_line":"deleted."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I2179a1e6e27a636bfca448fa18481eb11e64e6da"},{"line_number":15,"context_line":"Closes-Bug: #1518431"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"5a710552_7befe1b0","line":15,"in_reply_to":"7a740942_84a37342","updated":"2015-12-14 22:49:53.000000000","message":"So I am not Swift expert but why I am wondering why do we need to fix an issue what we can\u0027t reproduce(or it is never reproduced).","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"}],"glance_store/_drivers/swift/store.py":[{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"5d89f93d03b3571375eb4f651cb9fca7d2da6042","unresolved":false,"context_lines":[{"line_number":174,"context_line":"        else:"},{"line_number":175,"context_line":"            break"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"def _wait_for_object_creation(connection, container, name):"},{"line_number":179,"context_line":"    sleep_seconds \u003d 5"},{"line_number":180,"context_line":"    max_retries \u003d 2"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_6aaed099","line":177,"updated":"2015-12-10 23:45:53.000000000","message":"There\u0027s a library called \"retrying\", which is already in global-requirements, that should be able to help with this. It has utilities for things like this one you implemented.\n\nAny chance we can use that instead?\n\nhttps://pypi.python.org/pypi/retrying","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"c7550481b6c56ed1d1f7a57322f57ea7c367ecd1","unresolved":false,"context_lines":[{"line_number":174,"context_line":"        else:"},{"line_number":175,"context_line":"            break"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"def _wait_for_object_creation(connection, container, name):"},{"line_number":179,"context_line":"    sleep_seconds \u003d 5"},{"line_number":180,"context_line":"    max_retries \u003d 2"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_e1141045","line":177,"in_reply_to":"7a740942_6aaed099","updated":"2015-12-11 10:45:32.000000000","message":"++","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":8122,"name":"Cyril Roelandt","email":"cyril@redhat.com","username":"cyril.roelandt.enovance"},"change_message_id":"603916caaf752d4d4401eb8f183e028111b08696","unresolved":false,"context_lines":[{"line_number":174,"context_line":"        else:"},{"line_number":175,"context_line":"            break"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"def _wait_for_object_creation(connection, container, name):"},{"line_number":179,"context_line":"    sleep_seconds \u003d 5"},{"line_number":180,"context_line":"    max_retries \u003d 2"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_93ef9e1d","line":177,"in_reply_to":"7a740942_6aaed099","updated":"2015-12-11 14:21:03.000000000","message":"Oh yeah, I remember this library, jd wrote a blog article about it! I\u0027ll use it.","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"fbf6232ab93504fc6baebfd0d494081a8febb390","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"def _wait_for_object_creation(connection, container, name):"},{"line_number":179,"context_line":"    sleep_seconds \u003d 5"},{"line_number":180,"context_line":"    max_retries \u003d 2"},{"line_number":181,"context_line":"    retries \u003d 0"},{"line_number":182,"context_line":"    while retries \u003c max_retries:"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_4a99946d","line":179,"updated":"2015-12-10 23:46:47.000000000","message":"is 5s too high for a first attempt? A 1 second wait should be enough in most cases, I believe.","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":8122,"name":"Cyril Roelandt","email":"cyril@redhat.com","username":"cyril.roelandt.enovance"},"change_message_id":"603916caaf752d4d4401eb8f183e028111b08696","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"def _wait_for_object_creation(connection, container, name):"},{"line_number":179,"context_line":"    sleep_seconds \u003d 5"},{"line_number":180,"context_line":"    max_retries \u003d 2"},{"line_number":181,"context_line":"    retries \u003d 0"},{"line_number":182,"context_line":"    while retries \u003c max_retries:"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_1395ee85","line":179,"in_reply_to":"7a740942_4a99946d","updated":"2015-12-11 14:21:03.000000000","message":"Yeah, got a bit crazy there :)","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"c7550481b6c56ed1d1f7a57322f57ea7c367ecd1","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"def _wait_for_object_creation(connection, container, name):"},{"line_number":179,"context_line":"    sleep_seconds \u003d 5"},{"line_number":180,"context_line":"    max_retries \u003d 2"},{"line_number":181,"context_line":"    retries \u003d 0"},{"line_number":182,"context_line":"    while retries \u003c max_retries:"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_1e0f3d3a","line":179,"in_reply_to":"7a740942_4a99946d","updated":"2015-12-11 10:45:32.000000000","message":"agree with Flavio here.","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"c7550481b6c56ed1d1f7a57322f57ea7c367ecd1","unresolved":false,"context_lines":[{"line_number":184,"context_line":"            connection.get_object(container, name)"},{"line_number":185,"context_line":"        except swiftclient.ClientException as e:"},{"line_number":186,"context_line":"            if e.http_status \u003d\u003d http_client.NOT_FOUND:"},{"line_number":187,"context_line":"                retries +\u003d 1"},{"line_number":188,"context_line":"                time.sleep(sleep_seconds)"},{"line_number":189,"context_line":"            else:"},{"line_number":190,"context_line":"                raise"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_1e389ded","line":187,"updated":"2015-12-11 10:45:32.000000000","message":"Some debug messages would be useful here IMO.","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"c7550481b6c56ed1d1f7a57322f57ea7c367ecd1","unresolved":false,"context_lines":[{"line_number":188,"context_line":"                time.sleep(sleep_seconds)"},{"line_number":189,"context_line":"            else:"},{"line_number":190,"context_line":"                raise"},{"line_number":191,"context_line":"        except Exception:"},{"line_number":192,"context_line":"            raise"},{"line_number":193,"context_line":"        else:"},{"line_number":194,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_deb1b590","line":191,"updated":"2015-12-11 10:45:32.000000000","message":"I think this except is redundant: you can delete it because it just raises an exception.","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"c7550481b6c56ed1d1f7a57322f57ea7c367ecd1","unresolved":false,"context_lines":[{"line_number":191,"context_line":"        except Exception:"},{"line_number":192,"context_line":"            raise"},{"line_number":193,"context_line":"        else:"},{"line_number":194,"context_line":"            return"},{"line_number":195,"context_line":"    raise http_client.NOT_FOUND"},{"line_number":196,"context_line":""},{"line_number":197,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_8153cc16","line":194,"updated":"2015-12-11 10:45:32.000000000","message":"you can return immediately after connection.get_object. So don\u0027t need this else statement.","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"c7550481b6c56ed1d1f7a57322f57ea7c367ecd1","unresolved":false,"context_lines":[{"line_number":192,"context_line":"            raise"},{"line_number":193,"context_line":"        else:"},{"line_number":194,"context_line":"            return"},{"line_number":195,"context_line":"    raise http_client.NOT_FOUND"},{"line_number":196,"context_line":""},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"class StoreLocation(location.StoreLocation):"}],"source_content_type":"text/x-python","patch_set":3,"id":"7a740942_3e46b97b","line":195,"updated":"2015-12-11 10:45:32.000000000","message":"Not sure we should generate http_client error here.\ni believe there should glance_store exception.","commit_id":"ce1e492b8b983dc4da5352aaebd2449b57ad6099"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"02a73321f1d8f124bec8f628dc06406f122aa42e","unresolved":false,"context_lines":[{"line_number":22,"context_line":"from oslo_config import cfg"},{"line_number":23,"context_line":"from oslo_utils import excutils"},{"line_number":24,"context_line":"from oslo_utils import units"},{"line_number":25,"context_line":"from retrying import retry"},{"line_number":26,"context_line":"import six"},{"line_number":27,"context_line":"from six.moves import http_client"},{"line_number":28,"context_line":"from six.moves import urllib"}],"source_content_type":"text/x-python","patch_set":4,"id":"7a740942_740e958d","line":25,"updated":"2015-12-11 14:41:24.000000000","message":"Do we have this in glance_store requirements?","commit_id":"fff616b9ee469e695cf2c750a569061c0a7ece5f"},{"author":{"_account_id":8122,"name":"Cyril Roelandt","email":"cyril@redhat.com","username":"cyril.roelandt.enovance"},"change_message_id":"dfd6171d955d1ae30655715a5e9d740a6d337341","unresolved":false,"context_lines":[{"line_number":22,"context_line":"from oslo_config import cfg"},{"line_number":23,"context_line":"from oslo_utils import excutils"},{"line_number":24,"context_line":"from oslo_utils import units"},{"line_number":25,"context_line":"from retrying import retry"},{"line_number":26,"context_line":"import six"},{"line_number":27,"context_line":"from six.moves import http_client"},{"line_number":28,"context_line":"from six.moves import urllib"}],"source_content_type":"text/x-python","patch_set":4,"id":"7a740942_549cb90d","line":25,"in_reply_to":"7a740942_740e958d","updated":"2015-12-11 14:43:06.000000000","message":"Good catch! I misread Flavio\u0027s comment, who was saying that it was already in the *global* requirements.","commit_id":"fff616b9ee469e695cf2c750a569061c0a7ece5f"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"5e2225962c589d044e57f2d123714350c4a622cb","unresolved":false,"context_lines":[{"line_number":597,"context_line":"                                LOG.debug(\"Could not find %s\" % chunk_name)"},{"line_number":598,"context_line":"                            msg \u003d _(\"Cannot find chunk to delete\")"},{"line_number":599,"context_line":"                            raise glance_store.exceptions.NotFound(message\u003dmsg)"},{"line_number":600,"context_line":"                        connection.delete_object(location.container,"},{"line_number":601,"context_line":"                                                 chunk_name)"},{"line_number":602,"context_line":"                        break"},{"line_number":603,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_34de743e","line":600,"updated":"2015-12-14 15:57:05.000000000","message":"I think that we need to wrap connection.delete_object with retry instead of executing redundant connection.get_object.","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"93167da7e7a7da6e25c6346ee18a212a2d4d31d2","unresolved":false,"context_lines":[{"line_number":597,"context_line":"                                LOG.debug(\"Could not find %s\" % chunk_name)"},{"line_number":598,"context_line":"                            msg \u003d _(\"Cannot find chunk to delete\")"},{"line_number":599,"context_line":"                            raise glance_store.exceptions.NotFound(message\u003dmsg)"},{"line_number":600,"context_line":"                        connection.delete_object(location.container,"},{"line_number":601,"context_line":"                                                 chunk_name)"},{"line_number":602,"context_line":"                        break"},{"line_number":603,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"5a710552_75efeab0","line":600,"updated":"2015-12-15 01:55:21.000000000","message":"mmmh, I think I agree with Kairat here. I guess retrying deletes would make more sense as that\u0027s what is failing. If it fails 3 times then there\u0027s something worse happening. In addition, retrying deletes would make this change smaller and it won\u0027t add anything else to the current behavior/process.","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":8122,"name":"Cyril Roelandt","email":"cyril@redhat.com","username":"cyril.roelandt.enovance"},"change_message_id":"45f1efbe7c37f9a6dfc22c040aeb11e9b46dcf7f","unresolved":false,"context_lines":[{"line_number":597,"context_line":"                                LOG.debug(\"Could not find %s\" % chunk_name)"},{"line_number":598,"context_line":"                            msg \u003d _(\"Cannot find chunk to delete\")"},{"line_number":599,"context_line":"                            raise glance_store.exceptions.NotFound(message\u003dmsg)"},{"line_number":600,"context_line":"                        connection.delete_object(location.container,"},{"line_number":601,"context_line":"                                                 chunk_name)"},{"line_number":602,"context_line":"                        break"},{"line_number":603,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"5a710552_9d938afa","line":600,"in_reply_to":"5a710552_75efeab0","updated":"2015-12-15 11:16:46.000000000","message":"Oh right, I misread Kairat\u0027s comment.","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"78c062aa6931b7f794b1108c5068f0b748ccb5b7","unresolved":false,"context_lines":[{"line_number":597,"context_line":"                                LOG.debug(\"Could not find %s\" % chunk_name)"},{"line_number":598,"context_line":"                            msg \u003d _(\"Cannot find chunk to delete\")"},{"line_number":599,"context_line":"                            raise glance_store.exceptions.NotFound(message\u003dmsg)"},{"line_number":600,"context_line":"                        connection.delete_object(location.container,"},{"line_number":601,"context_line":"                                                 chunk_name)"},{"line_number":602,"context_line":"                        break"},{"line_number":603,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"5a710552_9b772dd4","line":600,"in_reply_to":"7a740942_04d223d1","updated":"2015-12-14 22:49:53.000000000","message":"It just more clear to implement retry on delete instead of doing get requests. I think that the main trouble here is that we don\u0027t know which kind of exception can be generated in this race condition(and the problem is the same: issue is never(or rarely) reproduced).\nIf we would know this then there would be no problem to implement retry on delete. The logic would be straight: try to delete until exception is generated.","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":8122,"name":"Cyril Roelandt","email":"cyril@redhat.com","username":"cyril.roelandt.enovance"},"change_message_id":"3fee8b60796d69893506266c0c547bbcc173fe4b","unresolved":false,"context_lines":[{"line_number":597,"context_line":"                                LOG.debug(\"Could not find %s\" % chunk_name)"},{"line_number":598,"context_line":"                            msg \u003d _(\"Cannot find chunk to delete\")"},{"line_number":599,"context_line":"                            raise glance_store.exceptions.NotFound(message\u003dmsg)"},{"line_number":600,"context_line":"                        connection.delete_object(location.container,"},{"line_number":601,"context_line":"                                                 chunk_name)"},{"line_number":602,"context_line":"                        break"},{"line_number":603,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_04d223d1","line":600,"in_reply_to":"7a740942_34de743e","updated":"2015-12-14 16:39:58.000000000","message":"That would make sense here, but I\u0027m not sure we\u0027re always going to be in a case like this - even though making one \"get\" request should not be too much of an overhead.","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"}],"glance_store/tests/unit/test_swift_store.py":[{"author":{"_account_id":14676,"name":"Kairat Kushaev","email":"kkushaev@mirantis.com","username":"kairat_kushaev"},"change_message_id":"5e2225962c589d044e57f2d123714350c4a622cb","unresolved":false,"context_lines":[{"line_number":744,"context_line":"        # where the image_size is specified in advance (there\u0027s no zero chunk"},{"line_number":745,"context_line":"        # in that case)."},{"line_number":746,"context_line":"        self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7)"},{"line_number":747,"context_line":"        self.assertEqual(SWIFT_GET_OBJECT_CALLS, 1)"},{"line_number":748,"context_line":""},{"line_number":749,"context_line":"        loc \u003d location.get_location_from_uri(expected_location, conf\u003dself.conf)"},{"line_number":750,"context_line":"        (new_image_swift, new_image_size) \u003d self.store.get(loc)"}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_299a9d62","line":747,"updated":"2015-12-14 15:57:05.000000000","message":"AFAIU I don\u0027t see the case with race condition itself.\nCould you please add that? IMO it improves quality cause extends list of test cases.","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"},{"author":{"_account_id":8122,"name":"Cyril Roelandt","email":"cyril@redhat.com","username":"cyril.roelandt.enovance"},"change_message_id":"3fee8b60796d69893506266c0c547bbcc173fe4b","unresolved":false,"context_lines":[{"line_number":744,"context_line":"        # where the image_size is specified in advance (there\u0027s no zero chunk"},{"line_number":745,"context_line":"        # in that case)."},{"line_number":746,"context_line":"        self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7)"},{"line_number":747,"context_line":"        self.assertEqual(SWIFT_GET_OBJECT_CALLS, 1)"},{"line_number":748,"context_line":""},{"line_number":749,"context_line":"        loc \u003d location.get_location_from_uri(expected_location, conf\u003dself.conf)"},{"line_number":750,"context_line":"        (new_image_swift, new_image_size) \u003d self.store.get(loc)"}],"source_content_type":"text/x-python","patch_set":6,"id":"7a740942_444fcb91","line":747,"in_reply_to":"7a740942_299a9d62","updated":"2015-12-14 16:39:58.000000000","message":"Really not sure how to fake a race condition in the tests. Any pointers?","commit_id":"8e53a2620dac8f6231abf792bbabc5e47bddee78"}]}