)]}'
{"id":"openstack%2Fpython-swiftclient~551956","triplet_id":"openstack%2Fpython-swiftclient~master~I3470b56e3f9cf9cdb8c2fc2a94b2c551927a3440","project":"openstack/python-swiftclient","branch":"master","topic":"force-auth-retry","hashtags":[],"change_id":"I3470b56e3f9cf9cdb8c2fc2a94b2c551927a3440","subject":"Add force auth retry mode in swiftclient","status":"MERGED","created":"2018-03-12 10:52:09.000000000","updated":"2018-03-19 02:04:02.000000000","submitted":"2018-03-19 02:04:02.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":5,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"551956-1521425042498-d6052c5b","meta_rev_id":"058a82554cf834f3ff062dd0f370b920e4bb5094","_number":551956,"virtual_id_number":551956,"owner":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"date":"2018-03-13 03:30:51.000000000","_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},{"value":0,"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"value":0,"_account_id":23317,"name":"ShangXiao","email":"shangxiaobj@inspur.com","username":"ShangXiao"},{"value":0,"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":2,"date":"2018-03-19 02:04:02.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"all":[{"value":0,"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},{"value":2,"date":"2018-03-19 00:27:09.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"value":1,"date":"2018-03-19 00:58:08.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":23317,"name":"ShangXiao","email":"shangxiaobj@inspur.com","username":"ShangXiao"},{"value":2,"date":"2018-03-13 23:10:19.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"all":[{"value":0,"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},{"value":1,"date":"2018-03-19 00:27:09.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"value":0,"_account_id":23317,"name":"ShangXiao","email":"shangxiaobj@inspur.com","username":"ShangXiao"},{"value":0,"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":23317,"name":"ShangXiao","email":"shangxiaobj@inspur.com","username":"ShangXiao"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2018-03-13 23:10:19.000000000","updated_by":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"reviewer":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"state":"REVIEWER"},{"updated":"2018-03-19 00:27:09.000000000","updated_by":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"reviewer":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"state":"REVIEWER"},{"updated":"2018-03-19 00:58:08.000000000","updated_by":{"_account_id":23317,"name":"ShangXiao","email":"shangxiaobj@inspur.com","username":"ShangXiao"},"reviewer":{"_account_id":23317,"name":"ShangXiao","email":"shangxiaobj@inspur.com","username":"ShangXiao"},"state":"REVIEWER"},{"updated":"2018-03-19 02:04:02.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"addf5fd6e213b4f9b1d9ef6f71203556519dd140","author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"date":"2018-03-12 10:52:09.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"4b85e58d80abd5d6ad52d074e48b04fc60e60717","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2018-03-12 12:07:19.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-pep8 http://logs.openstack.org/56/551956/1/check/openstack-tox-pep8/bd84bea/ : SUCCESS in 2m 01s\n- openstack-tox-py27 http://logs.openstack.org/56/551956/1/check/openstack-tox-py27/2476cf7/ : SUCCESS in 2m 38s\n- openstack-tox-py35 http://logs.openstack.org/56/551956/1/check/openstack-tox-py35/de48e82/ : SUCCESS in 2m 39s\n- build-openstack-sphinx-docs http://logs.openstack.org/56/551956/1/check/build-openstack-sphinx-docs/bef495d/html/ : SUCCESS in 4m 24s\n- openstack-tox-pypy http://logs.openstack.org/56/551956/1/check/openstack-tox-pypy/e50bea9/ : SUCCESS in 3m 08s (non-voting)\n- legacy-tempest-dsvm-neutron-src http://logs.openstack.org/56/551956/1/check/legacy-tempest-dsvm-neutron-src/7d0b0e1/ : SUCCESS in 1h 12m 50s\n- legacy-swift-dsvm-functional http://logs.openstack.org/56/551956/1/check/legacy-swift-dsvm-functional/5f1ad4a/ : SUCCESS in 26m 13s\n- legacy-swift-dsvm-functional-identity-v3-only http://logs.openstack.org/56/551956/1/check/legacy-swift-dsvm-functional-identity-v3-only/e707457/ : SUCCESS in 20m 55s (non-voting)\n- legacy-swiftclient-dsvm-functional http://logs.openstack.org/56/551956/1/check/legacy-swiftclient-dsvm-functional/dca4022/ : SUCCESS in 17m 49s\n- legacy-swiftclient-dsvm-functional-identity-v3-only http://logs.openstack.org/56/551956/1/check/legacy-swiftclient-dsvm-functional-identity-v3-only/92d49be/ : SUCCESS in 14m 19s (non-voting)\n- openstack-tox-py36 http://logs.openstack.org/56/551956/1/check/openstack-tox-py36/304b0e2/ : SUCCESS in 5m 11s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"ba83e5cbf13e32045f454d5d302a446ea1899da9","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2018-03-13 03:22:46.000000000","message":"Patch Set 1: Code-Review-1\n\n(2 comments)\n\nThere seems to be a missing link in the commit message, which might be important.\n\nOther then that, it sounds good. Though I don\u0027t have something like an nginx environment to play with it at hand. I\u0027ll see what I can do about building one into one of my SAIOs though.\n\nTests are good though which shows how it\u0027s suppose to behave. Of course this\u0027ll make the client more noisy in regards to talking to the auth server, but it\u0027s opt in, so does help people in the particular usecase.","accounts_in_message":[],"_revision_number":1},{"id":"035b2562e25c739a562c1951b7fa3ce433509726","author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"date":"2018-03-13 03:30:13.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"bd497ff8a74a171f95503e2b3d7decdca6514ff9","author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"date":"2018-03-13 03:30:51.000000000","message":"Patch Set 2:\n\n(2 comments)","accounts_in_message":[],"_revision_number":2},{"id":"fb79018cd5b8dfaac265e2ab6b1e27f76c19f1b7","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2018-03-13 04:44:36.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-pep8 http://logs.openstack.org/56/551956/2/check/openstack-tox-pep8/b390a98/ : SUCCESS in 2m 16s\n- openstack-tox-py27 http://logs.openstack.org/56/551956/2/check/openstack-tox-py27/528d380/ : SUCCESS in 2m 40s\n- openstack-tox-py35 http://logs.openstack.org/56/551956/2/check/openstack-tox-py35/16044e7/ : SUCCESS in 2m 38s\n- build-openstack-sphinx-docs http://logs.openstack.org/56/551956/2/check/build-openstack-sphinx-docs/146b88b/html/ : SUCCESS in 2m 32s\n- openstack-tox-pypy http://logs.openstack.org/56/551956/2/check/openstack-tox-pypy/4b13088/ : SUCCESS in 2m 58s (non-voting)\n- legacy-tempest-dsvm-neutron-src http://logs.openstack.org/56/551956/2/check/legacy-tempest-dsvm-neutron-src/6c8298f/ : SUCCESS in 1h 09m 58s\n- legacy-swift-dsvm-functional http://logs.openstack.org/56/551956/2/check/legacy-swift-dsvm-functional/126697e/ : SUCCESS in 21m 37s\n- legacy-swift-dsvm-functional-identity-v3-only http://logs.openstack.org/56/551956/2/check/legacy-swift-dsvm-functional-identity-v3-only/9eccf9a/ : SUCCESS in 21m 23s (non-voting)\n- legacy-swiftclient-dsvm-functional http://logs.openstack.org/56/551956/2/check/legacy-swiftclient-dsvm-functional/c67b0c5/ : SUCCESS in 13m 24s\n- legacy-swiftclient-dsvm-functional-identity-v3-only http://logs.openstack.org/56/551956/2/check/legacy-swiftclient-dsvm-functional-identity-v3-only/236077d/ : SUCCESS in 15m 18s (non-voting)\n- openstack-tox-py36 http://logs.openstack.org/56/551956/2/check/openstack-tox-py36/c5a524d/ : SUCCESS in 3m 48s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"18693b563d2359dab3abf444f31e20c81bfc3e94","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2018-03-13 23:10:19.000000000","message":"Patch Set 2: Code-Review+2\n\n(1 comment)\n\nseems liked an awkward mitigation given the commit message, but I could imagine some kinds of errors might want to reset the storage_url so ... maybe?\n\nthe code looks fine, and the change seems to do what it says on the tin","accounts_in_message":[],"_revision_number":2},{"id":"169e1d164c61c058de3ba87b5d02972749051c33","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2018-03-19 00:27:09.000000000","message":"Patch Set 2: Workflow+1 Code-Review+2\n\nI\u0027d rather have this fixed via the 100-continue heartbeat. However I realise we have issues there, and meanwhile we have this very specific retry bug biting people.\n\nI\u0027ve tested it on my SAIO and does what it says. So thanks Kota.","accounts_in_message":[],"_revision_number":2},{"id":"196a9b2bb915893168867f32b3a0e0586db089e4","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2018-03-19 00:27:21.000000000","message":"Patch Set 2: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":2},{"id":"90e513fbb446e16e220c383bd067385e757318de","author":{"_account_id":23317,"name":"ShangXiao","email":"shangxiaobj@inspur.com","username":"ShangXiao"},"date":"2018-03-19 00:58:08.000000000","message":"Patch Set 2: Code-Review+1","accounts_in_message":[],"_revision_number":2},{"id":"fd75e499b939d2b9c6b54c4d479a701737de79e3","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2018-03-19 02:04:02.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":2},{"id":"0e21fff38200bd10dc6e43c89399ebf91cbc9101","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2018-03-19 02:04:02.000000000","message":"Patch Set 2: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- openstack-tox-pep8 http://logs.openstack.org/56/551956/2/gate/openstack-tox-pep8/c706f33/ : SUCCESS in 2m 28s\n- openstack-tox-py27 http://logs.openstack.org/56/551956/2/gate/openstack-tox-py27/900656f/ : SUCCESS in 2m 28s\n- openstack-tox-py35 http://logs.openstack.org/56/551956/2/gate/openstack-tox-py35/81176e5/ : SUCCESS in 2m 19s\n- build-openstack-sphinx-docs http://logs.openstack.org/56/551956/2/gate/build-openstack-sphinx-docs/f963ab6/html/ : SUCCESS in 2m 08s\n- legacy-tempest-dsvm-neutron-src http://logs.openstack.org/56/551956/2/gate/legacy-tempest-dsvm-neutron-src/7ac9431/ : SUCCESS in 1h 31m 46s\n- legacy-swift-dsvm-functional http://logs.openstack.org/56/551956/2/gate/legacy-swift-dsvm-functional/c247e9e/ : SUCCESS in 21m 06s\n- legacy-swiftclient-dsvm-functional http://logs.openstack.org/56/551956/2/gate/legacy-swiftclient-dsvm-functional/44b562e/ : SUCCESS in 18m 00s","accounts_in_message":[],"_revision_number":2}],"current_revision_number":2,"current_revision":"e65070964c7b1e04119c87e5f344d39358780d18","revisions":{"2e7fae4c4c58cb236d349699c56b8584397bc20c":{"kind":"REWORK","_number":1,"created":"2018-03-12 10:52:09.000000000","uploader":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"ref":"refs/changes/56/551956/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/python-swiftclient","ref":"refs/changes/56/551956/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/1"}}},"commit":{"parents":[{"commit":"a08d9b109f7ac3c5b844429addbb7e8507268c8e","subject":"Merge \"Add a query_string option to head_object().\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/python-swiftclient/commit/a08d9b109f7ac3c5b844429addbb7e8507268c8e"}]}],"author":{"name":"Kota Tsuyuzaki","email":"tsuyuzaki.kota@lab.ntt.co.jp","date":"2018-03-12 08:54:17.000000000","tz":540},"committer":{"name":"Kota Tsuyuzaki","email":"tsuyuzaki.kota@lab.ntt.co.jp","date":"2018-03-12 10:39:49.000000000","tz":540},"subject":"Add force auth retry mode in swiftclient","message":"Add force auth retry mode in swiftclient\n\nThis patch attemps to add an option to force get_auth call while retrying\nan operation even if it gets errors other than 401 Unauthorized.\n\nWhy we need this:\nThe main reason why we need this is current python-swiftclient requests could\nnever get succeeded under certion situation using third party proxies/load balancers\nbetween the client and swift-proxy server. I think, it would be general situation\nof the use case.\n\nSpecifically describing nginx case, the nginx can close the socket from the client\nwhen the response code from swift is not 2xx series. In default, nginx can wait the\nbuffers from the client for a while (default 30s)[1] but after the time past, nginx\nwill close the socket immediately. Unfortunately, if python-swiftclient has still been\nsending the data into the socket, python-swiftclient will get socket error (EPIPE,\nBrokenPipe). From the swiftclient perspective, this is absolutely not an auth error,\nso current python-swiftclient will continue to retry without re-auth.\nHowever, if the root cause is sort of 401 (i.e. nginx got 401 unauthorized from the\nswift-proxy because of token expiration), swiftclient will loop 401 -\u003e EPIPE -\u003e 401...\nuntil it consume the max retry times.\n\nIn particlar, less time to live of the token and multipart object upload with large\nsegments could not get succeeded as below:\n\nConnection Model:\n\npython-swiftclient -\u003e nginx -\u003e swift-proxy -\u003e swift-backend\n\nCase: Try to create slo with large segments and the auth token expired with 1 hour\n\n1. client create a connection to nginx with successful response from swift-proxy and its auth\n2. client continue to put large segment objects\n   (e.g. 1~5GB for each and the total would 20~30GB, i.e. 20~30 segments)\n3. after some of segments uploaded, 1 hour past but client is still trying to\n   send remaining segment objects.\n4. nginx got 401 from swift-proxy for a request and wait that the connection is closed\n   from the client but timeout past because the python-swiftclient is still sending much data\n   into the socket before reading the 401 response.\n5. client got socket error because nginx closed the connection during sending the buffer.\n6. client retries a new connection to nginx without re-auth...\n\n\u003cloop 4-6\u003e\n\n7. finally python-swiftclient failed with socket error (Broken Pipe)\n\nIn operational perspective, setting longer timeout for lingering close would be an option but\nit\u0027s not complete solution because any other proxy/LB may not support the options.\n\nIf we actually do THE RIGHT THING in python-swiftclient, we should send expects: 100-continue\nheader and handle the first response to re-auth correctly.\n\nHOWEVER, the current python\u0027s httplib and requests module used by python-swiftclient doesn\u0027t\nsupport expects: 100-continue header [2] and the thread proposed a fix [3] is not super active.\nAnd we know the reason we depends on the library is to fix a security issue that existed\nin older python-swiftclient [4] so that we should touch around it super carefully.\n\nIn the reality, as the hot fix, this patch try to mitigate the unfortunate situation\ndescribed above WITHOUT 100-continue fix, just users can force to re-auth when any errors\noccurred during the retries that can be accepted in the upstream.\n\n1: http://nginx.org/en/docs/http/ngx_http_core_module.html#lingering_close\n2: https://github.com/requests/requests/issues/713\n3: https://bugs.python.org/issue1346874\n\nChange-Id: I3470b56e3f9cf9cdb8c2fc2a94b2c551927a3440\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/python-swiftclient/commit/2e7fae4c4c58cb236d349699c56b8584397bc20c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/python-swiftclient/commit/2e7fae4c4c58cb236d349699c56b8584397bc20c"}]},"branch":"refs/heads/master"},"e65070964c7b1e04119c87e5f344d39358780d18":{"kind":"REWORK","_number":2,"created":"2018-03-13 03:30:13.000000000","uploader":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"ref":"refs/changes/56/551956/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/python-swiftclient","ref":"refs/changes/56/551956/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/python-swiftclient refs/changes/56/551956/2"}}},"commit":{"parents":[{"commit":"a08d9b109f7ac3c5b844429addbb7e8507268c8e","subject":"Merge \"Add a query_string option to head_object().\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/python-swiftclient/commit/a08d9b109f7ac3c5b844429addbb7e8507268c8e"}]}],"author":{"name":"Kota Tsuyuzaki","email":"tsuyuzaki.kota@lab.ntt.co.jp","date":"2018-03-12 08:54:17.000000000","tz":540},"committer":{"name":"Kota Tsuyuzaki","email":"tsuyuzaki.kota@lab.ntt.co.jp","date":"2018-03-13 03:29:48.000000000","tz":540},"subject":"Add force auth retry mode in swiftclient","message":"Add force auth retry mode in swiftclient\n\nThis patch attemps to add an option to force get_auth call while retrying\nan operation even if it gets errors other than 401 Unauthorized.\n\nWhy we need this:\nThe main reason why we need this is current python-swiftclient requests could\nnever get succeeded under certion situation using third party proxies/load balancers\nbetween the client and swift-proxy server. I think, it would be general situation\nof the use case.\n\nSpecifically describing nginx case, the nginx can close the socket from the client\nwhen the response code from swift is not 2xx series. In default, nginx can wait the\nbuffers from the client for a while (default 30s)[1] but after the time past, nginx\nwill close the socket immediately. Unfortunately, if python-swiftclient has still been\nsending the data into the socket, python-swiftclient will get socket error (EPIPE,\nBrokenPipe). From the swiftclient perspective, this is absolutely not an auth error,\nso current python-swiftclient will continue to retry without re-auth.\nHowever, if the root cause is sort of 401 (i.e. nginx got 401 unauthorized from the\nswift-proxy because of token expiration), swiftclient will loop 401 -\u003e EPIPE -\u003e 401...\nuntil it consume the max retry times.\n\nIn particlar, less time to live of the token and multipart object upload with large\nsegments could not get succeeded as below:\n\nConnection Model:\n\npython-swiftclient -\u003e nginx -\u003e swift-proxy -\u003e swift-backend\n\nCase: Try to create slo with large segments and the auth token expired with 1 hour\n\n1. client create a connection to nginx with successful response from swift-proxy and its auth\n2. client continue to put large segment objects\n   (e.g. 1~5GB for each and the total would 20~30GB, i.e. 20~30 segments)\n3. after some of segments uploaded, 1 hour past but client is still trying to\n   send remaining segment objects.\n4. nginx got 401 from swift-proxy for a request and wait that the connection is closed\n   from the client but timeout past because the python-swiftclient is still sending much data\n   into the socket before reading the 401 response.\n5. client got socket error because nginx closed the connection during sending the buffer.\n6. client retries a new connection to nginx without re-auth...\n\n\u003cloop 4-6\u003e\n\n7. finally python-swiftclient failed with socket error (Broken Pipe)\n\nIn operational perspective, setting longer timeout for lingering close would be an option but\nit\u0027s not complete solution because any other proxy/LB may not support the options.\n\nIf we actually do THE RIGHT THING in python-swiftclient, we should send expects: 100-continue\nheader and handle the first response to re-auth correctly.\n\nHOWEVER, the current python\u0027s httplib and requests module used by python-swiftclient doesn\u0027t\nsupport expects: 100-continue header [2] and the thread proposed a fix [3] is not super active.\nAnd we know the reason we depends on the library is to fix a security issue that existed\nin older python-swiftclient [4] so that we should touch around it super carefully.\n\nIn the reality, as the hot fix, this patch try to mitigate the unfortunate situation\ndescribed above WITHOUT 100-continue fix, just users can force to re-auth when any errors\noccurred during the retries that can be accepted in the upstream.\n\n1: http://nginx.org/en/docs/http/ngx_http_core_module.html#lingering_close\n2: https://github.com/requests/requests/issues/713\n3: https://bugs.python.org/issue1346874\n4: https://review.openstack.org/#/c/69187/\n\nChange-Id: I3470b56e3f9cf9cdb8c2fc2a94b2c551927a3440\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/python-swiftclient/commit/e65070964c7b1e04119c87e5f344d39358780d18"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/python-swiftclient/commit/e65070964c7b1e04119c87e5f344d39358780d18"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}
