)]}'
{"id":"openstack%2Fswift~706361","triplet_id":"openstack%2Fswift~master~I1629075630036af650fbec49efee990a3b1c7090","project":"openstack/swift","branch":"master","hashtags":[],"change_id":"I1629075630036af650fbec49efee990a3b1c7090","subject":"Clean up concurrent get requests once we have a response","status":"NEW","created":"2020-02-06 18:44:29.000000000","updated":"2020-03-03 17:22:40.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":7,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"4802bbeb5e9c1eca9b9f91ce22ccafe71daad9e9","_number":706361,"virtual_id_number":706361,"owner":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"date":"2020-03-03 17:22:40.000000000","_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"date":"2020-03-03 16:12:51.000000000","_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"tag":"autogenerated:zuul:check","value":1,"date":"2020-03-03 01:28:39.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"date":"2020-03-02 23:54:01.000000000","_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-03-03 01:28:39.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2020-03-03 16:12:51.000000000","updated_by":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"reviewer":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"state":"REVIEWER"},{"updated":"2020-03-03 17:22:40.000000000","updated_by":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"reviewer":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"state":"REVIEWER"}],"messages":[{"id":"233792814ccf5f93c33ac4e8e81ed657d0f11a4f","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-02-06 18:44:29.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"3a3872dc7fc14a772180db646384dc3aa6f728a0","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-02-06 20:15:20.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/2a93195400874440a59970f252ad88d9 : SUCCESS in 8m 17s\n- grenade-py3 https://zuul.opendev.org/t/openstack/build/e88e9d8968074be5b7d779624c65afa3 : SUCCESS in 1h 23m 09s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/e8fcfd29b93842b08d49bd952cbee9ff : SUCCESS in 55m 37s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d6b4caf2c9fe4a70855ff13938a8401c : SUCCESS in 8m 58s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/4cc19d604c084ae5a175ed8b7c84f8c8 : SUCCESS in 10m 27s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/33e0cde617014326a6978573dc019d2d : SUCCESS in 13m 04s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/30d31219ffc14556b3706672e4ab4b19 : SUCCESS in 15m 27s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/8e2d4c411af342d0bc19203ae4521a7d : SUCCESS in 13m 44s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/cc52841a5b464154afc9b14c4801bd21 : SUCCESS in 13m 52s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/fe16cb02904c419bb184ff0faec74084 : SUCCESS in 10m 47s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/1cd734698a6a424ca1ef182258a57a50 : SUCCESS in 13m 05s\n- swift-tox-func-domain-remap-staticweb-py27 https://zuul.opendev.org/t/openstack/build/724866a6a28545b982b03485bc2dcd41 : SUCCESS in 11m 07s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/97c4f10a444741899ec36ef321c1f3e0 : SUCCESS in 11m 54s\n- swift-tox-func-s3api-py27 https://zuul.opendev.org/t/openstack/build/67f7e490b9f84d0f9cf9b17016622e02 : SUCCESS in 18m 08s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/60997293a5f447cb847c9a69202d0131 : SUCCESS in 13m 37s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/1c8f838b1dad49a6a480f328ba25537d : SUCCESS in 15m 52s\n- swift-tox-func-domain-remap-staticweb-py37 https://zuul.opendev.org/t/openstack/build/4c70e92ef9f94d8d85f6ca3caafc71d9 : SUCCESS in 10m 05s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/b9fb76ea9c3944238756f35d6a4b8309 : SUCCESS in 12m 30s\n- swift-tox-func-s3api-py37 https://zuul.opendev.org/t/openstack/build/ca226e4e66bf4d7dbbd3c96bb0598b8e : SUCCESS in 20m 24s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/f915e060115c47d0a7f049469a5c3d60 : SUCCESS in 18m 39s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/9c8ae9970c414c829e72b1f520ef7312 : SUCCESS in 42m 10s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/c23d6a77141a4fc3b481ed74c341766b : SUCCESS in 33m 21s\n- swift-dsvm-functional-py3 https://zuul.opendev.org/t/openstack/build/3a67a6a367aa40e5b8f2f82e0afffad8 : SUCCESS in 31m 25s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/ee557d18d6484cca9daabafd92c50768 : SUCCESS in 33m 08s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/5d4291a5f6a8455e9757b86d85c01d27 : SUCCESS in 12m 41s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/db03079670c34e488e9b294dea23ed62 : SUCCESS in 7m 28s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/d1774ade0aba46cc96c26c71104e06d9 : SUCCESS in 23m 08s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/09a56b0c789e4190b6aa4646543d79f0 : SUCCESS in 1h 21m 13s","accounts_in_message":[],"_revision_number":1},{"id":"41dbe97a3cd341796b623b0c79c0cef8844411ad","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-02-06 20:26:13.000000000","message":"Patch Set 1:\n\n(1 comment)\n\nCan you elaborate what exactly we\u0027re trying to solve here?  Do we think closing the connection early is primarily beneficial to the proxy or the remote service?","accounts_in_message":[],"_revision_number":1},{"id":"42f23e0d7ca5deb184884211b2f9cb6b7ad5d8b2","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-02-06 22:15:37.000000000","message":"Patch Set 1:\n\n(1 comment)\n\nI\u0027m mainly worried about the proxy -- it may help out the backend server, too (assuming it can tell that the other side hung up), but that\u0027d just be gravy.\n\nI\u0027ve been running down an issue where an (apparently overloaded) container server is popping node_timeouts across all proxies. I\u0027m also seeing a spike in object GET latencies that seems to correlate quite nicely with spikes in container listings. My working theory is that the added resource usage of having each proxy waiting on an extra ~500 container requests (over a couple minute window) is what\u0027s causing the latency spikes.","accounts_in_message":[],"_revision_number":1},{"id":"a58e0bab9749545a1fff0a70262de24ad50b04f8","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-02-06 22:30:15.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"eed68caf348cc32865cfd5f96464af2b443accae","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2020-02-12 05:58:58.000000000","message":"Patch Set 1: Code-Review-1\n\n\u003e I\u0027ve been running down an issue where an (apparently overloaded)\n \u003e container server is popping node_timeouts across all proxies. (...)\n\nThis answer and the comment you made in-line about the testbed need to go into the changelog.","accounts_in_message":[],"_revision_number":1},{"id":"b37ab3b6c27ce5dddc6f9c893e79b6469711eb3a","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-03-02 19:33:44.000000000","message":"Patch Set 2: Commit message was updated.","accounts_in_message":[],"_revision_number":2},{"id":"2a37799efdf3e28a727a26c0c51c49d289040b25","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-03-02 23:04:08.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/0748fc2200c44eac9b985d7f5d25a8e4 : SUCCESS in 10m 31s\n- grenade-py3 https://zuul.opendev.org/t/openstack/build/a04c39a0277444899543f252ba242f62 : SUCCESS in 1h 03m 44s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/b5a7e50c329b4f46910f8b26643bd465 : SUCCESS in 55m 35s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/7b54554852bf4717a279693894709197 : SUCCESS in 12m 33s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/ae4a5c5742744196a30d1d85e0529091 : SUCCESS in 12m 28s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/a8d9b095c85545e4ac3e5c2ababb52c9 : SUCCESS in 19m 44s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/eda614748df74948947e5e2efffe2bb6 : SUCCESS in 18m 51s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/48b4f89ba1964d5887274a90e3e64788 : SUCCESS in 16m 33s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/8e20c99d683a4f408369f452c798b29b : TIMED_OUT in 41m 43s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/b00ba790304944c588b9896208309e5e : SUCCESS in 17m 07s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/fe390301ae9d40ad84073699bbf076ad : SUCCESS in 15m 28s\n- swift-tox-func-domain-remap-staticweb-py27 https://zuul.opendev.org/t/openstack/build/4c04469bf6b5476b8658e8eb8ddaa2a2 : SUCCESS in 11m 51s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/1e47d9dd12a04f79b6393bb69ba5f679 : SUCCESS in 13m 23s\n- swift-tox-func-s3api-py27 https://zuul.opendev.org/t/openstack/build/c437455800734bc8bcf811880adcbd50 : SUCCESS in 20m 41s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/a87b92f871184899b3e58bcbc72d9ee9 : SUCCESS in 18m 36s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/6a90a0143b1643baab0a922892fedc81 : SUCCESS in 16m 35s\n- swift-tox-func-domain-remap-staticweb-py37 https://zuul.opendev.org/t/openstack/build/b66c197d866a4f878ede4815322efeda : SUCCESS in 17m 28s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/55723000f23f4425bceb0efde00b889d : SUCCESS in 18m 59s\n- swift-tox-func-s3api-py37 https://zuul.opendev.org/t/openstack/build/0388d6ca9b3a426e92d5b81707a312ac : SUCCESS in 22m 36s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/b567a8c0dbd24666a24af5bad6bfe5d5 : SUCCESS in 23m 38s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/23535ee4be8f424291929d09e4480903 : SUCCESS in 46m 13s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/1e1a04d2f03f4149be679b023ceaeb70 : SUCCESS in 38m 55s\n- swift-dsvm-functional-py3 https://zuul.opendev.org/t/openstack/build/d2edc29cff6d46668bb42c5c8dd8513c : SUCCESS in 38m 29s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/806d946b6ae94f7991109d322ad0f002 : SUCCESS in 38m 17s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/1c6702774a6d44578b292746b233852a : SUCCESS in 14m 38s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/575975e6fbed412580ec00386070f11d : SUCCESS in 9m 01s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/08bcaf0e3a494a27814efbca717298cd : SUCCESS in 44m 51s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/bae9d1fcf63c4526923c5d2f60685e60 : SUCCESS in 1h 10m 10s","accounts_in_message":[],"_revision_number":2},{"id":"64422e7902a7aa3e176c483b6ff57ea1468d6871","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-03-02 23:54:01.000000000","message":"Patch Set 2:\n\nrecheck","accounts_in_message":[],"_revision_number":2},{"id":"7e58f75968ad83e6868c9ecf9732d0ee8051b257","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-03-03 01:28:39.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/903e87a7ce2648c3a6d8fe2cd4cc39bf : SUCCESS in 10m 19s\n- grenade-py3 https://zuul.opendev.org/t/openstack/build/c1b8444996a041fca9808c907affca13 : SUCCESS in 1h 04m 28s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/9658bd7ec4af44fb938f74955f9fe182 : SUCCESS in 53m 36s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/61723f418ef541e3ae7b097c996dd584 : SUCCESS in 10m 22s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/52b4653b0a724b72bb2520ed82683c5e : SUCCESS in 11m 08s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/6aa34bf880cf4346a147b507a1376f12 : SUCCESS in 13m 11s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/1ab2dfe7bf4f4257a696030b4afb1300 : SUCCESS in 14m 46s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/580fe746c6884291bd9035d0ff27b035 : SUCCESS in 14m 28s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/9a9b5da85d7a4fe2ab3a1fd845dc493f : SUCCESS in 16m 01s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/25ef97e0b158433aa6e22cb820876187 : SUCCESS in 10m 51s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/8c2c19a86abb40b39e02a8316d26d65e : SUCCESS in 13m 33s\n- swift-tox-func-domain-remap-staticweb-py27 https://zuul.opendev.org/t/openstack/build/69263b7410594b6786f38c06c60c4511 : SUCCESS in 10m 37s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/1e6f057ee6714f23ac78bf6149459a95 : SUCCESS in 13m 05s\n- swift-tox-func-s3api-py27 https://zuul.opendev.org/t/openstack/build/a9d7ac6953624acb98b0de6a29c9cd22 : SUCCESS in 21m 57s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/75384cfe2e104030a20a276a9a451aa6 : SUCCESS in 15m 49s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/cd2039c96ccf45109f18534e217d2a66 : SUCCESS in 16m 09s\n- swift-tox-func-domain-remap-staticweb-py37 https://zuul.opendev.org/t/openstack/build/81c6cc9765454cb3af2ec7b9909b0178 : SUCCESS in 17m 19s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/b1d6bc1c600e4c8db7caaf5f50e44fd3 : SUCCESS in 12m 33s\n- swift-tox-func-s3api-py37 https://zuul.opendev.org/t/openstack/build/d4ee09c4a4c94ec9880830f840f2276d : SUCCESS in 16m 08s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/5cdd5bdb3ade4269826ff09252ea3400 : SUCCESS in 35m 45s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/59116fa0cf1c41a08f124826ea525158 : SUCCESS in 55m 58s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/a488ac6703024251bf2ef154938f6139 : SUCCESS in 35m 52s\n- swift-dsvm-functional-py3 https://zuul.opendev.org/t/openstack/build/26060e4f27994a1bb8cac6771c4b5a47 : SUCCESS in 36m 18s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/4535ecaecd5443328cbb6b1290684441 : SUCCESS in 36m 27s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/b11ebe6202ad449e890e7abb7b5b7d60 : SUCCESS in 15m 25s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/c7ebcf7168eb48eebc0c093f550a2398 : SUCCESS in 7m 13s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/71befe4cbfaa41d884a1518805e5a916 : SUCCESS in 28m 45s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/f8e5b6a63d634fa7820f5391ee9e6e89 : SUCCESS in 1h 14m 32s","accounts_in_message":[],"_revision_number":2},{"id":"1e5eea55d9bc92198139bfe117e7ed6e54fd724c","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-03-03 16:12:51.000000000","message":"Patch Set 2:\n\n(3 comments)\n\nI have not been able to demonstrate any objective benefit of this change.  I think you have to have concurrent_gets turned on to have any behavioral difference at all.\n\nI certainly agree with the analysis that the spawned greenthread connections that don\u0027t finish first will run to completion in the background after the proxy has already responded to the client and both connections remain open.  So some argument could be mounted there\u0027s a virtue of cleanliness in \"handling\" the connections we won\u0027t need to respond - but I\u0027ve not been able to verify the abrupt termination is cleaner.\n\nI\u0027m introducing delay into a working container server directly:\n\n    diff --git a/swift/container/server.py b/swift/container/server.py\n    index 5d89ec4f8..e93d06e81 100644\n    --- a/swift/container/server.py\n    +++ b/swift/container/server.py\n    @@ -847,6 +847,9 @@ class ContainerController(BaseStorageServer):\n             return HTTPNoContent(request\u003dreq)\n     \n         def __call__(self, env, start_response):\n    +        import random, eventlet\n    +        if random.random() \u003c 0.2:\n    +            eventlet.sleep(21)\n             start_time \u003d time.time()\n             req \u003d Request(env)\n             self.logger.txn_id \u003d req.headers.get(\u0027x-trans-id\u0027, None)\n\nThe primary difference may then be that none of my tcp or http protocols are violated - these are slow, but working connections and valid responses.\n\nMy counts of TCP connections in TIME_WAIT immediately after a burst of container GETs is the same with or without the patch.  I assume this is because of the tcp_tw setttings that prefer keeping working tcp connections to the backend lively which reduces delays \"connecting\" to backend servers in subsequent requests.\n\nThe number of backend responses that log after the client responses finish is the same before or after this change.  I assume this is because no amount of closing the connection on the proxy is going to trick the container server into not finishing and attempting to send it\u0027s response - we just push the problem of draining the response into the tcp buffer handling.\n\nThe biggest change I notice is the lack of errors in my logs when my servers are slow.  Instead of seeing the proxy log that it timed out a slow response from a backend server - I just see instead silence.  I don\u0027t get a log message a request was made.  I don\u0027t get a log message the request timedout.  I see the successful response to the client (maybe with an invisible concurrency_timeout delay) then I see the message from the backend server after it finishes.  This change in logging feels worse to me than what we get now.\n\nAside from some loss of visibility into the logs, I\u0027m also sure that we\u0027re loosing error limiting and timing tracking that we were getting by letting the responses finish or timeout so we can annotate the node timing/errors accurately.  This may in fact lead to WORSE aggregate performance with this patch in place by making additional requests to already slow nodes and waiting on connections to downed nodes.  This effect would be most problematic with a concurrency_timeout that\u0027s less than your connect_timeout which is fairly common for latency sensitive deployments.\n\nPerhaps more justification could be made for the benefit of this change (i.e. I\u0027m under valuing optimization of proxy CPU against other tradeoffs), or my perception of the neutral and/or negative effects might be mitigated.  I don\u0027t have a strong opinion that it should or should not merge - but I\u0027d lean not.","accounts_in_message":[],"_revision_number":2},{"id":"4802bbeb5e9c1eca9b9f91ce22ccafe71daad9e9","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2020-03-03 17:22:40.000000000","message":"Patch Set 2:\n\n(1 comment)\n\n\u003e My counts of TCP connections in TIME_WAIT immediately after a burst\n \u003e of container GETs is the same with or without the patch.\n\nThose worry me a lot for a couple of reasons.\n\nFirst, I have a bug outstanding in RHOS right now about this.\nIt turned out that our replicators can consume more sockets\nthan proxies, if traffic is light, and they easily eat all\nthe free sockets in PACO clusters.\n\nSecond, it appears that kernel people took a course on\nremoving all the tcp_tw_xxxx tunables. Now, kernel finds out\nwhich sockets to reuse, and there\u0027s nothing you can do about it.\nIn addition, you can have 60,000 TIME_WAIT sockets, yet once\nyou go and bind or connect, the operation succeeds, because\nkernel finds which TIME_WAIT socket to victimize, closes it,\nand reuses it. Bottom line, the output of netstat does not\ntell you anymore if there\u0027s a problem of not enough sockets.\nI know they had their reasons, but IMHO this sucks.","accounts_in_message":[],"_revision_number":2}],"current_revision_number":2,"current_revision":"280e4a90b53622e1edbf55a17c03a142e88df2a5","revisions":{"07101b18ffaccab9ccd91f12d17300bc785d5d46":{"kind":"REWORK","_number":1,"created":"2020-02-06 18:44:29.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/61/706361/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/61/706361/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/61/706361/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/61/706361/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/61/706361/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/61/706361/1"}}},"commit":{"parents":[{"commit":"c0b4d644dfc68be182a726792e35992efb6ebcf4","subject":"Merge \"added value and notes to a sample config file for s3token\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/c0b4d644dfc68be182a726792e35992efb6ebcf4"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2020-02-06 18:32:53.000000000","tz":-480},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2020-02-06 18:32:53.000000000","tz":-480},"subject":"Clean up concurrent get requests once we have a response","message":"Clean up concurrent get requests once we have a response\n\nChange-Id: I1629075630036af650fbec49efee990a3b1c7090\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/07101b18ffaccab9ccd91f12d17300bc785d5d46"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/07101b18ffaccab9ccd91f12d17300bc785d5d46"}]},"branch":"refs/heads/master"},"280e4a90b53622e1edbf55a17c03a142e88df2a5":{"kind":"NO_CODE_CHANGE","_number":2,"created":"2020-03-02 19:33:44.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/61/706361/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/61/706361/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/61/706361/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/61/706361/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/61/706361/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/61/706361/2"}}},"commit":{"parents":[{"commit":"c0b4d644dfc68be182a726792e35992efb6ebcf4","subject":"Merge \"added value and notes to a sample config file for s3token\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/c0b4d644dfc68be182a726792e35992efb6ebcf4"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2020-02-06 18:32:53.000000000","tz":-480},"committer":{"name":"Tim Burke","email":"tim@swiftstack.com","date":"2020-03-02 19:33:30.000000000","tz":0},"subject":"Clean up concurrent get requests once we have a response","message":"Clean up concurrent get requests once we have a response\n\nI\u0027ve been running down an issue where an (apparently overloaded)\ncontainer server is popping node_timeouts across all proxies. I\u0027m\n*also* seeing a spike in object GET latencies that seems to correlate\nquite nicely with spikes in container listings.\n\nMy working theory is that the added resource usage of having each\nproxy waiting on an extra ~500 container requests (over a couple\nminute window) is what\u0027s causing the latency spikes.\n\nLet\u0027s at least clean up the proxy-server connections and greenthreads\nwhen they\u0027re no longer necessary; depending on whether it\u0027s started\nprocessing the requests or not, this may also provide some relief to\nthe overwhelmed container-server.\n\nHere\u0027s my testing setup:\n\n * Create a new, empty container.\n * Stop all container-servers, then restart just *one* of them.\n * Run ncat on one or more of the stopped-container-server ports to\n   simulate a hung server; something like\n\n   ncat -lk --max-conns 1024 localhost 6031 \u003e/dev/null\n\nIn one window, watch the open connections like\n\n   watch -n .1 \u0027lsof -i TCP | grep 6031\u0027\n\nIn another, restart proxies (to make sure nothing\u0027s error-limited)\nand issue a bunch of requests like\n\n   swift-init proxy restart \u0026\u0026 ( for i in {1..100}; do curl http://saio:8090/v1/AUTH_test/c \u003e\u00262 \u0026 done; wait ) 2\u003e /dev/null\n\n(You can repeat this last step a few times, see whether your\nobservation was a one-off or repeatable thing.)\n\nOnce it all joins up again, check for open connections and check\nthe logs for timeouts.\n\nChange-Id: I1629075630036af650fbec49efee990a3b1c7090\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/280e4a90b53622e1edbf55a17c03a142e88df2a5"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/280e4a90b53622e1edbf55a17c03a142e88df2a5"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY"},{"label":"Workflow","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}