)]}'
{"id":"openstack%2Fswift~672186","triplet_id":"openstack%2Fswift~master~Ia832e9bab13167948f01bc50aa8a61974ce189fb","project":"openstack/swift","branch":"master","topic":"bug/1833612","hashtags":[],"change_id":"Ia832e9bab13167948f01bc50aa8a61974ce189fb","subject":"Ignore 404s from handoffs for objects when calculating quorum","status":"MERGED","created":"2019-07-22 22:21:20.000000000","updated":"2019-08-02 13:23:27.000000000","submitted":"2019-08-02 13:17:54.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":31,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"672186-1564751874112-efd57ca1","meta_rev_id":"9c0aa842cb6a65f5480056550e3e50ab13213aad","_number":672186,"virtual_id_number":672186,"owner":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},{"value":0,"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":2,"date":"2019-08-02 13:17:53.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"date":"2019-08-01 21:12:25.000000000","_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"all":[{"value":2,"date":"2019-08-02 11:44:25.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},{"value":2,"date":"2019-08-01 22:22:06.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"all":[{"value":1,"date":"2019-08-02 11:44:25.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},{"value":0,"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"date":"2019-08-02 04:34:33.000000000","_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2019-08-01 22:22:06.000000000","updated_by":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"reviewer":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"state":"REVIEWER"},{"updated":"2019-08-02 04:34:33.000000000","updated_by":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"reviewer":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"state":"REVIEWER"},{"updated":"2019-08-02 11:44:25.000000000","updated_by":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"reviewer":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"state":"REVIEWER"},{"updated":"2019-08-02 13:17:53.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"e79eaea3237231f4c3eecef47671315dafd05d23","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-22 22:21:20.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"f8d581c0757b78103d720a5ae2edf92e836287bd","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-22 23:00:45.000000000","message":"Patch Set 1: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- openstack-tox-docs http://logs.openstack.org/86/672186/1/check/openstack-tox-docs/3640ebe/html/ : SUCCESS in 4m 13s\n- swift-build-image http://logs.openstack.org/86/672186/1/check/swift-build-image/8ba0570/ : SUCCESS in 8m 55s (non-voting)\n- swift-tox-py27 http://logs.openstack.org/86/672186/1/check/swift-tox-py27/7badc8e/ : FAILURE in 12m 02s\n- swift-tox-py36 http://logs.openstack.org/86/672186/1/check/swift-tox-py36/f79d0a3/ : SUCCESS in 14m 20s (non-voting)\n- swift-tox-py37 http://logs.openstack.org/86/672186/1/check/swift-tox-py37/be669c5/ : FAILURE in 10m 30s\n- swift-tox-func http://logs.openstack.org/86/672186/1/check/swift-tox-func/a78ddc6/ : SUCCESS in 15m 25s\n- swift-tox-func-encryption http://logs.openstack.org/86/672186/1/check/swift-tox-func-encryption/1bdc34c/ : SUCCESS in 15m 34s\n- swift-tox-func-domain-remap-staticweb http://logs.openstack.org/86/672186/1/check/swift-tox-func-domain-remap-staticweb/285e9fd/ : SUCCESS in 9m 36s\n- swift-tox-func-ec http://logs.openstack.org/86/672186/1/check/swift-tox-func-ec/096bbe3/ : SUCCESS in 10m 58s\n- swift-tox-func-s3api http://logs.openstack.org/86/672186/1/check/swift-tox-func-s3api/5bc4f88/ : SUCCESS in 20m 01s\n- swift-tox-func-py37 http://logs.openstack.org/86/672186/1/check/swift-tox-func-py37/8dffb47/ : SUCCESS in 14m 32s\n- swift-tox-func-domain-remap-staticweb-py37 http://logs.openstack.org/86/672186/1/check/swift-tox-func-domain-remap-staticweb-py37/a1801ea/ : SUCCESS in 9m 49s\n- swift-tox-func-ec-py37 http://logs.openstack.org/86/672186/1/check/swift-tox-func-ec-py37/0d9ed22/ : SUCCESS in 9m 49s\n- swift-tox-func-s3api-ceph-s3tests-tempauth http://logs.openstack.org/86/672186/1/check/swift-tox-func-s3api-ceph-s3tests-tempauth/06580f6/ : SUCCESS in 20m 32s (non-voting)\n- swift-probetests-centos-7 http://logs.openstack.org/86/672186/1/check/swift-probetests-centos-7/aeb8f4c/ : SUCCESS in 33m 54s\n- swift-dsvm-functional http://logs.openstack.org/86/672186/1/check/swift-dsvm-functional/fb960c1/ : SUCCESS in 30m 23s\n- swift-dsvm-functional-py3 http://logs.openstack.org/86/672186/1/check/swift-dsvm-functional-py3/9bf701e/ : SUCCESS in 29m 02s\n- swift-dsvm-functional-ipv6 http://logs.openstack.org/86/672186/1/check/swift-dsvm-functional-ipv6/515ea0c/ : SUCCESS in 37m 06s\n- swift-tox-lower-constraints http://logs.openstack.org/86/672186/1/check/swift-tox-lower-constraints/5957fca/ : SUCCESS in 18m 15s\n- openstack-tox-pep8 http://logs.openstack.org/86/672186/1/check/openstack-tox-pep8/a53c8f5/ : SUCCESS in 5m 35s\n- swift-multinode-rolling-upgrade http://logs.openstack.org/86/672186/1/check/swift-multinode-rolling-upgrade/6bf2dc6/ : SUCCESS in 19m 41s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"b6b530f712799191a3273f80382654b484003312","author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"date":"2019-07-23 11:42:52.000000000","message":"Patch Set 1:\n\n(1 comment)\n\nThe code looks correct for the desired change. I also wrote some follow-on probe tests to give further confidence and help document the behavior.\n\nA couple of concerns: \n  1. we don\u0027t seem to have unit tests for the replication policy.\n  2. From the number of unit tests changed, I am concerned that we had purposely documented (if not *designed) for the behavior to be a 404 response and now we are changing that. Sure the new argument seems correct (especially in the EC case where we know we have something there and we just can\u0027t reconstruct) but I keep asking why our old selves chose the behavior we chose. It seems we were more willing to rely on the eventual consistency behavior of the system. \"If I can\u0027t get your data, then it must not be there (or at least I assume it is not there, but), eventually I\u0027ll get it\".","accounts_in_message":[],"_revision_number":1},{"id":"3dff26b1d08d6e87389e448c43a04ddd0c9b54fc","author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"date":"2019-07-23 11:43:56.000000000","message":"Patch Set 1:\n\nJust fyi, I\u0027m also seeing the flaky unit test on my env. looking into that next.","accounts_in_message":[],"_revision_number":1},{"id":"758fdb84dea90f779e45616ea361c2c3e9cf73c9","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-23 14:27:00.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"ef16bf512699655956927e00505c4ff194779c08","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-23 16:43:56.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs http://logs.openstack.org/86/672186/2/check/openstack-tox-docs/c917ccd/html/ : SUCCESS in 5m 10s\n- swift-build-image http://logs.openstack.org/86/672186/2/check/swift-build-image/ed98b47/ : SUCCESS in 9m 26s (non-voting)\n- swift-tox-py27 http://logs.openstack.org/86/672186/2/check/swift-tox-py27/a262f2c/ : SUCCESS in 11m 26s\n- swift-tox-py36 http://logs.openstack.org/86/672186/2/check/swift-tox-py36/6b3cf66/ : SUCCESS in 16m 09s (non-voting)\n- swift-tox-py37 http://logs.openstack.org/86/672186/2/check/swift-tox-py37/643ca57/ : SUCCESS in 12m 51s\n- swift-tox-func http://logs.openstack.org/86/672186/2/check/swift-tox-func/2d6715a/ : SUCCESS in 9m 38s\n- swift-tox-func-encryption http://logs.openstack.org/86/672186/2/check/swift-tox-func-encryption/a396117/ : SUCCESS in 12m 10s\n- swift-tox-func-domain-remap-staticweb http://logs.openstack.org/86/672186/2/check/swift-tox-func-domain-remap-staticweb/51fa4e3/ : SUCCESS in 8m 41s\n- swift-tox-func-ec http://logs.openstack.org/86/672186/2/check/swift-tox-func-ec/081c24d/ : SUCCESS in 10m 18s\n- swift-tox-func-s3api http://logs.openstack.org/86/672186/2/check/swift-tox-func-s3api/da12c3a/ : SUCCESS in 12m 06s\n- swift-tox-func-py37 http://logs.openstack.org/86/672186/2/check/swift-tox-func-py37/ae414f7/ : SUCCESS in 11m 24s\n- swift-tox-func-domain-remap-staticweb-py37 http://logs.openstack.org/86/672186/2/check/swift-tox-func-domain-remap-staticweb-py37/7f190d0/ : SUCCESS in 8m 42s\n- swift-tox-func-ec-py37 http://logs.openstack.org/86/672186/2/check/swift-tox-func-ec-py37/0cfb15b/ : SUCCESS in 11m 58s\n- swift-tox-func-s3api-ceph-s3tests-tempauth http://logs.openstack.org/86/672186/2/check/swift-tox-func-s3api-ceph-s3tests-tempauth/bb78384/ : SUCCESS in 20m 13s (non-voting)\n- swift-probetests-centos-7 http://logs.openstack.org/86/672186/2/check/swift-probetests-centos-7/17711c5/ : SUCCESS in 34m 49s\n- swift-dsvm-functional http://logs.openstack.org/86/672186/2/check/swift-dsvm-functional/e97138b/ : SUCCESS in 25m 54s\n- swift-dsvm-functional-py3 http://logs.openstack.org/86/672186/2/check/swift-dsvm-functional-py3/db60a3f/ : SUCCESS in 32m 15s\n- swift-dsvm-functional-ipv6 http://logs.openstack.org/86/672186/2/check/swift-dsvm-functional-ipv6/d0e22a0/ : SUCCESS in 33m 20s\n- swift-tox-lower-constraints http://logs.openstack.org/86/672186/2/check/swift-tox-lower-constraints/1cb7a25/ : SUCCESS in 11m 57s\n- openstack-tox-pep8 http://logs.openstack.org/86/672186/2/check/openstack-tox-pep8/188e8a0/ : SUCCESS in 5m 55s\n- swift-multinode-rolling-upgrade http://logs.openstack.org/86/672186/2/check/swift-multinode-rolling-upgrade/7a85cb3/ : SUCCESS in 18m 02s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"be69d4d135bee764863c8575678b5439488902d1","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2019-07-23 17:36:39.000000000","message":"Patch Set 2: Code-Review+2\n\n(7 comments)\n\nI\u0027m surprise more tests didn\u0027t break!\n\nMy opinion is strongly leaning towards this is a solid change in a positive direction!\n\nPTL would like additional input - please chime in!","accounts_in_message":[],"_revision_number":2},{"id":"c54794f690bbf2ad8149183fdf7c96912cc2db06","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-23 17:59:25.000000000","message":"Patch Set 2: Workflow-1\n\n(6 comments)\n\nIDK -- I think some of these test changes are indicating that I need to think this through at least a *little* more...","accounts_in_message":[],"_revision_number":2},{"id":"be1bbf1e3b482153bfa563dcd8ed61e1109cc991","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2019-07-23 18:05:54.000000000","message":"Patch Set 2:\n\n(4 comments)\n\nthanks for calling attention where needed PTL!  :hugs:","accounts_in_message":[],"_revision_number":2},{"id":"16ed3a30c9fd0c73d5c539c700d9ee1bd1c84e6a","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-24 20:56:36.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"0f2d2c98497d665dba9406ba05cff73b4915a874","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-24 20:58:39.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"aaa8f43b530f540c8b28f281e4d824502083c881","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-24 21:34:27.000000000","message":"Patch Set 4: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- openstack-tox-docs http://logs.openstack.org/86/672186/4/check/openstack-tox-docs/f80837d/html/ : SUCCESS in 4m 59s\n- swift-build-image http://logs.openstack.org/86/672186/4/check/swift-build-image/f9b2ddf/ : SUCCESS in 9m 05s (non-voting)\n- swift-tox-py27 http://logs.openstack.org/86/672186/4/check/swift-tox-py27/f910e66/ : SUCCESS in 10m 49s\n- swift-tox-py36 http://logs.openstack.org/86/672186/4/check/swift-tox-py36/db1b971/ : SUCCESS in 11m 14s\n- swift-tox-py37 http://logs.openstack.org/86/672186/4/check/swift-tox-py37/b9720d0/ : SUCCESS in 13m 24s\n- swift-tox-func http://logs.openstack.org/86/672186/4/check/swift-tox-func/7619806/ : SUCCESS in 9m 36s\n- swift-tox-func-encryption http://logs.openstack.org/86/672186/4/check/swift-tox-func-encryption/2211820/ : SUCCESS in 9m 07s\n- swift-tox-func-domain-remap-staticweb http://logs.openstack.org/86/672186/4/check/swift-tox-func-domain-remap-staticweb/4bb96d3/ : SUCCESS in 9m 50s\n- swift-tox-func-ec http://logs.openstack.org/86/672186/4/check/swift-tox-func-ec/7ca62eb/ : SUCCESS in 10m 35s\n- swift-tox-func-s3api http://logs.openstack.org/86/672186/4/check/swift-tox-func-s3api/c97a835/ : SUCCESS in 11m 40s\n- swift-tox-func-py37 http://logs.openstack.org/86/672186/4/check/swift-tox-func-py37/53d5855/ : SUCCESS in 10m 46s\n- swift-tox-func-domain-remap-staticweb-py37 http://logs.openstack.org/86/672186/4/check/swift-tox-func-domain-remap-staticweb-py37/73a61b1/ : SUCCESS in 8m 27s\n- swift-tox-func-ec-py37 http://logs.openstack.org/86/672186/4/check/swift-tox-func-ec-py37/46bbbb0/ : SUCCESS in 9m 40s\n- swift-tox-func-s3api-ceph-s3tests-tempauth http://logs.openstack.org/86/672186/4/check/swift-tox-func-s3api-ceph-s3tests-tempauth/2ea2363/ : SUCCESS in 12m 14s (non-voting)\n- swift-probetests-centos-7 http://logs.openstack.org/86/672186/4/check/swift-probetests-centos-7/35fa805/ : SUCCESS in 31m 10s\n- swift-dsvm-functional http://logs.openstack.org/86/672186/4/check/swift-dsvm-functional/94ff159/ : SUCCESS in 29m 12s\n- swift-dsvm-functional-py3 http://logs.openstack.org/86/672186/4/check/swift-dsvm-functional-py3/ab754f9/ : SUCCESS in 27m 51s\n- swift-dsvm-functional-ipv6 http://logs.openstack.org/86/672186/4/check/swift-dsvm-functional-ipv6/92cd47f/ : SUCCESS in 28m 34s\n- swift-tox-lower-constraints http://logs.openstack.org/86/672186/4/check/swift-tox-lower-constraints/7b56455/ : SUCCESS in 12m 42s\n- openstack-tox-pep8 http://logs.openstack.org/86/672186/4/check/openstack-tox-pep8/3d4ede9/ : FAILURE in 5m 50s\n- swift-multinode-rolling-upgrade http://logs.openstack.org/86/672186/4/check/swift-multinode-rolling-upgrade/069dc29/ : SUCCESS in 16m 25s","accounts_in_message":[],"_revision_number":4},{"id":"82cd2b98551a1ecdea53916ca2a6e2f065b44937","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-25 00:41:46.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"14ca38dd3371d54e5e218b1dc296f740bc3ba094","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-25 01:23:18.000000000","message":"Patch Set 5: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs http://logs.openstack.org/86/672186/5/check/openstack-tox-docs/4a1176d/html/ : SUCCESS in 5m 19s\n- swift-build-image http://logs.openstack.org/86/672186/5/check/swift-build-image/4c77513/ : SUCCESS in 7m 43s (non-voting)\n- swift-tox-py27 http://logs.openstack.org/86/672186/5/check/swift-tox-py27/4d4476b/ : SUCCESS in 9m 59s\n- swift-tox-py36 http://logs.openstack.org/86/672186/5/check/swift-tox-py36/f005008/ : SUCCESS in 10m 17s\n- swift-tox-py37 http://logs.openstack.org/86/672186/5/check/swift-tox-py37/11f894a/ : SUCCESS in 11m 25s\n- swift-tox-func http://logs.openstack.org/86/672186/5/check/swift-tox-func/a158bd0/ : SUCCESS in 8m 32s\n- swift-tox-func-encryption http://logs.openstack.org/86/672186/5/check/swift-tox-func-encryption/3d20b68/ : SUCCESS in 8m 52s\n- swift-tox-func-domain-remap-staticweb http://logs.openstack.org/86/672186/5/check/swift-tox-func-domain-remap-staticweb/676df4c/ : SUCCESS in 7m 42s\n- swift-tox-func-ec http://logs.openstack.org/86/672186/5/check/swift-tox-func-ec/6bc1dcf/ : SUCCESS in 8m 59s\n- swift-tox-func-s3api http://logs.openstack.org/86/672186/5/check/swift-tox-func-s3api/67cc8aa/ : SUCCESS in 11m 41s\n- swift-tox-func-py37 http://logs.openstack.org/86/672186/5/check/swift-tox-func-py37/3c335f1/ : SUCCESS in 9m 30s\n- swift-tox-func-domain-remap-staticweb-py37 http://logs.openstack.org/86/672186/5/check/swift-tox-func-domain-remap-staticweb-py37/153cbba/ : SUCCESS in 7m 00s\n- swift-tox-func-ec-py37 http://logs.openstack.org/86/672186/5/check/swift-tox-func-ec-py37/5605d2e/ : SUCCESS in 7m 23s\n- swift-tox-func-s3api-ceph-s3tests-tempauth http://logs.openstack.org/86/672186/5/check/swift-tox-func-s3api-ceph-s3tests-tempauth/07b5973/ : SUCCESS in 13m 12s (non-voting)\n- swift-probetests-centos-7 http://logs.openstack.org/86/672186/5/check/swift-probetests-centos-7/cce0991/ : SUCCESS in 33m 23s\n- swift-dsvm-functional http://logs.openstack.org/86/672186/5/check/swift-dsvm-functional/6f902da/ : SUCCESS in 26m 29s\n- swift-dsvm-functional-py3 http://logs.openstack.org/86/672186/5/check/swift-dsvm-functional-py3/3bc50a3/ : SUCCESS in 34m 52s\n- swift-dsvm-functional-ipv6 http://logs.openstack.org/86/672186/5/check/swift-dsvm-functional-ipv6/d499f96/ : SUCCESS in 28m 45s\n- swift-tox-lower-constraints http://logs.openstack.org/86/672186/5/check/swift-tox-lower-constraints/ca0de91/ : SUCCESS in 11m 38s\n- openstack-tox-pep8 http://logs.openstack.org/86/672186/5/check/openstack-tox-pep8/746ab29/ : SUCCESS in 5m 08s\n- swift-multinode-rolling-upgrade http://logs.openstack.org/86/672186/5/check/swift-multinode-rolling-upgrade/9704d7a/ : SUCCESS in 15m 00s","accounts_in_message":[],"_revision_number":5},{"id":"dc80187608b3988f2a21e0b92da555ecfebd506c","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-25 01:29:39.000000000","message":"Patch Set 5:\n\n(7 comments)","accounts_in_message":[],"_revision_number":5},{"id":"8ce0503c43a71469fdb5571be45fa45c513b51e8","author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"date":"2019-07-25 12:26:54.000000000","message":"Patch Set 5: Code-Review+2\n\nI think this patch gets us to the latest described in the etherpad[1]. reconstructing from non-durable fragments should be a follow-on patch. For now I\u0027ve written a new probe test[2] to confirm the 404 behavior.\n\n[1] - https://etherpad.openstack.org/p/swift-fail-responses\n[2] - http://paste.openstack.org/show/754847/","accounts_in_message":[],"_revision_number":5},{"id":"b9f26a36ff9919b4771368d634531d212afd40ba","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-25 15:39:01.000000000","message":"Patch Set 5:\n\n(2 comments)\n\nThanks for all the probe tests, Thiago! Adding that one, too.","accounts_in_message":[],"_revision_number":5},{"id":"6a9d5524c6f9ca22fde0b271a128bf55bc1453a9","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-25 15:39:15.000000000","message":"Uploaded patch set 6.","accounts_in_message":[],"_revision_number":6},{"id":"4420dd409b08f2ee5a47cfeee8ef201a93d9a55f","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-25 18:03:47.000000000","message":"Patch Set 6: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs http://logs.openstack.org/86/672186/6/check/openstack-tox-docs/0687d6b/html/ : SUCCESS in 6m 49s\n- swift-build-image http://logs.openstack.org/86/672186/6/check/swift-build-image/1252e22/ : SUCCESS in 8m 50s (non-voting)\n- swift-tox-py27 http://logs.openstack.org/86/672186/6/check/swift-tox-py27/716cf49/ : SUCCESS in 11m 59s\n- swift-tox-py36 http://logs.openstack.org/86/672186/6/check/swift-tox-py36/e30de4d/ : SUCCESS in 9m 37s\n- swift-tox-py37 http://logs.openstack.org/86/672186/6/check/swift-tox-py37/a5017af/ : SUCCESS in 10m 31s\n- swift-tox-func http://logs.openstack.org/86/672186/6/check/swift-tox-func/11d4f24/ : SUCCESS in 9m 18s\n- swift-tox-func-encryption http://logs.openstack.org/86/672186/6/check/swift-tox-func-encryption/fefd071/ : SUCCESS in 10m 09s\n- swift-tox-func-domain-remap-staticweb http://logs.openstack.org/86/672186/6/check/swift-tox-func-domain-remap-staticweb/2210d56/ : SUCCESS in 9m 18s\n- swift-tox-func-ec http://logs.openstack.org/86/672186/6/check/swift-tox-func-ec/7fc1d66/ : SUCCESS in 9m 52s\n- swift-tox-func-s3api http://logs.openstack.org/86/672186/6/check/swift-tox-func-s3api/4a203ba/ : SUCCESS in 12m 11s\n- swift-tox-func-py37 http://logs.openstack.org/86/672186/6/check/swift-tox-func-py37/710f824/ : SUCCESS in 9m 54s\n- swift-tox-func-domain-remap-staticweb-py37 http://logs.openstack.org/86/672186/6/check/swift-tox-func-domain-remap-staticweb-py37/cf27581/ : SUCCESS in 8m 17s\n- swift-tox-func-ec-py37 http://logs.openstack.org/86/672186/6/check/swift-tox-func-ec-py37/838b3a8/ : SUCCESS in 8m 31s\n- swift-tox-func-s3api-ceph-s3tests-tempauth http://logs.openstack.org/86/672186/6/check/swift-tox-func-s3api-ceph-s3tests-tempauth/a8d686d/ : SUCCESS in 11m 27s (non-voting)\n- swift-probetests-centos-7 http://logs.openstack.org/86/672186/6/check/swift-probetests-centos-7/f2a2b53/ : SUCCESS in 33m 12s\n- swift-dsvm-functional http://logs.openstack.org/86/672186/6/check/swift-dsvm-functional/9f5caca/ : SUCCESS in 28m 20s\n- swift-dsvm-functional-py3 http://logs.openstack.org/86/672186/6/check/swift-dsvm-functional-py3/86d06a6/ : SUCCESS in 31m 09s\n- swift-dsvm-functional-ipv6 http://logs.openstack.org/86/672186/6/check/swift-dsvm-functional-ipv6/8c8306c/ : SUCCESS in 36m 32s\n- swift-tox-lower-constraints http://logs.openstack.org/86/672186/6/check/swift-tox-lower-constraints/a483c9d/ : SUCCESS in 13m 15s\n- openstack-tox-pep8 http://logs.openstack.org/86/672186/6/check/openstack-tox-pep8/be1d1e6/ : SUCCESS in 8m 55s\n- swift-multinode-rolling-upgrade http://logs.openstack.org/86/672186/6/check/swift-multinode-rolling-upgrade/c97de92/ : SUCCESS in 18m 05s","accounts_in_message":[],"_revision_number":6},{"id":"2fef05016dabdf1daf78ea1e1b571862bcf0d95a","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-07-29 23:18:56.000000000","message":"Uploaded patch set 7.","accounts_in_message":[],"_revision_number":7},{"id":"a3d38687b810062ca139d79268f8ec3758ecf621","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-30 00:17:32.000000000","message":"Patch Set 7: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs http://logs.openstack.org/86/672186/7/check/openstack-tox-docs/b63c2d4/html/ : SUCCESS in 6m 24s\n- grenade-py3 http://logs.openstack.org/86/672186/7/check/grenade-py3/6c39362/ : SUCCESS in 53m 48s\n- tempest-integrated-object-storage http://logs.openstack.org/86/672186/7/check/tempest-integrated-object-storage/48e6307/ : SUCCESS in 46m 17s\n- swift-build-image http://logs.openstack.org/86/672186/7/check/swift-build-image/0bf72c2/ : SUCCESS in 9m 10s (non-voting)\n- swift-tox-py27 http://logs.openstack.org/86/672186/7/check/swift-tox-py27/9d1f126/ : SUCCESS in 12m 33s\n- swift-tox-py36 http://logs.openstack.org/86/672186/7/check/swift-tox-py36/f55bc80/ : SUCCESS in 10m 34s\n- swift-tox-py37 http://logs.openstack.org/86/672186/7/check/swift-tox-py37/a1ddc9b/ : SUCCESS in 11m 35s\n- swift-tox-func http://logs.openstack.org/86/672186/7/check/swift-tox-func/c8fd16b/ : SUCCESS in 9m 07s\n- swift-tox-func-encryption http://logs.openstack.org/86/672186/7/check/swift-tox-func-encryption/643e7fc/ : SUCCESS in 9m 30s\n- swift-tox-func-domain-remap-staticweb http://logs.openstack.org/86/672186/7/check/swift-tox-func-domain-remap-staticweb/8a28609/ : SUCCESS in 9m 59s\n- swift-tox-func-ec http://logs.openstack.org/86/672186/7/check/swift-tox-func-ec/ec8de86/ : SUCCESS in 9m 28s\n- swift-tox-func-s3api http://logs.openstack.org/86/672186/7/check/swift-tox-func-s3api/fc5d9ea/ : SUCCESS in 11m 31s\n- swift-tox-func-py37 http://logs.openstack.org/86/672186/7/check/swift-tox-func-py37/aede45b/ : SUCCESS in 10m 09s\n- swift-tox-func-domain-remap-staticweb-py37 http://logs.openstack.org/86/672186/7/check/swift-tox-func-domain-remap-staticweb-py37/71c720b/ : SUCCESS in 10m 15s\n- swift-tox-func-ec-py37 http://logs.openstack.org/86/672186/7/check/swift-tox-func-ec-py37/6bd94f5/ : SUCCESS in 8m 33s\n- swift-tox-func-s3api-ceph-s3tests-tempauth http://logs.openstack.org/86/672186/7/check/swift-tox-func-s3api-ceph-s3tests-tempauth/28f5ecc/ : SUCCESS in 13m 40s (non-voting)\n- swift-probetests-centos-7 http://logs.openstack.org/86/672186/7/check/swift-probetests-centos-7/be5fcd6/ : SUCCESS in 34m 46s\n- swift-dsvm-functional http://logs.openstack.org/86/672186/7/check/swift-dsvm-functional/f3e229b/ : SUCCESS in 29m 19s\n- swift-dsvm-functional-py3 http://logs.openstack.org/86/672186/7/check/swift-dsvm-functional-py3/6da03b4/ : SUCCESS in 31m 45s\n- swift-dsvm-functional-ipv6 http://logs.openstack.org/86/672186/7/check/swift-dsvm-functional-ipv6/6d9f5d2/ : SUCCESS in 33m 41s\n- swift-tox-lower-constraints http://logs.openstack.org/86/672186/7/check/swift-tox-lower-constraints/698bdb8/ : SUCCESS in 12m 42s\n- openstack-tox-pep8 http://logs.openstack.org/86/672186/7/check/openstack-tox-pep8/a173d71/ : SUCCESS in 9m 29s\n- swift-multinode-rolling-upgrade http://logs.openstack.org/86/672186/7/check/swift-multinode-rolling-upgrade/07c9fb5/ : SUCCESS in 25m 59s","accounts_in_message":[],"_revision_number":7},{"id":"c0202ae4c65bcba778a342a0edf45f97672273aa","author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"date":"2019-07-30 10:50:29.000000000","message":"Patch Set 7:\n\n(1 comment)","accounts_in_message":[],"_revision_number":7},{"id":"b74ed7a5c51c8168d4a15f4fb1c09915e6588175","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2019-07-30 13:02:39.000000000","message":"Patch Set 7: Code-Review-1\n\n(3 comments)\n\nI\u0027m a little lost on this, hopefully we can find a reasonable way forward in the meeting tomorrow?","accounts_in_message":[],"_revision_number":7},{"id":"a3df929df83644e6a2a16d5a54ee34dadf60060b","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-08-01 21:08:42.000000000","message":"Uploaded patch set 8.","accounts_in_message":[],"_revision_number":8},{"id":"1bffbabbb6570f0cd15c1dad91595f4e92227d10","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2019-08-01 21:12:25.000000000","message":"Patch Set 8:\n\nDropped the option; I agree, it\u0027s hard to come up with a solid justification for it. Kept an UpgradeImpact to remind me to mention a potential uptick in 503s in release notes.\n\nAdded a test to demonstrate replicated behavior.","accounts_in_message":[],"_revision_number":8},{"id":"63f2178f2b24ccf0e0d803baf022506cf7ef190d","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2019-08-01 22:22:06.000000000","message":"Patch Set 8: Code-Review+2\n\nWFM!\n\nFWIW I prefer to add tests to controller/test_obj instead of the already _massive_ proxy/test_server when possible:\n\n\tdiff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py\n\tindex b98b6ad0f..cab78bcd7 100644\n\t--- a/test/unit/proxy/controllers/test_obj.py\n\t+++ b/test/unit/proxy/controllers/test_obj.py\n\t@@ -1570,6 +1570,14 @@ class TestReplicatedObjController(CommonObjectControllerMixin,\n\t\t     resp \u003d req.get_response(self.app)\n\t\t self.assertEqual(resp.status_int, 404)\n\t \n\t+    def test_GET_primaries_timeout_error(self):\n\t+        req \u003d swift.common.swob.Request.blank(\u0027/v1/a/c/o\u0027)\n\t+        codes \u003d ([Timeout()] * self.obj_ring.replicas) + (\n\t+            [404] * self.obj_ring.max_more_nodes)\n\t+        with set_http_connect(*codes):\n\t+            resp \u003d req.get_response(self.app)\n\t+        self.assertEqual(resp.status_int, 503)\n\t+\n\t     def test_GET_not_found_when_404_newer(self):\n\t\t # if proxy receives a 404, it keeps waiting for other connections until\n\t\t # max number of nodes in hopes of finding an object, but if 404 is\n\nP.S. I\u0027m always terrified that GreenAsyncPile hangs when I throw an exception in that _make_node_request block -\u003e https://review.opendev.org/#/c/623360/","accounts_in_message":[],"_revision_number":8},{"id":"80d535aad03d40a78bb2277ef1d282aa9d6abe3d","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-02 02:07:36.000000000","message":"Patch Set 8: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://logs.opendev.org/86/672186/8/check/openstack-tox-docs/ce8858d/html/ : SUCCESS in 4m 09s\n- grenade-py3 https://logs.opendev.org/86/672186/8/check/grenade-py3/4a3ecff/ : SUCCESS in 1h 03m 57s\n- tempest-integrated-object-storage https://logs.opendev.org/86/672186/8/check/tempest-integrated-object-storage/ed93657/ : SUCCESS in 51m 17s\n- swift-build-image https://logs.opendev.org/86/672186/8/check/swift-build-image/78a0637/ : SUCCESS in 9m 03s (non-voting)\n- swift-tox-py27 https://logs.opendev.org/86/672186/8/check/swift-tox-py27/94069b4/ : SUCCESS in 11m 59s\n- swift-tox-py36 https://logs.opendev.org/86/672186/8/check/swift-tox-py36/24cd7a5/ : SUCCESS in 11m 08s\n- swift-tox-py37 https://logs.opendev.org/86/672186/8/check/swift-tox-py37/573a60f/ : SUCCESS in 10m 20s\n- swift-tox-func https://logs.opendev.org/86/672186/8/check/swift-tox-func/6c07765/ : SUCCESS in 9m 40s\n- swift-tox-func-encryption https://logs.opendev.org/86/672186/8/check/swift-tox-func-encryption/a1511e8/ : SUCCESS in 9m 42s\n- swift-tox-func-domain-remap-staticweb https://logs.opendev.org/86/672186/8/check/swift-tox-func-domain-remap-staticweb/066e1cd/ : SUCCESS in 11m 17s\n- swift-tox-func-ec https://logs.opendev.org/86/672186/8/check/swift-tox-func-ec/28d784f/ : SUCCESS in 10m 11s\n- swift-tox-func-s3api https://logs.opendev.org/86/672186/8/check/swift-tox-func-s3api/9833d72/ : SUCCESS in 13m 32s\n- swift-tox-func-py37 https://logs.opendev.org/86/672186/8/check/swift-tox-func-py37/f1748e0/ : SUCCESS in 11m 25s\n- swift-tox-func-domain-remap-staticweb-py37 https://logs.opendev.org/86/672186/8/check/swift-tox-func-domain-remap-staticweb-py37/15d9474/ : SUCCESS in 10m 51s\n- swift-tox-func-ec-py37 https://logs.opendev.org/86/672186/8/check/swift-tox-func-ec-py37/81346cd/ : SUCCESS in 8m 59s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://logs.opendev.org/86/672186/8/check/swift-tox-func-s3api-ceph-s3tests-tempauth/b4c9b57/ : SUCCESS in 15m 46s (non-voting)\n- swift-probetests-centos-7 https://logs.opendev.org/86/672186/8/check/swift-probetests-centos-7/628f04c/ : SUCCESS in 32m 14s\n- swift-dsvm-functional https://logs.opendev.org/86/672186/8/check/swift-dsvm-functional/2c055f9/ : SUCCESS in 32m 59s\n- swift-dsvm-functional-py3 https://logs.opendev.org/86/672186/8/check/swift-dsvm-functional-py3/7d4c46b/ : SUCCESS in 33m 07s\n- swift-dsvm-functional-ipv6 https://logs.opendev.org/86/672186/8/check/swift-dsvm-functional-ipv6/443881c/ : SUCCESS in 29m 57s\n- swift-tox-lower-constraints https://logs.opendev.org/86/672186/8/check/swift-tox-lower-constraints/ac73363/ : SUCCESS in 14m 27s\n- openstack-tox-pep8 https://logs.opendev.org/86/672186/8/check/openstack-tox-pep8/44a148a/ : SUCCESS in 5m 31s\n- swift-multinode-rolling-upgrade https://logs.opendev.org/86/672186/8/check/swift-multinode-rolling-upgrade/64683ee/ : SUCCESS in 19m 44s","accounts_in_message":[],"_revision_number":8},{"id":"5e4462dd4168019dd54413b686981e9c1064c018","author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"date":"2019-08-02 11:44:25.000000000","message":"Patch Set 8: Code-Review+2 Workflow+1\n\nLet\u0027s land this and we can adjust tests on a follow-on patch...","accounts_in_message":[],"_revision_number":8},{"id":"b42ba1a70b7f7176473040e44edc1c0abf21bed0","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-02 11:44:48.000000000","message":"Patch Set 8: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":8},{"id":"c12491faf0aa37f4219b450ee8a06515525b8166","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-02 13:17:53.000000000","message":"Patch Set 8: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- openstack-tox-docs https://logs.opendev.org/86/672186/8/gate/openstack-tox-docs/d95ff6e/html/ : SUCCESS in 6m 05s\n- grenade-py3 https://logs.opendev.org/86/672186/8/gate/grenade-py3/09437c8/ : SUCCESS in 1h 05m 09s\n- tempest-integrated-object-storage https://logs.opendev.org/86/672186/8/gate/tempest-integrated-object-storage/bf0a2a0/ : SUCCESS in 50m 12s\n- swift-upload-image https://logs.opendev.org/86/672186/8/gate/swift-upload-image/2296b0a/ : SUCCESS in 11m 04s (non-voting)\n- swift-tox-py27 https://logs.opendev.org/86/672186/8/gate/swift-tox-py27/fa7ed99/ : SUCCESS in 15m 29s\n- swift-tox-py36 https://logs.opendev.org/86/672186/8/gate/swift-tox-py36/8d2cc2b/ : SUCCESS in 12m 24s\n- swift-tox-py37 https://logs.opendev.org/86/672186/8/gate/swift-tox-py37/3f2ab6e/ : SUCCESS in 11m 30s\n- swift-tox-func https://logs.opendev.org/86/672186/8/gate/swift-tox-func/5bbded5/ : SUCCESS in 10m 34s\n- swift-tox-func-encryption https://logs.opendev.org/86/672186/8/gate/swift-tox-func-encryption/91f8bda/ : SUCCESS in 8m 53s\n- swift-tox-func-domain-remap-staticweb https://logs.opendev.org/86/672186/8/gate/swift-tox-func-domain-remap-staticweb/9be84f5/ : SUCCESS in 10m 07s\n- swift-tox-func-ec https://logs.opendev.org/86/672186/8/gate/swift-tox-func-ec/765972b/ : SUCCESS in 9m 56s\n- swift-tox-func-s3api https://logs.opendev.org/86/672186/8/gate/swift-tox-func-s3api/1c94043/ : SUCCESS in 15m 27s\n- swift-tox-func-py37 https://logs.opendev.org/86/672186/8/gate/swift-tox-func-py37/5adf93a/ : SUCCESS in 10m 57s\n- swift-tox-func-domain-remap-staticweb-py37 https://logs.opendev.org/86/672186/8/gate/swift-tox-func-domain-remap-staticweb-py37/1290dce/ : SUCCESS in 8m 38s\n- swift-tox-func-ec-py37 https://logs.opendev.org/86/672186/8/gate/swift-tox-func-ec-py37/a168888/ : SUCCESS in 9m 42s\n- swift-probetests-centos-7 https://logs.opendev.org/86/672186/8/gate/swift-probetests-centos-7/0c9f83b/ : SUCCESS in 34m 41s\n- swift-dsvm-functional https://logs.opendev.org/86/672186/8/gate/swift-dsvm-functional/fbec516/ : SUCCESS in 30m 59s\n- swift-dsvm-functional-py3 https://logs.opendev.org/86/672186/8/gate/swift-dsvm-functional-py3/4ee5a6e/ : SUCCESS in 30m 24s\n- swift-dsvm-functional-ipv6 https://logs.opendev.org/86/672186/8/gate/swift-dsvm-functional-ipv6/7ae866a/ : SUCCESS in 28m 55s\n- swift-tox-lower-constraints https://logs.opendev.org/86/672186/8/gate/swift-tox-lower-constraints/9378441/ : SUCCESS in 13m 04s\n- openstack-tox-pep8 https://logs.opendev.org/86/672186/8/gate/openstack-tox-pep8/c17d923/ : SUCCESS in 7m 37s","accounts_in_message":[],"_revision_number":8},{"id":"fda778a3d558e3d1bc64e5553e517352652c688e","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-02 13:17:54.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":8},{"id":"ea8543aa17164c806a371c857c5fcd1865c45944","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-02 13:23:27.000000000","message":"Patch Set 8:\n\nBuild succeeded (promote pipeline).\n\n- swift-promote-image https://logs.opendev.org/86/672186/8/promote/swift-promote-image/092c409/ : SUCCESS in 1m 43s (non-voting)","accounts_in_message":[],"_revision_number":8}],"current_revision_number":8,"current_revision":"3189410f9d5be3a845dbd47fdd77584f4b76da8a","revisions":{"67f1a6314c6ccb3be61899d1962d7943bad49f93":{"kind":"REWORK","_number":1,"created":"2019-07-22 22:21:20.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/1"}}},"commit":{"parents":[{"commit":"ef5a37c2bf5a32b0fa6404974aa76332a88c0841","subject":"slo: Better handle non-manifest responses when refetching manifest","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ef5a37c2bf5a32b0fa6404974aa76332a88c0841"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 22:19:33.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/67f1a6314c6ccb3be61899d1962d7943bad49f93"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/67f1a6314c6ccb3be61899d1962d7943bad49f93"}]},"branch":"refs/heads/master"},"ed39dbdfea0fcb885c1f3f55ae67d463e73de1af":{"kind":"REWORK","_number":2,"created":"2019-07-23 14:27:00.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/2"}}},"commit":{"parents":[{"commit":"ef5a37c2bf5a32b0fa6404974aa76332a88c0841","subject":"slo: Better handle non-manifest responses when refetching manifest","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ef5a37c2bf5a32b0fa6404974aa76332a88c0841"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-23 14:26:36.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ed39dbdfea0fcb885c1f3f55ae67d463e73de1af"}]},"branch":"refs/heads/master"},"aea2da88ccfcd14517bdcf71de2994d8d455c048":{"kind":"REWORK","_number":3,"created":"2019-07-24 20:56:36.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/3"}}},"commit":{"parents":[{"commit":"d6e790d1b71139623ee06f767a16f3adc576ffbb","subject":"Merge \"Fix up errno checking\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d6e790d1b71139623ee06f767a16f3adc576ffbb"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-24 20:56:27.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/aea2da88ccfcd14517bdcf71de2994d8d455c048"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/aea2da88ccfcd14517bdcf71de2994d8d455c048"}]},"branch":"refs/heads/master"},"8fc252e6a42c647f53cdf364dfa4dfa952c28c1d":{"kind":"REWORK","_number":4,"created":"2019-07-24 20:58:39.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/4"}}},"commit":{"parents":[{"commit":"d6e790d1b71139623ee06f767a16f3adc576ffbb","subject":"Merge \"Fix up errno checking\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d6e790d1b71139623ee06f767a16f3adc576ffbb"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-24 20:57:40.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nCo-Authored-By: Thiago da Silva \u003cthiagodasilva@gmail.com\u003e\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/8fc252e6a42c647f53cdf364dfa4dfa952c28c1d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/8fc252e6a42c647f53cdf364dfa4dfa952c28c1d"}]},"branch":"refs/heads/master"},"79617532d0b3ea4d31350bd004276c466b446a62":{"kind":"REWORK","_number":5,"created":"2019-07-25 00:41:46.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/5"}}},"commit":{"parents":[{"commit":"d6e790d1b71139623ee06f767a16f3adc576ffbb","subject":"Merge \"Fix up errno checking\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d6e790d1b71139623ee06f767a16f3adc576ffbb"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-25 00:41:27.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nCo-Authored-By: Thiago da Silva \u003cthiagodasilva@gmail.com\u003e\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nCloses-Bug: #1837819\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/79617532d0b3ea4d31350bd004276c466b446a62"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/79617532d0b3ea4d31350bd004276c466b446a62"}]},"branch":"refs/heads/master"},"2e3a069b90c6e358ebfeed302c32efafd33ae37c":{"kind":"REWORK","_number":6,"created":"2019-07-25 15:39:15.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/6"}}},"commit":{"parents":[{"commit":"d6e790d1b71139623ee06f767a16f3adc576ffbb","subject":"Merge \"Fix up errno checking\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d6e790d1b71139623ee06f767a16f3adc576ffbb"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-25 15:37:32.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nCo-Authored-By: Thiago da Silva \u003cthiagodasilva@gmail.com\u003e\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nCloses-Bug: #1837819\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/2e3a069b90c6e358ebfeed302c32efafd33ae37c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/2e3a069b90c6e358ebfeed302c32efafd33ae37c"}]},"branch":"refs/heads/master"},"b29d43397e3ed8ada1c7ffa16120faf193675db6":{"kind":"REWORK","_number":7,"created":"2019-07-29 23:18:56.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/7","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/7","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/7"}}},"commit":{"parents":[{"commit":"d6e790d1b71139623ee06f767a16f3adc576ffbb","subject":"Merge \"Fix up errno checking\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d6e790d1b71139623ee06f767a16f3adc576ffbb"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-29 23:12:20.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nUpgradeImpact:\n--------------\nThis may cause an increase in 503 Service Unavailable responses served\nby proxy-servers. This should more accurately reflect the state of the\nsystem. However, if you need to return to the previous behavior where\nsuch requests would have been served with a 404 Not Found response, you\ncan set\n\n    consider_404s_from_object_handoffs \u003d True\n\nin your proxy-server.conf\n\nCo-Authored-By: Thiago da Silva \u003cthiagodasilva@gmail.com\u003e\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nCloses-Bug: #1837819\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/b29d43397e3ed8ada1c7ffa16120faf193675db6"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/b29d43397e3ed8ada1c7ffa16120faf193675db6"}]},"branch":"refs/heads/master"},"3189410f9d5be3a845dbd47fdd77584f4b76da8a":{"kind":"REWORK","_number":8,"created":"2019-08-01 21:08:42.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/86/672186/8","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/86/672186/8","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/86/672186/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/86/672186/8"}}},"commit":{"parents":[{"commit":"d6e790d1b71139623ee06f767a16f3adc576ffbb","subject":"Merge \"Fix up errno checking\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d6e790d1b71139623ee06f767a16f3adc576ffbb"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-07-22 19:38:30.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2019-08-01 21:07:39.000000000","tz":-420},"subject":"Ignore 404s from handoffs for objects when calculating quorum","message":"Ignore 404s from handoffs for objects when calculating quorum\n\nWe previously realized we needed to do that for accounts and containers\nwhere the consequences of treating the 404 as authoritative were more\nobvious: we\u0027d cache the non-existence which prevented writes until it\nfell out of cache.\n\nThe same basic logic applies for objects, though: if we see\n\n    (Timeout, Timeout, Timeout, 404, 404, 404)\n\non a triple-replica policy, we don\u0027t really have any reason to think\nthat a 404 is appropriate. In fact, it seems reasonably likely that\nthere\u0027s a thundering-herd problem where there are too many concurrent\nrequests for data that *definitely is there*. By responding with a 503,\nwe apply some back-pressure to clients, who hopefully have some\nexponential backoff in their retries.\n\nThe situation gets a bit more complicated with erasure-coded data, but\nthe same basic principle applies. We\u0027re just more likely to have\nconfirmation that there *is* data out there, we just can\u0027t reconstruct\nit (right now).\n\nNote that we *still want to check* those handoffs, of course. Our\nfail-in-place strategy has us replicate (and, more recently,\nreconstruct) to handoffs to maintain durability; it\u0027d be silly *not* to\nlook.\n\nUpgradeImpact:\n--------------\nBe aware that this may cause an increase in 503 Service Unavailable\nresponses served by proxy-servers. However, this should more accurately\nreflect the state of the system.\n\nCo-Authored-By: Thiago da Silva \u003cthiagodasilva@gmail.com\u003e\nChange-Id: Ia832e9bab13167948f01bc50aa8a61974ce189fb\nCloses-Bug: #1837819\nRelated-Bug: #1833612\nRelated-Change: I53ed04b5de20c261ddd79c98c629580472e09961\nRelated-Change: Ief44ed39d97f65e4270bf73051da9a2dd0ddbaec\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/3189410f9d5be3a845dbd47fdd77584f4b76da8a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/3189410f9d5be3a845dbd47fdd77584f4b76da8a"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}