)]}'
{"id":"openstack%2Fswift~754242","triplet_id":"openstack%2Fswift~master~I29acc1302a75ed52c935f42485f775cd41648e4d","project":"openstack/swift","branch":"master","topic":"fixCrossReplicationRace","hashtags":[],"change_id":"I29acc1302a75ed52c935f42485f775cd41648e4d","subject":"Fix a race condition in case of cross-replication","status":"MERGED","created":"2020-09-25 00:57:19.000000000","updated":"2020-11-11 21:26:24.000000000","submitted":"2020-11-11 21:24:48.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":39,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"754242-1605129889196-b2dd4156","meta_rev_id":"262f94d788fdc5e451d80d52085d77253912128f","_number":754242,"virtual_id_number":754242,"owner":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"value":0,"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"tag":"autogenerated:zuul:gate","value":2,"date":"2020-11-11 21:24:48.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"date":"2020-10-22 00:21:08.000000000","_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":0,"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"all":[{"value":2,"date":"2020-11-07 00:59:50.000000000","_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"value":0,"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":2,"date":"2020-11-11 18:49:41.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"all":[{"value":0,"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"value":0,"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":1,"date":"2020-11-11 18:49:41.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-09-30 18:59:04.000000000","updated_by":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"reviewer":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"state":"REVIEWER"},{"updated":"2020-11-07 00:59:50.000000000","updated_by":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"reviewer":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"state":"REVIEWER"},{"updated":"2020-11-11 18:49:41.000000000","updated_by":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"reviewer":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"state":"REVIEWER"},{"updated":"2020-11-11 21:24:48.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"d958b30e942fa6b5014933015872d2df056056fa","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-25 00:57:19.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"dae99021404ad694f85d0ae386cf1b98de639792","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-25 02:00:53.000000000","message":"Patch Set 1: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b7dfc5cfafa44b539ad104859d7c4b23 : SUCCESS in 9m 55s\n- grenade https://zuul.opendev.org/t/openstack/build/88e111fed6604d4f853b4e88c152a500 : SUCCESS in 1h 02m 38s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/c428b08b631f402fac172f9a66fb4291 : FAILURE in 45m 11s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/565bf07175cb4fa790d40af1016e1c1e : SUCCESS in 8m 40s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/21fb0c2cb83246c885dd6e5eac0027fb : SUCCESS in 9m 17s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/4ea6954ceaf74847a9a8561fb053ec92 : SUCCESS in 13m 02s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/b497ad7837f44d93b4b2f7cea491f8db : SUCCESS in 13m 26s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/0d49c8095a1444e29f7339351552d631 : SUCCESS in 12m 54s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/1efae369b8eb432f87333f4156e6d3da : SUCCESS in 15m 28s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/b6bd9c3119294080a8b88d59b0f53827 : SUCCESS in 18m 49s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/015d392880f8410d9fefad88baeababa : SUCCESS in 16m 03s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/36172117c9eb4b2f85c143d10c12dea3 : SUCCESS in 19m 30s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/615542124ed949db915c1d0fe8cbcf4a : SUCCESS in 19m 32s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/7bc91091725540889784a3a6a0444fee : SUCCESS in 19m 27s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/a220932a0f954d3aa93055b77e9be7a9 : SUCCESS in 24m 36s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/31385e2b10234a73bb02edb5cca1974e : SUCCESS in 24m 36s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/c7581fa77fcb484dbb0f87c47ba87063 : SUCCESS in 48m 29s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/3e3c6d79f56c436faf06499cf379df79 : SUCCESS in 40m 01s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/dd6a6d7178b349f0b53076c043497ab4 : SUCCESS in 43m 00s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/ab37098f2364400fbadb1bc29fd4195a : SUCCESS in 12m 46s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/a2681db240824577b7a06f9c3b7369c0 : SUCCESS in 6m 09s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/d7bfbae9396d45388bf59d28ef6e7cb6 : POST_FAILURE in 18m 12s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/6437c0d680e84ac6bfa49df6e7c41b63 : SUCCESS in 57m 13s","accounts_in_message":[],"_revision_number":1},{"id":"aa9d0bd2cff681fbea081430cdbdde9ddf107dd0","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-09-25 02:07:59.000000000","message":"Patch Set 1:\n\nrecheck","accounts_in_message":[],"_revision_number":1},{"id":"775c99ee3bc8a78bb68a62658081709ee3552919","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-25 03:16:03.000000000","message":"Patch Set 1:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/ad650868ba22439790a0c2879500ab05 : SUCCESS in 10m 22s\n- grenade https://zuul.opendev.org/t/openstack/build/04fa12111f1949b9ba104086d38e2ef0 : SUCCESS in 1h 03m 33s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/306cab4ca0294c7996582a3fb1fca146 : SUCCESS in 40m 38s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/6dcfe970777442d08d011bc734c21c70 : SUCCESS in 10m 29s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/d79340d309f445afb9c45cc1713ab5cc : SUCCESS in 14m 20s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/0532ec1811614a03894a1f699640a5bf : SUCCESS in 26m 08s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/99ee3d9ac30441be96cbea4b80dc8e2a : SUCCESS in 18m 53s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/9176aaf8b2ea4673838543f020fb6d7d : SUCCESS in 18m 35s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/4a86743ea5f647b6bd4647f2437b934f : SUCCESS in 18m 07s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/1cee60b48ed04f0f871c504aa8acbcc6 : SUCCESS in 23m 15s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/edba969cafab44bbbdd3dd2d18343207 : SUCCESS in 27m 09s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/ce78c058917945e7ad36ce2472b042c3 : SUCCESS in 28m 10s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/3f634eebb8fb4a38b3bfef2df35a1960 : SUCCESS in 29m 32s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/60ba594952aa4a67a0f6e71cdfa11c86 : SUCCESS in 26m 46s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/8dc4c43476c44f09b08418bceb7b3e2c : SUCCESS in 26m 54s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/b30364b1c3d94c269431054b0f79baad : SUCCESS in 28m 10s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/c3ac7ace793b41d4a73287e461a0b20d : SUCCESS in 47m 16s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/b1833cec50b34b21b091296284ad3c55 : TIMED_OUT in 1h 02m 35s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/5142e31020ad425bbbc7754c6cce0741 : SUCCESS in 48m 32s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/1e2c7a6df8c3426b87573e95e1ce9455 : SUCCESS in 21m 52s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/89786066801c46ea9d1cce92bd4da611 : SUCCESS in 8m 35s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/8a1f7c8df9ce4f2290b6fd5757543c07 : TIMED_OUT in 1h 01m 06s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/d5a781135b3c44589229aa5106d908d7 : SUCCESS in 1h 01m 33s","accounts_in_message":[],"_revision_number":1},{"id":"b102d238d1f52bf6ee275127201c6887692b0186","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-09-25 03:56:53.000000000","message":"Patch Set 1:\n\nrecheck","accounts_in_message":[],"_revision_number":1},{"id":"bc47be1cd8d9d75803ab27c046b7b382d4b6e038","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-25 05:06:53.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/af772e9074154c2487aea76cc71438fd : SUCCESS in 8m 29s\n- grenade https://zuul.opendev.org/t/openstack/build/0e12eb26bf16490b9657ed0d9d393bd8 : SUCCESS in 1h 03m 16s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/9cc6ee2d53f445908c1db3857b1ebdc6 : SUCCESS in 51m 26s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/1949f72d248947cabb002c13df0e017c : SUCCESS in 9m 01s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/09ed076909e543fb8eb45f6d1e31a9f9 : SUCCESS in 11m 07s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/d373164db84e4e7eaed6ab7b42b6f1f4 : SUCCESS in 12m 10s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/00872ab76938436392fe5a9b00851349 : SUCCESS in 11m 16s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/eb2007b2b3024987b8e0d52e66ef3143 : SUCCESS in 13m 42s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/6581fb00c2e646908da5edf1ff43c2f9 : SUCCESS in 12m 32s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/210138f0ccb041e48b7f43c8875b9591 : SUCCESS in 15m 46s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/963276456728465e9905128bf616e8c6 : SUCCESS in 16m 45s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/a79b2bfe5bc04e6ea896552a0ea48c62 : SUCCESS in 19m 55s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/5558b31fb47b440f8da6c2b44b90ce47 : SUCCESS in 17m 08s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/f4ad699a643c4657b8f1ec2c02dbf639 : SUCCESS in 17m 08s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/8c28380c9f2a464692ddbaff0ab926da : SUCCESS in 20m 31s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/e41c641ab26a445191c100c7d71fa0b9 : SUCCESS in 20m 36s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/b25411e9eccc4034bf8730f021e9183b : SUCCESS in 43m 30s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/641b972a25104c1485048ed72bd417fb : SUCCESS in 44m 37s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/3306cc923d594eefadfc06a4a5b18b4c : SUCCESS in 41m 09s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/0804803d4bab4eb7bf3f9f32ff5c02fa : SUCCESS in 12m 49s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/bd1f549d802d42cba52307962586e81b : SUCCESS in 5m 31s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/a196d52751b143508b7f0e0150c1f568 : SUCCESS in 21m 20s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/d733504d145241ad912bce70b9755c77 : SUCCESS in 1h 01m 00s","accounts_in_message":[],"_revision_number":1},{"id":"13e3fe86333f2880da195c9f42e940e9d58da52b","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-25 14:50:26.000000000","message":"Patch Set 1:\n\nI\u0027m not sure how to write a test for that. I can test there is a lock, but it does not test the race condition is fixed.\n\nIt should be a probetest I guess, but it\u0027s hard to reproduce without tampering with the replicator/reconstructor process (to slow it down to get a chance to trigger the race).\n\nAnd running a replicator/reconstructor+obj-server in unit test does not seem appropriate neither.\n\nAny advices welcome :)","accounts_in_message":[],"_revision_number":1},{"id":"077b107937c9434daf98f33c8ee9923c8272cf00","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-09-25 15:17:02.000000000","message":"Patch Set 1: Code-Review+1\n\n(4 comments)\n\nI didn\u0027t realize (at first) that replication_concurrency_per_device locks the partition with a limit of 1 inside of the configured N per-device semaphores - is it desirable for outgoing replication workers to consume one of your allowed incoming ssync connections, or was that just for expediency?\n\nI think it would be more efficient to reject the incoming REPLICATE request when it comes from a node with an older ring version.  But we could probably reasonably do BOTH - there\u0027s not a lot of useful stuff I could imagine coming into a partition we\u0027re currently trying to revert.\n\nWhat sort of tests could we add if we wanted to validate alternatives (I\u0027ve never tried to probe-test different ring versions)","accounts_in_message":[],"_revision_number":1},{"id":"9ec801443c5b4a4811ea8aef04ce795b5fc4d1a7","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-25 15:39:08.000000000","message":"Patch Set 1: Workflow-1\n\nI just realized it does not only lock the partition, but also the device. Not sure though if it\u0027s good or bad","accounts_in_message":[],"_revision_number":1},{"id":"55e3e995e9032f98f14c41a43eb3bb72e00a51b5","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-28 23:18:35.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"7ae73665102ebd2f36335d23c8ac5277a9f4437c","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-28 23:22:39.000000000","message":"Patch Set 2:\n\npatchset 2 does not lock the whle device anymore. It also better handles exceptions on failure to lock the partition.\n\nI don\u0027t think comaring ring version is the proper solution because:\n- in case the ring changes on new primary while a replication is already running, the race condition can still occur\n- the object-server does not load the ring right now, it would be mandatory (thus, memory usage \u0026 co as already discussed)","accounts_in_message":[],"_revision_number":2},{"id":"ce21f624ad311a5f9b373ac995edabb5414fdb23","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-29 00:25:04.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/798fb31408f44c149c4949aad23b8de7 : SUCCESS in 8m 09s\n- grenade https://zuul.opendev.org/t/openstack/build/4054484381c5463bb75ce7e4d3eeaf8e : SUCCESS in 1h 02m 14s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/f1e077d003264dd1b205dc64b6c0f46e : SUCCESS in 52m 01s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/2172c5a7dd944a61a21656f95a6598fd : SUCCESS in 8m 25s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/d97e94fe16674e15bcdb808896d9e586 : SUCCESS in 8m 39s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/a23351cb9ee843b3909734f5ded3f0a9 : SUCCESS in 11m 43s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/34449d2675e54b03bf62572495ef696e : SUCCESS in 12m 56s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/17430799dfb9450c96151c9bbe7e7db0 : SUCCESS in 13m 04s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/5f58c12b409b476c8dbf376ac61a195c : SUCCESS in 12m 40s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/082c0d58fca14cc0842effcb27894018 : SUCCESS in 16m 08s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/68022f43924b41838e9de7f33a590b03 : SUCCESS in 16m 32s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/a4a32f54e2bc4eba985e152ea771fd3c : SUCCESS in 16m 46s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/b62d31df1e494af0a556954ef9e8ba2e : SUCCESS in 15m 43s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/d6b21dfb23264a0c8a0749a44ab946b3 : SUCCESS in 18m 19s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/6e9941e06dd242b99deb80ba48a1df30 : SUCCESS in 16m 35s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/590b8c3bcea74a05935626ed1696a6ed : SUCCESS in 22m 05s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/5c897f01f19646f68a4b4f55bef958ef : SUCCESS in 46m 05s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/cfe0bfaf082a4f2aa3ec2f131cc01fc5 : SUCCESS in 41m 18s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/08e4996ad42d4ece8458044588dbb0b0 : SUCCESS in 48m 24s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/16a901085e0e4ba19a0f1006f1f1b4b0 : SUCCESS in 12m 30s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/9f2cbb078bda432d81b64f205d29607f : SUCCESS in 5m 30s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/7ea75dcb94744852a509cf981b4fce87 : SUCCESS in 21m 06s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/be482b3742e94c36a3aa6d8f9362d42f : SUCCESS in 1h 00m 22s","accounts_in_message":[],"_revision_number":2},{"id":"8734219394792102a769b1d54b5f29a2b7af63ce","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-29 14:07:44.000000000","message":"Patch Set 2:\n\nPlease note this patch only fix the issue for ssync. It cannot apply to rsync because rsync does not take a swift lock on the remote device before starting to replicate. The lokcing mecanism is the one from the rsync daemon","accounts_in_message":[],"_revision_number":2},{"id":"cb87b8eb77e256a13cb076b409d338508bd5ad94","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-09-29 14:50:05.000000000","message":"Patch Set 2: Code-Review+2\n\n(4 comments)\n\nYou\u0027ve convinced me it\u0027s necessary, sufficient and desirable to use a serializing lock when trying to revert/delete a non-primary partition.  I agree optimization to REPLICATE requests are orthogonal.\n\nI would suggest extracting the \"partition_lock\" to a helper in disk_file would make consumers MUCH more obvious, and perhaps simplify the configuration of replication_lock_timeout.\n\nI\u0027d look to test_ssync_reciever to demonstrate at least the re-constructor and object-server\u0027s locks will serialize revert jobs.\n\nThe improved PartitionLockTimeout error handling is quite nice.\n\nSince this change addresses a high-priority bug, it could probably merge as is assuming we\u0027d like to address additional improvements and tests in followups.  I\u0027d be happy to help as soon as I set down some other stuff.  I\u0027ve added this change to tomorrow\u0027s meeting agenda:\n\nhttps://wiki.openstack.org/wiki/Meetings/Swift","accounts_in_message":[],"_revision_number":2},{"id":"32993c37f7ac3f325975b24e107fef161450f3c8","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-29 19:40:09.000000000","message":"Patch Set 2: Workflow+1","accounts_in_message":[],"_revision_number":2},{"id":"65c85a668d753ca4a58e6e734ea68d1f1da9798b","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-29 19:40:34.000000000","message":"Patch Set 2: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":2},{"id":"03594a04be3c896743681c85a38f0371bb06b379","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-29 19:41:49.000000000","message":"Patch Set 2: Workflow-1","accounts_in_message":[],"_revision_number":2},{"id":"8101eea31a19bbf841925efdbdbc06a35f578772","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-29 19:45:32.000000000","message":"Patch Set 2:\n\nThe patch does not cover every edge cases:\n- the delete_reverted_objs() in reconstructor must be covered by the partition lock (race condition observed in production)\n- if the operator set replication_concurrency_per_device\u003d0 (unlimited), there is no lock taken on partition during a SSYNC replication\n- other part of code run recursive deletion of partitions. I think they must be covered by the partition lock too","accounts_in_message":[],"_revision_number":2},{"id":"13e594916c5ac0227e995bf514bbc374ed9213ef","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-29 22:29:47.000000000","message":"Patch Set 2: Verified-2\n\nBuild failed (gate pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/4495fbf8a5db439091b6c87ca152cd01 : SUCCESS in 9m 20s\n- grenade https://zuul.opendev.org/t/openstack/build/d69930967a1f44b78f30aa0e913ce5d6 : SUCCESS in 56m 15s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/05feebdbd7f64907a234a6ce9dc76d81 : SUCCESS in 40m 17s\n- swift-upload-image https://zuul.opendev.org/t/openstack/build/4a9af5b4aa454ce8aaa85a823fb0bc78 : SUCCESS in 11m 16s (non-voting)\n- swift-upload-image-py3 https://zuul.opendev.org/t/openstack/build/2164b1c051484a75a85ffb451555107b : SUCCESS in 11m 53s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/4bdc361d559e489b8e81ad8254218c6f : SUCCESS in 15m 05s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/63edf7c416044894af11d4f39b6e3f92 : SUCCESS in 14m 17s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/177473b689544e46b84337696e35b322 : SUCCESS in 14m 46s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/f8ca0eec729849ea98425837901b89a9 : SUCCESS in 13m 55s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/49946af7f0164eb8a1cda60172326244 : SUCCESS in 18m 38s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/46e8bb8a9e1d4c79ad9081c03bf4d156 : SUCCESS in 19m 46s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/364d7c6ed11240ba9bb4df55ab8020b6 : SUCCESS in 39m 54s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/491f23aa292f43e6911817ab2c891b57 : SUCCESS in 40m 42s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/4877818965bb43d2af2432f9d8e7350d : SUCCESS in 21m 58s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/b46551fe19644e2fba88ce327bf64ef8 : SUCCESS in 23m 27s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/e336018b435c46d9a096f7fe07ce7955 : SUCCESS in 58m 12s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/efe1b6d0e12d40d59929fd5a4915476f : TIMED_OUT in 1h 03m 06s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/3f8e2f06233045759041942d54fe08e7 : TIMED_OUT in 1h 03m 55s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/e0a8befc8c8a4ecc98f835c030807629 : SUCCESS in 24m 19s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/02d8a281df7046ca84d0e197b13459ea : SUCCESS in 6m 55s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/1f0a4df256484728b6e2c1abb47277ab : SUCCESS in 1h 03m 21s","accounts_in_message":[],"_revision_number":2},{"id":"0a1805efb13eb88f5c2e19ddba065ed0e81d170d","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-30 18:32:13.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"ecc09655c7cb926a6a44d96d8a21b99d1bd41da6","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-30 18:38:25.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"41b17cef6730091e67273e60e745dc3332620a79","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-09-30 18:42:53.000000000","message":"Patch Set 2:\n\n(3 comments)\n\nI created a partition_lock helper. Thx for the remark, it helped me see an other potential failure if the operator disabled concurrency_per_device.\n\nI\u0027m quite okay with the current patch except for:\n- the missing probe test\n- maybe reconstructor.py/delete_partition() needs to be protected by partition_lock() too","accounts_in_message":[],"_revision_number":2},{"id":"5241112dc7a069f409eb32952648e99b9df3d514","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-09-30 18:59:04.000000000","message":"Patch Set 4: Code-Review+1\n\n(12 comments)\n\nlatest patch set has a lot of improvements!  thanks for sharing all the progress...","accounts_in_message":[],"_revision_number":4},{"id":"1e8b26985ccd394ea7b769c19adb018401fdf633","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-30 22:18:44.000000000","message":"Patch Set 4: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/e65205cda09c4b33ae2e364d4fd610ef : SUCCESS in 7m 54s\n- grenade https://zuul.opendev.org/t/openstack/build/dfc23e899b0a443ab9113fe11dded28b : SUCCESS in 54m 27s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/a66bec4b0ebb4f4cb3e15b123d0c17ba : SUCCESS in 51m 03s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d952a6dc00c0449e9d130f44901a8738 : SUCCESS in 8m 39s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/2d8e8225229e4cc2a5b9dc7efc49dbea : SUCCESS in 8m 46s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/ba2ed832fdcc4d56b32b43acd14b088e : FAILURE in 17m 16s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/ddf813a7178a4d2ca595fe04b1372f29 : FAILURE in 13m 41s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/e7a8c09d547748f683abf3ab14c097ae : FAILURE in 12m 51s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/13b176e459a34878af0bfc2e6e3a1088 : FAILURE in 19m 11s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/81eff97498c8412ea8cce85417ada38a : SUCCESS in 16m 21s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/5572076ea8c1439caa24ca7e959ed0ad : SUCCESS in 17m 29s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/7f9b664bf9934620b87a67b3f8d28f3a : SUCCESS in 18m 34s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/0d511c8b69cf49b88a585bd8c03bb461 : SUCCESS in 18m 42s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/e7caef12311d49bba3579c018d2ffa1d : SUCCESS in 18m 54s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/3ee4c7cc689b4acb8cb999d5fdc0a85f : SUCCESS in 17m 53s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/11e5507d8e21433cafa4bf0add8fda84 : SUCCESS in 25m 11s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/b2de1fd17a21440d891ad0d0dfde94da : SUCCESS in 47m 51s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/5614a672b5e84c8482c812e407bd45b8 : SUCCESS in 43m 50s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/e02b99464d784300af73c9ac34831a66 : SUCCESS in 43m 26s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/b423e6e754e94042801346f5cf7a7d06 : FAILURE in 13m 07s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/83c185fd69914f0b92267c118f64e3bf : SUCCESS in 7m 29s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/4a59ba685bfd477d8a3afa6399dcf3af : FAILURE in 23m 34s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/850fc275906c49a78338aeba8e8d97cb : SUCCESS in 1h 01m 15s","accounts_in_message":[],"_revision_number":4},{"id":"dc750202503bfb60f05592d5290969b5d7fcdef7","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-01 22:06:20.000000000","message":"Patch Set 4:\n\n(5 comments)","accounts_in_message":[],"_revision_number":4},{"id":"704e3d22f3e4b29ddf927bf17e0df7dd991ccf31","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-01 22:06:31.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"3fbe003f1c3caecfa1864df6b6674fc83f540080","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-02 01:52:55.000000000","message":"Patch Set 5: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c100047e3a2c4fc7b3fe0daf8978e8cc : SUCCESS in 9m 11s\n- grenade https://zuul.opendev.org/t/openstack/build/b8094e5305954eb4b689896aee82347e : SUCCESS in 54m 43s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/337d4a4314aa42b89586b54dee73fd29 : SUCCESS in 48m 27s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/ac23e89522c341788620794c41d4179b : SUCCESS in 8m 44s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/84797fbc599949b98b7d188074c79f1d : SUCCESS in 10m 22s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/f51d9e4bdb3049068b58786d9adbd28d : FAILURE in 4m 23s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/7a4715be7e5a4a9bba83789beef4b56a : SUCCESS in 12m 30s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/d62afff4ad3948c6908b26580d9e1975 : SUCCESS in 14m 39s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/2dece38180ca4115aaa64a7f2a11f754 : SUCCESS in 14m 28s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/fe925cde00da400eaa430c3ff79906f5 : SUCCESS in 29m 29s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/cb75ea0bf5814976915402873737605a : SUCCESS in 18m 26s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/be26870ef6ce475f8a704e71712020fa : SUCCESS in 31m 17s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/57d4d9af569b43c8955036a0169f0458 : SUCCESS in 16m 32s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/07c22057db904bd3a8e5f66413dba716 : SUCCESS in 16m 35s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/cea6d82746ef4782842af45a48aaab2f : SUCCESS in 17m 46s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/0827dd35d75f421688b3a7d9ac7b640b : SUCCESS in 19m 52s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/e9d816d50fe1487c8d7e030015b494f7 : SUCCESS in 1h 17m 34s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/8dc05e445e1d4428aac7c9e531b9f7df : SUCCESS in 41m 43s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/235a95c99cb045b4b2f026b2c87a799b : SUCCESS in 39m 26s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/a16020a35f4e42719c24c084b5628b20 : SUCCESS in 12m 20s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/35a6e5c3eb70463b862f8d1d4573179a : FAILURE in 4m 44s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/0c70645cdee3470798f7807e70040b66 : SUCCESS in 21m 01s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/79ba98888b294140888a8492fd13b197 : SUCCESS in 1h 04m 57s","accounts_in_message":[],"_revision_number":5},{"id":"ba901cbf3c3e40b5ef3c0f4c0f83c371ee9cd457","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-02 11:56:11.000000000","message":"Patch Set 5:\n\nrecheck","accounts_in_message":[],"_revision_number":5},{"id":"f3d42eae4fc654481253d20df680fe438ea23513","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-02 17:48:58.000000000","message":"Patch Set 5:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/b34c8c3a981b404a9618aa26dfab097b : SUCCESS in 8m 58s\n- grenade https://zuul.opendev.org/t/openstack/build/70b22e294abc48c4acc5c78d907b1db0 : SUCCESS in 57m 48s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/a15dfde34f114da187640bef022794aa : SUCCESS in 51m 33s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/096090596ae34cfd85fac3c756471519 : SUCCESS in 9m 14s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/22b5fce8541e4b478af664613cb3d0af : SUCCESS in 9m 34s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/6c9619606eee4fb6bcca3d479ae82012 : SUCCESS in 57m 09s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/2281c887211a44b1beddc7e39008c8a0 : SUCCESS in 14m 39s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/f7d154771cc4428aaad549f39833ee0d : SUCCESS in 13m 46s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/5eaae35c610648aca0f94801e0fc0168 : SUCCESS in 13m 17s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/a8975056878142d098250f53b2d112fa : SUCCESS in 19m 59s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/e784895511ac42e180bdc471c6533eb4 : SUCCESS in 19m 00s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/431aa319fcc644bba68fbbbd0986c89e : SUCCESS in 16m 01s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/7257db0ce8b74408be9e8db8bb6af9bd : SUCCESS in 20m 31s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/7205794e0d5d4bf9a2489ec5b6462f6f : SUCCESS in 20m 10s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/d230cfc56c4e460b8065dc4223e918a3 : SUCCESS in 17m 49s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/d8b7e286ff8246ccbed49b32f7097868 : SUCCESS in 23m 56s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/08818490481f450ca6cf2a8699a61b0c : FAILURE in 51m 39s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/8ed077f8c92d402ab36e01f5306cff28 : SUCCESS in 48m 31s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/b89ab01ca62c44b88baa302624852ff2 : SUCCESS in 46m 14s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/3968a9fad8d3493fa301021a3c293fea : SUCCESS in 12m 54s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/bc675b1ecadb4aa0be669a6e9fcd3c49 : SUCCESS in 7m 09s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/9e9cb479e2374f3b8987f8ab67bcd576 : SUCCESS in 27m 09s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/bb231c669a2a49d9a10a3e2f5ae9dff1 : SUCCESS in 1h 02m 37s","accounts_in_message":[],"_revision_number":5},{"id":"f245b1fc183712acb6b0f7f020175aaa25214877","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-02 18:15:18.000000000","message":"Patch Set 5:\n\nrecheck","accounts_in_message":[],"_revision_number":5},{"id":"a2badca64e42527c8799d812b040bba08f51b8a3","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-02 22:40:22.000000000","message":"Patch Set 5:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f72548cf4e3c4504a8d69412325a2c89 : SUCCESS in 9m 11s\n- grenade https://zuul.opendev.org/t/openstack/build/36a3a57e7e804e4794857b0e477eda05 : SUCCESS in 1h 03m 54s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/f1b9ca23896b4ef2810bb03648539d70 : SUCCESS in 47m 20s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/dddd19f2478049eaad0d2c3f5010f518 : SUCCESS in 8m 56s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/0438b309475748e3ac82efbb0a3e4b41 : SUCCESS in 10m 28s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/4fb70b0c73ba45b4a4c0b2da62032c85 : SUCCESS in 12m 03s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/2b66c41b6f8b4bae9fc45668d4a67077 : SUCCESS in 13m 20s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/078ff84d1fc04d238a26c0bc596ec7ef : SUCCESS in 12m 38s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/bba1d76823134c1b84980689dbe6188d : SUCCESS in 12m 44s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/a6cf9f29591441619607536dc27b8886 : SUCCESS in 17m 07s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/ca9b25d90225416aa352bd707f3e71a5 : SUCCESS in 25m 05s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/73366a7563fb471b89c2c1388b1b4df2 : FAILURE in 18m 06s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/2645f712b4d54931b15aa3044e8842cb : SUCCESS in 25m 08s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/f9907c5cb3a3485a942a213c18cf9531 : SUCCESS in 24m 41s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/892b513000f640a9a9b911c450a8230f : SUCCESS in 16m 53s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/86f692d8fa914289b08c8fce1f53f579 : SUCCESS in 21m 42s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/10e22216fe824d53a13dd4cf0ae29cf8 : FAILURE in 42m 16s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/ad80c40cf1db433fbad0d103522374d4 : SUCCESS in 33m 17s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/15eba3db9b974a85941d78e5974d3b3e : SUCCESS in 44m 47s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/69efa52a66644bd6aae65eebc4ebd348 : SUCCESS in 12m 39s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/c5f4cf402d4e4545864689164667e569 : SUCCESS in 5m 43s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/a5a541a72b5b42908d3270adb134e8ce : SUCCESS in 20m 38s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/60c5da5c38b340c5b35966fcfe903358 : SUCCESS in 1h 04m 23s","accounts_in_message":[],"_revision_number":5},{"id":"120e8df90c9252947ebbac2350d077dd0dcda13a","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-08 13:06:28.000000000","message":"Patch Set 5:\n\nSome experimentations on the timeout value of replication_lock()\n\nDuration of rebalance with per lock timeout duration in reconstructor:\n- (master) no lock: ~1h\n- (this PR) default of 15s from replication_lock_timeout: ~3h\n- (this PR) lock timeout of 0.2s: ~1h10\n\nSo I\u0027d like to hardcode a small value when called from replicator/reconstructor. I\u0027m not sure adding a new config parameter has a great value. It could confuse even more the operator.\n\nAny thoughts?","accounts_in_message":[],"_revision_number":5},{"id":"cfa5126baa00ae49c482e8c5ce14471ecc3d8eac","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-14 23:16:47.000000000","message":"Uploaded patch set 6.","accounts_in_message":[],"_revision_number":6},{"id":"e78ac20eb2e64eac16472c1c85f3433ccc8d2cef","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-15 02:04:57.000000000","message":"Patch Set 6: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/6540316c973c4e5594fedb25cc409d66 : SUCCESS in 8m 08s\n- grenade https://zuul.opendev.org/t/openstack/build/193904e318714300b75cfb3579ef0281 : SUCCESS in 1h 00m 00s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/1adb5b30b9f3473da47fdf4be875422a : SUCCESS in 44m 30s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/49225cf53f5247829499bcd0b529ff7d : SUCCESS in 9m 10s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/3e742508361e46b798093f46d8eb7ec8 : SUCCESS in 10m 34s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/c189f214079540d59198960fa263ee7c : FAILURE in 14m 01s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/4f91b2f9c9474fcdb4d8bca94da511c7 : SUCCESS in 11m 18s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/d5de744142d349f2a698fe5b016e0c4a : SUCCESS in 14m 34s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/9eb05d331f144d9c9b995f26b55caa94 : SUCCESS in 12m 21s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/e4fd9526e8a94a799d3f6d4315044cae : SUCCESS in 16m 01s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/a4ac451d1c784a19822a26373ec3cfe3 : SUCCESS in 15m 39s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/30ad9abaf8de49f987ccd4bcc9cc45ee : SUCCESS in 22m 15s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/2169cb077a924676bdf284a883dfb838 : SUCCESS in 21m 30s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/e7376a8239de4c8db64cdcb1cee59e36 : SUCCESS in 25m 02s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/47140d968ae24b519ad29fe29cdee968 : SUCCESS in 17m 51s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/dcd8fa99a7ba4bd8b9f8763594744c9b : SUCCESS in 24m 49s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/132c377c16304547a84097bdd9bdd54e : FAILURE in 46m 08s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/a827d8b36d724a0d856e5d56ce28839e : SUCCESS in 40m 29s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/071e5b1e27e042e68961c84883928a53 : SUCCESS in 44m 52s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/727e8ada16db4fd594820dddf6f4b7a2 : SUCCESS in 12m 14s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/4e0b3ccae766413692413fab023121c3 : SUCCESS in 7m 45s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/337f860770f843b291ca49be289e84b4 : SUCCESS in 21m 57s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/c7a98e83e0b847c3b1e8ed567e0d83c9 : SUCCESS in 55m 36s","accounts_in_message":[],"_revision_number":6},{"id":"15abbceef29b7ceb7e79521048632c4643933ee4","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2020-10-21 04:10:38.000000000","message":"Patch Set 6: Code-Review+2\n\n(1 comment)\n\nI am not certain that I understand all implications, but I don\u0027t see a reason why this is wrong, therefore +2.\n\nIt jumped at me too that the lock order was changed, so I looked everywhere else if someone sneakily uses lock_patch from another primitive, but found nothing.\n\nReplicators will be a bit slower now, because they lock partitions. By how much slower, I cannot tell. Probably by not much.","accounts_in_message":[],"_revision_number":6},{"id":"0669c43c6f4211d313e1ea7224dbc249ac19a25e","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-21 17:39:32.000000000","message":"Patch Set 6:\n\nrecheck","accounts_in_message":[],"_revision_number":6},{"id":"f1a754019b7d27b80965ae3fbf42c0446d04555c","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-21 21:13:40.000000000","message":"Patch Set 6:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/30c9c740eaba4d23a9ca096259ab2932 : SUCCESS in 12m 47s\n- grenade https://zuul.opendev.org/t/openstack/build/e537ce9f22b84f73b8fe58066512dc15 : SUCCESS in 1h 08m 01s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/7f12830e475849c9a93fb5f2fb503b8f : SUCCESS in 49m 52s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/a0e7ba77b2dc46cba3b915eabd38efd4 : SUCCESS in 10m 49s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/df03a27439964a919c0d24e38f243534 : SUCCESS in 12m 06s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/7473c51ef2944f1da54cca5a97d9ec39 : SUCCESS in 14m 49s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/88402a15029149b0af0f8cc5b370a82e : SUCCESS in 14m 08s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/b9b710dfe16047c49e7efd7455c1154f : SUCCESS in 14m 44s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/fdf43d94d98549e9bc350190df47f385 : SUCCESS in 14m 10s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/66553ac5b96c4f77b92444fe297211ce : SUCCESS in 25m 29s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/03e87ccabd064afe96d1ed6605820f06 : SUCCESS in 23m 16s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/2a5aac52c1984886ad729a9baebca563 : SUCCESS in 21m 48s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/9c645d7d6b31441c9a4bc9e26ca91a9b : SUCCESS in 22m 44s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/b70e94dbb69b4abc9392441ea2cc17ff : SUCCESS in 24m 51s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/20e22d8dbde74a0cbc8718d3c8ee3f13 : SUCCESS in 24m 39s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/b70463b78a0441f38012b5f73c7673d8 : SUCCESS in 24m 45s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/3568e38f8d804675ab5c12e2eed18487 : SUCCESS in 48m 48s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/b2b71ab2a78945568475d6bd62fa02d1 : SUCCESS in 50m 32s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/9a1b8f7812ac4a408c72230c4cfd59f4 : SUCCESS in 43m 12s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/ec33a72ec44b40e8a38b3b3f3277a2e3 : SUCCESS in 15m 44s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/1df652ad2f6e4641b980ee79f22e656f : SUCCESS in 8m 16s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/bebb7f3b730b44d8839d5db7c6b23000 : FAILURE in 28m 53s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/3fc1ce5b2cba4ba7afad9f78f38f6c61 : SUCCESS in 1h 02m 07s","accounts_in_message":[],"_revision_number":6},{"id":"ef93211b84be3b75bd30180aafd27beedc421848","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-10-22 00:21:08.000000000","message":"Patch Set 6:\n\nrecheck","accounts_in_message":[],"_revision_number":6},{"id":"09eccb376e801acf50da3d7ace47e7eeef46ac7b","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-22 01:59:45.000000000","message":"Patch Set 6: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/5e71922a8fb64c9ca2b0fe6ce5a21824 : SUCCESS in 10m 06s\n- grenade https://zuul.opendev.org/t/openstack/build/748609d62f884aafaf19369612318f59 : SUCCESS in 1h 05m 16s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/b5732164bc0f4c6aa8423ec105717225 : SUCCESS in 49m 42s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/7a4b1d97f23d415c946d0bb7b0b3005c : SUCCESS in 9m 33s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/7c22528d3bb14b23b9dab82910674405 : SUCCESS in 11m 03s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/f31beb535da34ba5857b2ffea67fae62 : SUCCESS in 16m 14s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/3659dfa7caac43d68f4654ea50777ffb : SUCCESS in 13m 57s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/ff910a9cf292440ebefaac74164cd394 : SUCCESS in 13m 58s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/c4c067dcec2e44449576f7639cffef8c : SUCCESS in 13m 38s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/3dc370a26de140199db5cf4278efd1e1 : SUCCESS in 30m 29s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/7900214e361347c5ba92d6f929793b81 : SUCCESS in 17m 29s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/647c21f74c004c9f9f615b867d0198a5 : SUCCESS in 30m 42s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/1acd0936869342d980c9297f51991f36 : SUCCESS in 34m 58s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/bcf56bce990f48228123381cbf8c3d5c : SUCCESS in 29m 53s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/de9c346b666241439a774b1d311bb26e : SUCCESS in 32m 36s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/9be7025518034b6e90eb8b27f736ad52 : SUCCESS in 27m 37s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/3abd7b0187b2409499455ded8812facd : SUCCESS in 52m 28s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/b95edd3d16914f32bddd56799bd4364d : SUCCESS in 46m 47s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/e88285df72164f68853a52a4063472b5 : SUCCESS in 41m 42s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/87ec69376dc74ecfa38a1ce09f114c3a : SUCCESS in 13m 59s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/cc670eeedb4a485f88a40c14ddb606f1 : SUCCESS in 7m 31s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/4c2ac706fdfb461fb3ff935be1dc07f4 : SUCCESS in 22m 45s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/5d03c45ed3784f7ba096453c4eb555e4 : SUCCESS in 1h 05m 29s","accounts_in_message":[],"_revision_number":6},{"id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-11-04 23:04:26.000000000","message":"Patch Set 6:\n\n(7 comments)","accounts_in_message":[],"_revision_number":6},{"id":"74cccae7a2f047257be9ab56079d917fcc9bb40f","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2020-11-05 18:08:03.000000000","message":"Patch Set 6: Code-Review-2\n\nActually, never mind. Clearly the probe test fails.","accounts_in_message":[],"_revision_number":6},{"id":"9630a535381dba3a7134c08f8fcd90e6ef819ea2","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2020-11-05 18:08:22.000000000","message":"Patch Set 6: -Code-Review\n\nGrr wrong review click click","accounts_in_message":[],"_revision_number":6},{"id":"bb766ab0e289b9aa35325f07823b5ecb877418ee","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2020-11-07 00:59:50.000000000","message":"Patch Set 6: Code-Review+2\n\n(2 comments)","accounts_in_message":[],"_revision_number":6},{"id":"3e8aaf1c1867777dbf34651b4a057f7338adfd1b","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-11-11 18:49:41.000000000","message":"Patch Set 6: Code-Review+2 Workflow+1\n\n(1 comment)\n\nSo I set up a split-brain:\n\n* upload an object\n* find a server responsible for 2 (of 6) frags\n* set its swift_dir to some new location\n* copy over the rings, builders, and swift.conf\n* update the object ring to drop all weights for that server to 0, rebalance, write_ring\n\nI could run the reconstructors and see frags bouncing back and forth, great. I then turned on handoffs_first, dropped my interval to 1s, added a 10s sleep at the top of reconstructor\u0027s delete_reverted_objs, and let them run for a while; sure enough, without this patch both sets of \"handoffs\" would delete and I\u0027d be down to only four frags. *With* this patch, it all worked great!\n\nIt took me a moment to realize why I wasn\u0027t getting any of the lock-timeout stats, but once I added a 1s sleep in _finalize_durable, those showed up as expected.\n\nI wonder if we should centralize the 0.2s timeout constant somewhere; not sure where makes the most sense, though.\n\nI\u0027ve only tested with reconstructor, but replicator flow seems straightforward enough. (Minus rsync, of course -- did we ever write up a separate bug for that?)","accounts_in_message":[],"_revision_number":6},{"id":"815d503706e569f14d356bf422e7e0c0cec2651f","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-11 18:50:38.000000000","message":"Patch Set 6: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":6},{"id":"8b0378190533213aab8cfc3e84a762874e2e3b2c","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-11 21:24:48.000000000","message":"Patch Set 6: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/1bcb6a6200784ec6ab1ad0df3bf9504e : SUCCESS in 12m 02s\n- grenade https://zuul.opendev.org/t/openstack/build/5f399a5e674d499280d439abba5ebb7d : SUCCESS in 1h 04m 20s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/4f0d4988edce4876baab45d8e5dacf0d : SUCCESS in 46m 03s\n- swift-upload-image https://zuul.opendev.org/t/openstack/build/c8eabcd50b5b437a9a67e7d7d7c26e65 : SUCCESS in 10m 44s (non-voting)\n- swift-upload-image-py3 https://zuul.opendev.org/t/openstack/build/d75fa55c08ec46729f25545fdeb6974d : SUCCESS in 11m 32s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/7a645f850b134463b8efc99de3432cef : SUCCESS in 12m 54s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/cb4ef7b08aac4e0ab4c40bb187b78702 : SUCCESS in 13m 18s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/680bb035d00b43d1b5a92d581610aa07 : SUCCESS in 14m 13s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/e1d5de5565c54f3ebb0db300cfbdca06 : SUCCESS in 19m 32s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/6ec2ec8bbe0944fcb59ee0262a8a6822 : SUCCESS in 27m 00s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/12d1dfdd861947cd8b2bd74d2487fa7f : SUCCESS in 26m 45s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/bb2d83bff39f42a990fef82c82a27821 : SUCCESS in 19m 42s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/9c335c14b63f47abb360dbf4949a1abc : SUCCESS in 16m 01s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/00e7cc17dc8b41e8801d9e91f41ff784 : SUCCESS in 19m 02s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/0562c3e41a1e452c80a71610dc538de6 : SUCCESS in 19m 24s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/15ce36763db545fc8136d3401c66d5f6 : SUCCESS in 48m 01s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/c4121715a4b24fb3b7a1cebd2b8da493 : SUCCESS in 44m 16s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/b98eaf68c7a0434caab4be697279c553 : SUCCESS in 50m 56s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/c4d8c74dd2a1475b92e9ce746608dc2c : SUCCESS in 13m 49s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/18ed4911654040af98d1b4459b525261 : SUCCESS in 6m 38s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/96e7147dfd734bcba4f9fa98f08e63e4 : SUCCESS in 1h 06m 56s","accounts_in_message":[],"_revision_number":6},{"id":"66c8ea588015e60949f990087628e5dcf62de19b","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-11 21:24:49.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":6},{"id":"777bf69212a59c3a3033f445ae5c0f71dd8b9794","tag":"autogenerated:zuul:promote","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-11 21:26:24.000000000","message":"Patch Set 6:\n\nBuild succeeded (promote pipeline).\n\n- promote-openstack-tox-docs https://zuul.opendev.org/t/openstack/build/ffbd45e43d17412ab2a104b39a3dafb6 : SUCCESS in 1m 04s\n- swift-promote-image https://zuul.opendev.org/t/openstack/build/b8e9626fc3134b2985d5522ec2fbc0f8 : SUCCESS in 51s (non-voting)\n- swift-promote-image-py3 https://zuul.opendev.org/t/openstack/build/8986d609d1bc4365895f397d4fb47960 : FAILURE in 44s (non-voting)","accounts_in_message":[],"_revision_number":6}],"current_revision_number":6,"current_revision":"8c0a1abf744a11b5c289239e3ac830786a9de4e9","revisions":{"5787c9f92a7df55473d5c074dd085fdae21d72e3":{"kind":"REWORK","_number":1,"created":"2020-09-25 00:57:19.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/42/754242/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/42/754242/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/42/754242/1"}}},"commit":{"parents":[{"commit":"004052dc65406e012822312f267d8374e760d34e","subject":"Fixing S3 credential caching region support","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/004052dc65406e012822312f267d8374e760d34e"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-25 00:36:36.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-25 00:37:45.000000000","tz":-240},"subject":"Fix a race condition in case of cross-replication","message":"Fix a race condition in case of cross-replication\n\nIn a situation where two nodes does not have the same version of a ring\nand they both think the other node is the primary node of a partition,\na race condition can lead to the loss of some of the objects of the\npartition.\n\nThe following sequence leads to the loss of some of the objects:\n\n  1. A gets and reloads the new ring\n  2. A starts to replicate/revert the partition P to node B\n  3. B (with the old ring) starts to replicate/revert the (partial)\n     partition P to node A\n     \u003d\u003e replication should be fast as all objects are already on node A\n  4. B finished replication of (partial) partition P to node A\n  5. B remove the (partial) partition P after replication succeeded\n  6. A finishes replication of partition P to node B\n  7. A removes the partition P\n  8. B gets and reloads the new ring\n\nAll data transfered between steps 2 and 5 will be lost as they are not\nanymore on node B and they are also removed from node A.\n\nThis commit make the replicator/reconstructor to hold a replication_lock\non partition P so that remote node cannot start an opposite replication.\n\nChange-Id: I29acc1302a75ed52c935f42485f775cd41648e4d\nCloses-Bug: #1897177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5787c9f92a7df55473d5c074dd085fdae21d72e3"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5787c9f92a7df55473d5c074dd085fdae21d72e3"}]},"branch":"refs/heads/master"},"5c313607336299bd46be67523d3036df7a4614ee":{"kind":"REWORK","_number":2,"created":"2020-09-28 23:18:35.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/42/754242/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/42/754242/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/42/754242/2"}}},"commit":{"parents":[{"commit":"004052dc65406e012822312f267d8374e760d34e","subject":"Fixing S3 credential caching region support","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/004052dc65406e012822312f267d8374e760d34e"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-25 00:36:36.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-28 22:13:04.000000000","tz":-240},"subject":"Fix a race condition in case of cross-replication","message":"Fix a race condition in case of cross-replication\n\nIn a situation where two nodes does not have the same version of a ring\nand they both think the other node is the primary node of a partition,\na race condition can lead to the loss of some of the objects of the\npartition.\n\nThe following sequence leads to the loss of some of the objects:\n\n  1. A gets and reloads the new ring\n  2. A starts to replicate/revert the partition P to node B\n  3. B (with the old ring) starts to replicate/revert the (partial)\n     partition P to node A\n     \u003d\u003e replication should be fast as all objects are already on node A\n  4. B finished replication of (partial) partition P to node A\n  5. B remove the (partial) partition P after replication succeeded\n  6. A finishes replication of partition P to node B\n  7. A removes the partition P\n  8. B gets and reloads the new ring\n\nAll data transfered between steps 2 and 5 will be lost as they are not\nanymore on node B and they are also removed from node A.\n\nThis commit make the replicator/reconstructor to hold a replication_lock\non partition P so that remote node cannot start an opposite replication.\n\nChange-Id: I29acc1302a75ed52c935f42485f775cd41648e4d\nCloses-Bug: #1897177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5c313607336299bd46be67523d3036df7a4614ee"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5c313607336299bd46be67523d3036df7a4614ee"}]},"branch":"refs/heads/master"},"57a92efed34073b94df9c98fb7e038c43882f08a":{"kind":"REWORK","_number":3,"created":"2020-09-30 18:32:13.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/42/754242/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/42/754242/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/42/754242/3"}}},"commit":{"parents":[{"commit":"004052dc65406e012822312f267d8374e760d34e","subject":"Fixing S3 credential caching region support","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/004052dc65406e012822312f267d8374e760d34e"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-25 00:36:36.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-30 18:19:43.000000000","tz":-240},"subject":"Fix a race condition in case of cross-replication","message":"Fix a race condition in case of cross-replication\n\nIn a situation where two nodes does not have the same version of a ring\nand they both think the other node is the primary node of a partition,\na race condition can lead to the loss of some of the objects of the\npartition.\n\nThe following sequence leads to the loss of some of the objects:\n\n  1. A gets and reloads the new ring\n  2. A starts to replicate/revert the partition P to node B\n  3. B (with the old ring) starts to replicate/revert the (partial)\n     partition P to node A\n     \u003d\u003e replication should be fast as all objects are already on node A\n  4. B finished replication of (partial) partition P to node A\n  5. B remove the (partial) partition P after replication succeeded\n  6. A finishes replication of partition P to node B\n  7. A removes the partition P\n  8. B gets and reloads the new ring\n\nAll data transfered between steps 2 and 5 will be lost as they are not\nanymore on node B and they are also removed from node A.\n\nThis commit make the replicator/reconstructor to hold a replication_lock\non partition P so that remote node cannot start an opposite replication.\n\nChange-Id: I29acc1302a75ed52c935f42485f775cd41648e4d\nCloses-Bug: #1897177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/57a92efed34073b94df9c98fb7e038c43882f08a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/57a92efed34073b94df9c98fb7e038c43882f08a"}]},"branch":"refs/heads/master"},"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3":{"kind":"REWORK","_number":4,"created":"2020-09-30 18:38:25.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/42/754242/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/42/754242/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/42/754242/4"}}},"commit":{"parents":[{"commit":"004052dc65406e012822312f267d8374e760d34e","subject":"Fixing S3 credential caching region support","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/004052dc65406e012822312f267d8374e760d34e"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-25 00:36:36.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-30 18:38:15.000000000","tz":-240},"subject":"Fix a race condition in case of cross-replication","message":"Fix a race condition in case of cross-replication\n\nIn a situation where two nodes does not have the same version of a ring\nand they both think the other node is the primary node of a partition,\na race condition can lead to the loss of some of the objects of the\npartition.\n\nThe following sequence leads to the loss of some of the objects:\n\n  1. A gets and reloads the new ring\n  2. A starts to replicate/revert the partition P to node B\n  3. B (with the old ring) starts to replicate/revert the (partial)\n     partition P to node A\n     \u003d\u003e replication should be fast as all objects are already on node A\n  4. B finished replication of (partial) partition P to node A\n  5. B remove the (partial) partition P after replication succeeded\n  6. A finishes replication of partition P to node B\n  7. A removes the partition P\n  8. B gets and reloads the new ring\n\nAll data transfered between steps 2 and 5 will be lost as they are not\nanymore on node B and they are also removed from node A.\n\nThis commit make the replicator/reconstructor to hold a replication_lock\non partition P so that remote node cannot start an opposite replication.\n\nChange-Id: I29acc1302a75ed52c935f42485f775cd41648e4d\nCloses-Bug: #1897177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"}]},"branch":"refs/heads/master"},"0655611d1e5611437e278bb27939da50f69d140d":{"kind":"REWORK","_number":5,"created":"2020-10-01 22:06:31.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/42/754242/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/42/754242/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/42/754242/5"}}},"commit":{"parents":[{"commit":"004052dc65406e012822312f267d8374e760d34e","subject":"Fixing S3 credential caching region support","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/004052dc65406e012822312f267d8374e760d34e"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-25 00:36:36.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-10-01 22:05:32.000000000","tz":-240},"subject":"Fix a race condition in case of cross-replication","message":"Fix a race condition in case of cross-replication\n\nIn a situation where two nodes does not have the same version of a ring\nand they both think the other node is the primary node of a partition,\na race condition can lead to the loss of some of the objects of the\npartition.\n\nThe following sequence leads to the loss of some of the objects:\n\n  1. A gets and reloads the new ring\n  2. A starts to replicate/revert the partition P to node B\n  3. B (with the old ring) starts to replicate/revert the (partial)\n     partition P to node A\n     \u003d\u003e replication should be fast as all objects are already on node A\n  4. B finished replication of (partial) partition P to node A\n  5. B remove the (partial) partition P after replication succeeded\n  6. A finishes replication of partition P to node B\n  7. A removes the partition P\n  8. B gets and reloads the new ring\n\nAll data transfered between steps 2 and 5 will be lost as they are not\nanymore on node B and they are also removed from node A.\n\nThis commit make the replicator/reconstructor to hold a replication_lock\non partition P so that remote node cannot start an opposite replication.\n\nChange-Id: I29acc1302a75ed52c935f42485f775cd41648e4d\nCloses-Bug: #1897177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/0655611d1e5611437e278bb27939da50f69d140d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/0655611d1e5611437e278bb27939da50f69d140d"}]},"branch":"refs/heads/master"},"8c0a1abf744a11b5c289239e3ac830786a9de4e9":{"kind":"REWORK","_number":6,"created":"2020-10-14 23:16:47.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/42/754242/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/42/754242/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/42/754242/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/42/754242/6"}}},"commit":{"parents":[{"commit":"004052dc65406e012822312f267d8374e760d34e","subject":"Fixing S3 credential caching region support","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/004052dc65406e012822312f267d8374e760d34e"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-09-25 00:36:36.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-10-14 23:16:18.000000000","tz":-240},"subject":"Fix a race condition in case of cross-replication","message":"Fix a race condition in case of cross-replication\n\nIn a situation where two nodes does not have the same version of a ring\nand they both think the other node is the primary node of a partition,\na race condition can lead to the loss of some of the objects of the\npartition.\n\nThe following sequence leads to the loss of some of the objects:\n\n  1. A gets and reloads the new ring\n  2. A starts to replicate/revert the partition P to node B\n  3. B (with the old ring) starts to replicate/revert the (partial)\n     partition P to node A\n     \u003d\u003e replication should be fast as all objects are already on node A\n  4. B finished replication of (partial) partition P to node A\n  5. B remove the (partial) partition P after replication succeeded\n  6. A finishes replication of partition P to node B\n  7. A removes the partition P\n  8. B gets and reloads the new ring\n\nAll data transfered between steps 2 and 5 will be lost as they are not\nanymore on node B and they are also removed from node A.\n\nThis commit make the replicator/reconstructor to hold a replication_lock\non partition P so that remote node cannot start an opposite replication.\n\nChange-Id: I29acc1302a75ed52c935f42485f775cd41648e4d\nCloses-Bug: #1897177\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/8c0a1abf744a11b5c289239e3ac830786a9de4e9"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/8c0a1abf744a11b5c289239e3ac830786a9de4e9"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}