)]}'
{"id":"openstack%2Fswift~811833","triplet_id":"openstack%2Fswift~master~Idacc0d80ec1a73bad56696056463deeb075c2171","project":"openstack/swift","branch":"master","topic":"account_stats_blocked_probetest_1","attention_set":{"597":{"account":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"last_update":"2022-02-02 05:15:59.000000000","reason":"Matthew Oliver replied on the change"},"7847":{"account":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"last_update":"2022-02-02 06:21:25.000000000","reason":"Matthew Oliver replied on the change"}},"removed_from_attention_set":{"7233":{"account":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"last_update":"2022-02-07 23:29:32.000000000","reason":"Matthew Oliver replied on the change"},"15343":{"account":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"last_update":"2022-02-04 05:34:34.000000000","reason":"Tim Burke replied on the change"}},"hashtags":[],"change_id":"Idacc0d80ec1a73bad56696056463deeb075c2171","subject":"container-updater: no incoming syncs no account update","status":"NEW","created":"2021-09-30 03:53:02.000000000","updated":"2022-05-02 02:11:26.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":40,"unresolved_comment_count":7,"has_review_started":true,"meta_rev_id":"d242a19bc814f6463b65101369fff8172b29ef0b","_number":811833,"virtual_id_number":811833,"owner":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"tag":"autogenerated:zuul:check","value":1,"date":"2022-02-08 00:37:24.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"CC":[{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"REVIEWER":[{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2021-09-30 05:07:18.000000000","updated_by":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"reviewer":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"state":"REVIEWER"},{"updated":"2021-09-30 05:12:33.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2021-09-30 05:14:03.000000000","updated_by":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"reviewer":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"state":"CC"},{"updated":"2021-09-30 09:26:24.000000000","updated_by":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"reviewer":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"state":"REVIEWER"}],"messages":[{"id":"4e9538e6f8c8bb83a236414e016fc4dc9aea641e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-09-30 03:53:02.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"4a15d8571b698a728a22ed101cbd922cc299705c","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2021-09-30 05:07:18.000000000","message":"Patch Set 1: Code-Review-1\n\n(2 comments)\n\nNothing is wrong with the patch, but I have questions inside.","accounts_in_message":[],"_revision_number":1},{"id":"e0e6be045c87454a157031dd2d6c3c21a89d5961","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-09-30 05:12:33.000000000","message":"Patch Set 1: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/69796f43a3724212a2cdbcb5ac3c172e : SUCCESS in 11m 05s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/7ada7a3f4e044c8f9e1c602a7d88a92d : SUCCESS in 1h 18m 26s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/f96f0cf62b4046bf95f3dac39217055c : SUCCESS in 49m 16s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/218ffe658b7b4d9892dda1eac1b8559c : FAILURE in 5m 48s","accounts_in_message":[],"_revision_number":1},{"id":"3422ee26081be8fe81ca75f0ae755b6df2aa3faa","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2021-09-30 05:14:03.000000000","message":"Patch Set 1:\n\n(3 comments)","accounts_in_message":[],"_revision_number":1},{"id":"339760b2aca925128e525a6b2a12d42cfbe72eec","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-09-30 05:57:17.000000000","message":"Patch Set 1:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/24525c3510f9405a9f6f44592daafd25 : SUCCESS in 30m 17s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/3f7df265303245f5a598b1071783b723 : SUCCESS in 1h 25m 27s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/5c3bebcac4a14b3f988866aee46e44f6 : TIMED_OUT in 2h 01m 25s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/45f681d640944e0084256d38068c6647 : SUCCESS in 1h 10m 05s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/969ecad8b5004c6d9c6d9c6be7fd116a : SUCCESS in 1h 06m 46s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"cbde6e48e9372a89cbbd865058691f711331cb4c","author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"date":"2021-09-30 09:26:24.000000000","message":"Patch Set 1: Code-Review-1\n\n(2 comments)\n\nThanks Matt!\n\nA few ideas to kick off thinking as to how to fix this:\n\n1. Stop handoffs sending updates\n\n2. Stop nodes that have never replicated sending updates (by inspecting sync_tables??)\n\n3. Make the replicator replicate the reported_* fields between nodes. The handoffs might send more bogus updates, but eventually the handoff will go away.\n\n4. Write down a x-container-sysmeta-last-update-time which would be replicated and the most recent value would propagate to all nodes, then use that to trigger periodic updates i.e.:\n\n  if ((now - x-container-sysmeta-last-update-time \u003e N)\n      OR (actual obj count !\u003d last reported obj count)) \n\nThe handoffs might send periodic bogus updates, but eventually the handoff will go away.\n\n5. As per 4, but also write down x-container-sysmeta-last-update-object-count, and send an update \n\n  if ((now - x-container-sysmeta-last-update-time \u003e N \n          AND local object count \u003e x-container-sysmeta-last-update-object-count)\n      OR (actual obj count !\u003d last reported obj count)) \n\nThe handoffs won\u0027t send further periodic bogus updates. The account stat will only reduce when a node sees that its own object count has reduced since last reported (apart from the initial zero update from the handoff).","accounts_in_message":[],"_revision_number":1},{"id":"d9d3f79df06683e89f2cbdd830ca02ee00c98358","author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"date":"2021-09-30 09:30:42.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"42bbbb0b4edbecc913768e18a0e1a4fb7e8ebf3a","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-09-30 21:20:06.000000000","message":"Patch Set 1:\n\n(5 comments)\n\nThanks everyone! I\u0027ll get this tidied up and maybe have a play with a fix, there are some good suggestions there.","accounts_in_message":[],"_revision_number":1},{"id":"bd72b1f02abe36fe435c1dfdb0623fe25fd441cc","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-01 07:34:02.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"d635b2431dee2a51023493eb5174a6d857fec6f0","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-01 07:42:18.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"3aa540b60ff278a375aaf30eb0d5f1e39c1be593","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-01 08:36:21.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/7a7e2f64dc3d495e97a01ccf825fedf5 : SUCCESS in 11m 11s\n- grenade https://zuul.opendev.org/t/openstack/build/c5136be3076a46a195a298bdd0913f05 : SUCCESS in 53m 13s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/259f082310c34d68b77b64a4366737fb : SUCCESS in 36m 33s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/36209c89eee548b3b0696e71fdd9e36a : SUCCESS in 8m 28s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/bbdfea9a49494876b81fc918ae55fa05 : SUCCESS in 9m 03s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/f730afacfe9c4af09f22318826f6f058 : SUCCESS in 16m 24s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/66a818ec50524b788f74294e8e5ad20f : SUCCESS in 22m 20s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/e46c47a8a02045da8f50d5b7f7cb5b6d : SUCCESS in 16m 27s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/76c729f480264865aab493d1965d4867 : SUCCESS in 16m 05s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/fe25aac34d9244f39f3486697fe8345a : SUCCESS in 17m 13s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/06b2fb70b84344f0972363a860190127 : SUCCESS in 16m 56s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/8674fe528fb6420180220eb0884dbd14 : SUCCESS in 18m 17s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/c6c739a4553e418bb5c8132615653de2 : SUCCESS in 19m 07s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/4cfec0587f534eb59860d150c7a2e063 : SUCCESS in 18m 06s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/8e0008b0f524441c8a2ca0dcd3593753 : SUCCESS in 17m 06s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/162a5efab139410691b54af9ea087db9 : SUCCESS in 28m 46s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/5b7a3d18119f4355ad6fea0fe4880593 : SUCCESS in 8m 06s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/e4d76d22ff944fd796b5b957f94e08df : SUCCESS in 30m 04s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/b7227d052e564030a6db3bbc6d65728e : FAILURE in 59m 15s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/5bbb6a52eb4c4327ad6b08fbbd1b4b9d : FAILURE in 1h 00m 04s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/ff45fdc928a549a99328c4d458474030 : SUCCESS in 44m 08s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/2124c46048ff46f39202925a40a009d4 : SUCCESS in 43m 04s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/9e2911eb73b04e23b0d2ba91c1f149f1 : SUCCESS in 16m 05s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/8b75aa64c2644c709e07eef5474801c8 : SUCCESS in 6m 01s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/d0e1ea9227db453eb4da1493523e0d99 : SUCCESS in 30m 55s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/124eb086f6b74ac893d6253972bf7f25 : SUCCESS in 46m 03s","accounts_in_message":[],"_revision_number":2},{"id":"97f105faea88b17388d1bbe7f979f89a4f15ecb0","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-01 10:08:53.000000000","message":"Patch Set 2:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/b8d0e65bc2db40a6b1ca827a3b7681d1 : SUCCESS in 32m 08s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/132f9152902941fe948fecd739bee094 : SUCCESS in 49m 13s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/96b904744f86406fb64992acfb70d666 : TIMED_OUT in 2h 01m 23s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/2c78c6425a0441a2be93c31d1afaa3fd : SUCCESS in 1h 00m 42s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/e1832b6335d04156b9101c27313dc5e1 : SUCCESS in 51m 41s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"64d0a7a30e1b6c9cac3c4297ff379e89acb366f5","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2021-10-02 05:03:38.000000000","message":"Patch Set 2: Code-Review-1\n\n(3 comments)\n\nbikeshedding","accounts_in_message":[],"_revision_number":2},{"id":"cd8b5354bde9d314ebea5dcc21055aaa156a8200","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-03 22:26:51.000000000","message":"Patch Set 2:\n\n(3 comments)\n\nThanks Pete! completely missed the account_ring already being there, not bikeshedding this time, thanks for making the code cleaner!","accounts_in_message":[],"_revision_number":2},{"id":"4cd2532728f1066cd955ea1f00b321d7b2ee7370","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-03 22:27:29.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"1210ada68f53af23e9bb885fd21e0f422556f044","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-03 23:36:17.000000000","message":"Patch Set 3: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/ae7d73c31bdc4735bc15803127814492 : SUCCESS in 9m 14s\n- grenade https://zuul.opendev.org/t/openstack/build/7f44399ed91c4ff8a0942d7a3058c904 : SUCCESS in 1h 02m 45s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/b33e4d66baa6444d9e8a7e26aac2ba3d : SUCCESS in 44m 19s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/f54ff2f76ebd4888b0cb4617b96f40b2 : SUCCESS in 8m 26s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/87bcdb9aa3e640c79c2437df4dc17abb : SUCCESS in 8m 53s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/6bbf26ba595647db803215ff69512a4a : SUCCESS in 13m 49s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/53b8a8b45e6640d2b982b3998d4818a7 : SUCCESS in 33m 01s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/7c36542cf5d048468379ed5d5790e654 : SUCCESS in 15m 42s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/322a168042aa41abaf2de9ae778a2d9a : SUCCESS in 21m 37s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/1c2af5152dc84354ab11bb4c3a2282bc : SUCCESS in 16m 29s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/7c4d8676b4994427831c6c41206be9b8 : SUCCESS in 21m 08s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/0103696fc68c48c5a595e1ed4050da7c : SUCCESS in 16m 28s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/46851433b24b4d6b8e55187bbbbc52b8 : SUCCESS in 35m 34s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/fe4db9390b2343e8841be76813d261b6 : SUCCESS in 17m 06s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/f0acf1c600784a3db83dab4513a3b21e : SUCCESS in 21m 31s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/e328cc66b3b34b6b9d70fbf014a8669e : SUCCESS in 17m 24s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/169e861d23fd43f2a364a42fb526cc4e : SUCCESS in 8m 14s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/7c20067376bf40098d455d7f55862c6a : SUCCESS in 22m 02s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/854194de5ba149ccb4d217ce8e0becdb : FAILURE in 56m 56s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/4843b5a566e740bc83477ea624888e1d : FAILURE in 54m 47s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/6006cd4a469b49ea91276a9e3036ed8d : SUCCESS in 47m 29s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/cbf7c8c5102c46679670f7a0d3d3d0b7 : SUCCESS in 37m 27s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/c920992a3f974e5b9340cd53ede333e5 : SUCCESS in 32m 29s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/53e7db1ccec546ae842a59481cfff357 : FAILURE in 6m 25s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/3f9071e1b37f47b28fda11c83e58781b : SUCCESS in 26m 18s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/4c8057966a78404abbb678a8332d9b1f : SUCCESS in 54m 06s","accounts_in_message":[],"_revision_number":3},{"id":"71af36d3b5d7880aab1c6f68263664df6c58aa9b","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-04 00:07:34.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"f465e6e521f10cad5a235627570558ff6813b283","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-04 01:09:37.000000000","message":"Patch Set 4: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c726bb0f682a407b98c71a2ebf8d3687 : SUCCESS in 9m 12s\n- grenade https://zuul.opendev.org/t/openstack/build/a79822a1afa94ee8b97b5d5adc5fcb11 : SUCCESS in 48m 54s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/2816092e76a64183ad440bf0ef69a1a6 : SUCCESS in 39m 05s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/4397b59cf493431a9863dbb0fea34196 : SUCCESS in 7m 55s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/f02953c63397453ab9889ff5b3c940fc : SUCCESS in 9m 03s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/2fce27ffafc54fa99749a614a254649e : SUCCESS in 22m 06s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/8425ba04d7124b7daa76abcc2f40fef0 : SUCCESS in 24m 59s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/d190d69cd229424196231f6a34f8f256 : SUCCESS in 24m 29s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/06bfe77cbd844616bb242599198e999e : SUCCESS in 20m 35s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/51f9f7048a7e4b1cb59abadc0c143360 : SUCCESS in 24m 19s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/658615e6ead14e41a1585bb406f117cf : SUCCESS in 26m 19s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/85b508f429124c718e037abaf72869cf : SUCCESS in 25m 59s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/23732c5a04704324ab0fdcd7f3cba7f1 : SUCCESS in 27m 18s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/37290b19a3854ed7b72fc1b78fe1f9c2 : SUCCESS in 26m 03s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/8795b06ebd61426c916010ba901fafff : SUCCESS in 26m 39s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/f085b468710c4c46b60c45d3e53236a2 : SUCCESS in 23m 49s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/f528480d83c44c469f974386214ac1ed : SUCCESS in 6m 00s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/f865d96855374ad8bc0ae5e329673d64 : SUCCESS in 20m 16s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/b19b8cb68a9d491ebb7b002a5b6173f9 : FAILURE in 49m 39s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/5c725215849f4c7685d10148187bcca5 : FAILURE in 56m 36s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/798502182aab4bd38912ec56323fe683 : SUCCESS in 33m 56s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/c31bce545c3047be93194ca0c1f408da : SUCCESS in 42m 52s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/25a0db0f0ef4404c8cd53a632c5e74c0 : SUCCESS in 13m 34s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/1a3769bb87cd4fcf9d2511abc664c813 : SUCCESS in 5m 44s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/0d1f130d2dd846bca7f949b4acf19319 : SUCCESS in 25m 22s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/d0d65958a0c147b8ab3b3434a89b0af4 : SUCCESS in 39m 33s","accounts_in_message":[],"_revision_number":4},{"id":"055fc5d3ff173c7d2852cd279064ed6f1006066d","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-04 01:57:54.000000000","message":"Patch Set 4:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/33f8be0c3687425ebcc9bbedc3fcf5e3 : SUCCESS in 43m 22s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/064841e01ad34670a5b3ed714a5c8e4d : SUCCESS in 39m 37s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/ad9837cb4dce4e21b728ad95189ffa67 : FAILURE in 1h 47m 47s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/05c5742163bb46938cd19a8ae23c3ead : SUCCESS in 45m 25s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/6671a890158549209aa6eb6fb74661b7 : SUCCESS in 42m 18s (non-voting)","accounts_in_message":[],"_revision_number":4},{"id":"c6f5141fdb3d318f6182b382a73a3103b4795f1b","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-04 05:38:20.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"5dd1a4333f6c131ebc82293000adb03a249f1dff","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-04 06:58:22.000000000","message":"Patch Set 5: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/5ee8858160bc4b3cb7cefef7b68d94e4 : SUCCESS in 9m 46s\n- grenade https://zuul.opendev.org/t/openstack/build/561354f4c6c5487b98b490f07f7d3350 : SUCCESS in 52m 45s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/c1a918bed913489b9ea5433b0b0bbef9 : SUCCESS in 41m 13s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/faa9140518954edea4c9cbd038947294 : SUCCESS in 8m 07s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/da12c4e6db664a32b31fe29330002e65 : SUCCESS in 8m 43s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/23e84d25d5f74111a826494499657051 : SUCCESS in 20m 24s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/6cd9ed456c5d4fceaa9cf22e33ef8e4d : SUCCESS in 20m 56s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/b86efbd53cfe4baeb6978424d498b6b4 : SUCCESS in 15m 29s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/d1dbfd6dd25d4c468ac771129339dc52 : SUCCESS in 20m 26s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/494c9358a5cf4b6a98f3e6552899de4a : SUCCESS in 20m 10s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/f5ea286068a247d0928831c393342d72 : SUCCESS in 21m 11s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/23dfe7c38a5446e6b8b335eba93a4a1f : SUCCESS in 21m 01s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/528eee33a3bc49eaa9659b25b572f7bf : SUCCESS in 22m 01s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/706bb0be5dea49819deb8fdfdb49c37e : SUCCESS in 18m 46s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/7410c45758634527b6813551604b3b43 : SUCCESS in 23m 30s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/3ab43d5e8f264b008b6ea9d7099baf6a : SUCCESS in 25m 48s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/9662342ddf514655826f6f763129718b : SUCCESS in 7m 43s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/7e9c19cd57e14c9492dc957ef17f5e7a : SUCCESS in 24m 49s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/86badfc598a04c87a93cba0155043ee8 : FAILURE in 1h 13m 22s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/b457e1b98c8542fbaa6216199a1198c7 : FAILURE in 58m 55s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/18e19b74a39f445c990ee3c2f72e66ef : SUCCESS in 42m 16s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/1f2d36a33ce44b278ab6553695c1b90e : SUCCESS in 36m 05s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/7869c755d06f48bda6deaeb67180614f : SUCCESS in 15m 13s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/07a14520531949899310530d9d3f54b5 : SUCCESS in 6m 33s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/824c6b9ad04f47baa6f571a661766171 : SUCCESS in 24m 20s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/d8546fec56954396833efbae56e00326 : SUCCESS in 50m 45s","accounts_in_message":[],"_revision_number":5},{"id":"b27c83e7c36263764957f8b84dcccb6eb9f86caa","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-04 07:42:14.000000000","message":"Patch Set 5:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/95caf02902ad4fd78d46667bc0b1f016 : SUCCESS in 33m 09s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/0759f05802d94b23a8bf7883761d7f6c : SUCCESS in 42m 26s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/3d321e987e404973a6a081a4fa218d15 : TIMED_OUT in 2h 01m 18s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/8996beb27f57427c9bf8fd46b16deb07 : SUCCESS in 57m 53s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/6c69e37ab087469580a1330b825f5229 : SUCCESS in 1h 01m 52s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"8e4357db0452c8eb0470b7338a40256a11ea36ed","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2021-10-05 00:00:34.000000000","message":"Patch Set 5:\n\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\nFAIL: test_update_pending (test.probe.test_orphan_container.TestOrphanContainer)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n  File \"/home/zuul/src/opendev.org/openstack/swift/test/probe/test_orphan_container.py\", line 144, in test_update_pending\n    self.assertLengthEqual(outstanding_files_new, 0)\n  File \"/home/zuul/src/opendev.org/openstack/swift/test/probe/common.py\", line 438, in assertLengthEqual\n    obj, obj_len, length))\nAssertionError: len([u\u0027/srv/3/node/sdb3/containers/775/04f/c1edf5341a0828c8cb0e07df6d44a04f/c1edf5341a0828c8cb0e07df6d44a04f.db\n\nHmm","accounts_in_message":[],"_revision_number":5},{"id":"1181fe7fa68118c89b78a0f5ba283b0aca900b4e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-05 08:58:37.000000000","message":"Uploaded patch set 6.","accounts_in_message":[],"_revision_number":6},{"id":"42638cbb96793cc946da92a5458933b9f9b98c2e","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-05 09:04:28.000000000","message":"Patch Set 6:\n\n(1 comment)","accounts_in_message":[],"_revision_number":6},{"id":"45b28b3f0a684db76971b656d5b8230fb0ad0d17","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-05 09:55:59.000000000","message":"Patch Set 6: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/6fd00945d6674a2081900512f7ad119f : SUCCESS in 10m 21s\n- grenade https://zuul.opendev.org/t/openstack/build/a1bfa776ba04478088bb617300156cc3 : SUCCESS in 54m 01s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/a662eb875470490a9a150c42618d7f2f : SUCCESS in 52m 00s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d581762e055a48f38b26ecca9be13186 : SUCCESS in 11m 10s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/acb5322aa01d4cb1865610299214c185 : SUCCESS in 10m 31s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/b75078e14d6d4def84d03e3ab72259f7 : FAILURE in 25m 55s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/3963457305c8414290c4a54d93716036 : FAILURE in 26m 00s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/2149eda4bb9848a79e9bff66170c9f3e : FAILURE in 25m 54s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/e180e803c96d4c6c8352bfec96425744 : FAILURE in 19m 23s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/2c09295913454863aff13b1f6aa9a4ab : FAILURE in 19m 52s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/758365bc2fcc4ba98ff8a7c478f0343b : SUCCESS in 17m 23s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/d2c2c4dd3a57475e8ae868d5d2343d3c : SUCCESS in 18m 04s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/fc9a387f225d45f49ec6411a1f5d02da : SUCCESS in 18m 25s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/8cf4f2c40ca642df9d9c388c1974e2c3 : SUCCESS in 18m 55s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/3e1c3dc895924c1f994a979084d1a44d : SUCCESS in 17m 25s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/eaec90c90b034f6cbb4c2e609578e968 : SUCCESS in 18m 09s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/15f8f9376f8f4f5c8231c2a55e834ef2 : SUCCESS in 7m 12s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/cc47670caf55456bad1387c3df1d33f3 : SUCCESS in 20m 18s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/e4f741b5cef04c80b4b168416ebd767e : SUCCESS in 50m 17s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/cd87eb0c53664f2b9a5f5b4bd3a3824a : SUCCESS in 52m 10s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/aeca0be673ef4ff5be99c3558c49f58b : SUCCESS in 48m 55s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/a512a3e71ad446f6a3b74fd1105a7923 : SUCCESS in 40m 10s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/91607c5857c94945a29878625ed44ef9 : FAILURE in 16m 42s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/baab742667d441409dfe148bc591a97c : SUCCESS in 7m 24s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/7baf5af9b7d2451bb0a43907f4927717 : SUCCESS in 23m 11s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/2887e8b75a4f4df9a72696a02d13bf20 : SUCCESS in 51m 34s","accounts_in_message":[],"_revision_number":6},{"id":"a1881fff7eefe8bc0c0307ee76d222f2bc031619","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-05 11:07:19.000000000","message":"Patch Set 6:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/60fa54a221e34a19bf0ee26965f09bac : FAILURE in 52m 29s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/04023efe5dbb450c93e0beeb07a48e17 : FAILURE in 1h 23m 17s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/04d9a757fa3f41088c96f4a4970e0952 : TIMED_OUT in 2h 01m 30s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/1b004b57c91b418c94f59aa2e73dfb0e : SUCCESS in 1h 05m 18s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/d3af8b72a1934d868748f942b175ed76 : SUCCESS in 1h 02m 16s (non-voting)","accounts_in_message":[],"_revision_number":6},{"id":"05389a482dd9351e03f7e9bbd98ce7a96981e8f2","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-06 01:26:14.000000000","message":"Uploaded patch set 7.","accounts_in_message":[],"_revision_number":7},{"id":"0bf9730d3f4f0873ca2a931e38d3b68f3facaff0","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-06 02:25:55.000000000","message":"Patch Set 7: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c0faee5269fb4e97909277d9ced7782d : SUCCESS in 10m 41s\n- grenade https://zuul.opendev.org/t/openstack/build/e1a22d9b8900425b9f341203a462a84c : SUCCESS in 52m 19s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/c24114dfcac84987970492bfe28482aa : SUCCESS in 33m 17s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d1f8d2a9e26e48b4b91e158985d678af : SUCCESS in 8m 51s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/7d465db4c0c44e8fa42cb082123a5342 : SUCCESS in 8m 13s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/91189665e6e045efae237f069104f8e3 : SUCCESS in 13m 17s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/a87f360c83304a5284b50309bd565267 : SUCCESS in 14m 21s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/96a5a9709a224203961321b995964d69 : SUCCESS in 15m 52s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/828607595077431eaf985940dc8f56a9 : SUCCESS in 14m 02s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/f1d83de6bdae446e8bf80977dc4bbc12 : SUCCESS in 15m 20s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/e0266be153b042d7b7ffa13bb4b19120 : SUCCESS in 15m 10s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/28782255d2164cf0b9e9c5ed18c4f8cb : SUCCESS in 20m 14s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/d463270750d04be4969557dc2c0ac284 : SUCCESS in 16m 36s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/483bf233df7948eb9893622729f49854 : SUCCESS in 17m 21s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/20026c5c237b4af4b08bde46dc1636d2 : SUCCESS in 15m 25s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/c090bfc23839419ea6da0ffd669c022f : SUCCESS in 18m 44s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/6ce32cfd82f144a2a227dd3a7ed18ba1 : SUCCESS in 6m 17s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/ef66b723258d4613a1e7c622d39342cd : SUCCESS in 23m 11s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/f9d0a85bba9a4795b34ec218b8bee1f8 : FAILURE in 47m 05s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/fbedcff71d034ff4962f2e716a008e23 : SUCCESS in 56m 56s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/917dfa650880462abdaca8d468c2c66e : SUCCESS in 32m 14s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/67a72c39fae14092ae28bb02bd5e28d4 : SUCCESS in 41m 34s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/0889ef49cecc4ddbbaa8a38bbb32fe8d : SUCCESS in 14m 41s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/237f0ac94a3f498caa90baee4bffe7ae : SUCCESS in 6m 37s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/1511dcdf7a2a47208417543b8ff00e95 : SUCCESS in 19m 04s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/6e8f6a383f7547b89e07930e823e2648 : SUCCESS in 43m 13s","accounts_in_message":[],"_revision_number":7},{"id":"50c77369e062df27412dd794bcea101c68826c83","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-06 03:26:52.000000000","message":"Patch Set 7:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/65f280669c4b4588b7713f9b011666c6 : SUCCESS in 48m 26s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/9821cf6e7eea4a77930fd116ea22988e : SUCCESS in 45m 59s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/7c6645ff9bee436682243c9a54855fab : FAILURE in 1h 58m 03s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/e34129c141c044569c95022e77240a89 : SUCCESS in 59m 23s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/cf45dc5c3fb24cf8bc9366bc987f061f : SUCCESS in 1h 05m 40s (non-voting)","accounts_in_message":[],"_revision_number":7},{"id":"077abf1d4f4bb2bb44bae1e3c3c5c5cc84bbda66","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2021-10-07 04:08:53.000000000","message":"Patch Set 7: Code-Review-1\n\nI do not like the new option very much. Did you think about reading the container ring in the updater and using ring.replica_count property instead? Yes, it would make it load an extra ring (it loads the account ring already), but that seems like a preferable approach. Ring can be unreferenced and not pinned in self.","accounts_in_message":[],"_revision_number":7},{"id":"1f650b085682b723f488cd78e1027104f2faa719","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-07 04:50:21.000000000","message":"Patch Set 7:\n\n\u003e Patch Set 7: Code-Review-1\n\u003e \n\u003e I do not like the new option very much. Did you think about reading the container ring in the updater and using ring.replica_count property instead? Yes, it would make it load an extra ring (it loads the account ring already), but that seems like a preferable approach. Ring can be unreferenced and not pinned in self.\n\nLoading up the container ring is exactly what I was trying to avoid. The config only really effects single replca rings and yes it\u0027s annoying to add this complexity just for this.\n\nI guess if we just loaded it up at startup and stored the replica count in self, then there will be an initial bunch of memory used initially but should be garbaged quickly. Darn these single replica container rings.","accounts_in_message":[],"_revision_number":7},{"id":"e6e2c0aa71a05c10024f6af73089e2d8df19d8b7","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2021-10-07 05:12:04.000000000","message":"Patch Set 7:\n\n\u003e Patch Set 7:\n\n\u003e I guess if we just loaded it up at startup and stored the replica count in self, then there will be an initial bunch of memory used initially but should be garbaged quickly.\n\nTake a look at the metadata_only\u003dTrue parameter.","accounts_in_message":[],"_revision_number":7},{"id":"9fd0dcce51bb5ed428a5ee95b3b2254166521bd5","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2021-10-07 21:47:55.000000000","message":"Patch Set 7:\n\n(1 comment)\n\nmetadata_only\u003dTrue won\u0027t be sufficient: https://bugs.launchpad.net/swift/+bug/1942481\n\nAs far as I can tell, replica_count has always been zero for rings loaded with metadata_only\u003dTrue, ever since we introduced the property on RingData back in https://github.com/openstack/swift/commit/89a5c9d5","accounts_in_message":[],"_revision_number":7},{"id":"ca350bea97312a9378c9ccb00c77a65125421832","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2021-10-07 22:54:25.000000000","message":"Patch Set 7:\n\n\u003e Patch Set 7:\n\u003e As far as I can tell, replica_count has always been zero for rings loaded with metadata_only\u003dTrue, ...\n\nFine, let\u0027s go to the old idea of load-and-discard.","accounts_in_message":[],"_revision_number":7},{"id":"3c868c5a6f07b506d957f3d427ef20de30f0f0b3","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-13 08:48:08.000000000","message":"Uploaded patch set 8.","accounts_in_message":[],"_revision_number":8},{"id":"b17a6fef8a99d95328026b6696385ddf63634da2","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-13 08:58:00.000000000","message":"Patch Set 8:\n\n(2 comments)","accounts_in_message":[],"_revision_number":8},{"id":"c00e409fac3c657d6becd7dcc7550f219515be86","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-13 10:26:38.000000000","message":"Patch Set 8: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/fd528be02ae140ed8918c23a1dcd5762 : POST_FAILURE in 9m 07s\n- grenade https://zuul.opendev.org/t/openstack/build/80ebdf14386b4f7388e755a9e054cc04 : SUCCESS in 45m 15s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/92423228c5424c09af8a22d1feb56a8a : SUCCESS in 35m 05s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d8991de7c6dc49039eb83ac353399a00 : SUCCESS in 8m 14s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/df1cf9ef8283471faac6b4ad1f7d4a77 : SUCCESS in 9m 01s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/177be1641a0047888f1b918b6dcf7c0d : TIMED_OUT in 1h 31m 14s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/b07330de722b4b00879f3bbb1d3ab1a0 : TIMED_OUT in 1h 31m 17s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/4e934a0e53104119995d08d97efc1468 : TIMED_OUT in 1h 31m 23s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/7d7d3b72bb9740ce9b7e831c969b4e77 : TIMED_OUT in 1h 31m 25s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/b8a0679d045a4df0a827e90300cc3e64 : TIMED_OUT in 1h 31m 15s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/d269700437934f018f791d8466001689 : POST_FAILURE in 16m 16s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/897fce2a4502462c8cc120c5ad4ea83e : SUCCESS in 16m 11s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/4d9c6e456a9a47fc970119338b08daeb : SUCCESS in 17m 33s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/63a8beebe0d64b6181930c4735b124b9 : SUCCESS in 15m 41s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/791cb3e0bfbe4187b081233f726f224e : POST_FAILURE in 14m 27s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/4f5a1d74cfc64c31adf288df7c27ae77 : SUCCESS in 16m 14s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/290d5094ffae42d8be1de32bd7955c1b : POST_FAILURE in 7m 55s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/5a8de9dbccf742f887da46a1b1292f2d : POST_FAILURE in 23m 50s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/c00e00a7a88e46cabb79310cb2d25346 : SUCCESS in 43m 50s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/2e29e7685b724a9cbfed507bf944c210 : SUCCESS in 52m 52s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/c2b8a233216c4138842dc87893d6ea0b : SUCCESS in 28m 07s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/c27f028faa38429f86160d7f5426e334 : SUCCESS in 49m 17s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/355b36c77bdd4f77a37daa6bbc2e6368 : TIMED_OUT in 41m 19s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/a13da35e19254209aec91250d5d15bab : FAILURE in 6m 40s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/eccfcdfe9c114752aa2ca2096ff540ad : SUCCESS in 23m 12s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/c6db0d36fbec490cab491f723fc744c3 : SUCCESS in 51m 13s","accounts_in_message":[],"_revision_number":8},{"id":"b346d6c39ec36391d3f047d263a63a2847406334","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-13 11:00:15.000000000","message":"Patch Set 8:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/0845bc4ace7a49b0ae25489f0c4b406d : TIMED_OUT in 1h 31m 26s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/623bb8f13ef0460a997692e036a78cea : TIMED_OUT in 1h 31m 18s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/e30c114801514b39b08f650f27a5b635 : FAILURE in 2h 00m 40s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/33d969ba665f4f2db0b91f11b544cc62 : SUCCESS in 58m 35s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/92bb1ce0efc54171989ae02bd65f75d0 : SUCCESS in 57m 22s (non-voting)","accounts_in_message":[],"_revision_number":8},{"id":"4b2d7fdfb01902596eaf476b4802f24b673d7ca6","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-10-13 22:28:09.000000000","message":"Uploaded patch set 9.","accounts_in_message":[],"_revision_number":9},{"id":"4ee4b5d817f56efe1bfb8199f986e76a0c9166e0","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-14 00:49:17.000000000","message":"Patch Set 9: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/cc9904f2b9a34948977660085716ace0 : SUCCESS in 9m 48s\n- grenade https://zuul.opendev.org/t/openstack/build/a2b9611e64424986af316c9601436b6f : SUCCESS in 1h 01m 01s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/4def2bb355744b36b07779b54ce650d3 : SUCCESS in 40m 28s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d15b23a0c3ab42ccb3ea856fed77c6f5 : POST_FAILURE in 8m 55s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/fb24cb22fe7444daa31e5c7ee253216f : POST_FAILURE in 11m 01s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/b3025cabc0e640e1b50d81b2d2d17cca : TIMED_OUT in 1h 31m 13s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/e90d035da9df4ed59b3094826479bc80 : TIMED_OUT in 1h 31m 12s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/4b8c9f55a0204d19898957e3e78887d2 : TIMED_OUT in 1h 31m 08s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/b8c6a6dcc28e405f8ea8427456906504 : TIMED_OUT in 1h 31m 09s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/a6454945ae8148f4a0a72a29693f8ad9 : TIMED_OUT in 1h 31m 55s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/a75a1b61421e4e3e9dc46ec96960cdd6 : SUCCESS in 21m 12s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/729bd8646e0c4183aeefaa3dee4b9a4b : SUCCESS in 18m 56s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/de636b6151ee411d8bc6e0466a7cc036 : SUCCESS in 17m 15s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/587b7cc0a93148f39363ac9f74a26661 : SUCCESS in 16m 26s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/8fea9b7c080c4598b6e5e8d5dbad2a17 : SUCCESS in 16m 14s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/d23b3573a9934a39ae41212d8e88ea7a : SUCCESS in 18m 46s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/cd9afb00cf344c319f4708572aa03037 : SUCCESS in 10m 14s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/aa3e3268a34845c0b1ec245416c25849 : SUCCESS in 28m 57s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/a4a9eba42e654743902edf5b662a4a35 : SUCCESS in 51m 00s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/9b0fd07a6c254bf49930095fb9ed4741 : SUCCESS in 57m 20s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/bd93c5d5cfd84e3b9e098f0401497a09 : SUCCESS in 45m 58s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/d9ceb3c942b44a46b0f5f468bfa52816 : SUCCESS in 46m 13s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/7a0469d46d454318b952fa0dffb69cd1 : TIMED_OUT in 41m 18s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/ecf2382230dd433f9cf0f5595b11bf45 : SUCCESS in 7m 39s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/5c2d370679d841b4bd25fa962e1c7f53 : SUCCESS in 25m 09s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/d65b6d5705c048c6bc6ab724ce3473dc : SUCCESS in 51m 09s","accounts_in_message":[],"_revision_number":9},{"id":"fee971cd55d7edd5d206e5695b4cd01217d65eac","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-10-14 01:09:57.000000000","message":"Patch Set 9:\n\nBuild succeeded (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/367f18452dd54244b7a65104f2398835 : TIMED_OUT in 1h 31m 19s (non-voting)\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/a03b401025be4c2e8fadab2470317d8f : TIMED_OUT in 1h 31m 26s (non-voting)\n- swift-probetests-centos-8-arm64 https://zuul.opendev.org/t/openstack/build/3cfea67d89934ba7909f563180c021d5 : FAILURE in 1h 50m 03s (non-voting)\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/6492f02b26424f7eb8f516faeaa628f8 : SUCCESS in 45m 11s (non-voting)\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/823fc0ff8b0244729734c60cf22ce351 : SUCCESS in 47m 22s (non-voting)","accounts_in_message":[],"_revision_number":9},{"id":"bd1d262f0dc6a33dcb011ae09e5aa4679732dd0f","author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"date":"2021-10-16 04:34:53.000000000","message":"Patch Set 9: Code-Review+1\n\n(2 comments)","accounts_in_message":[],"_revision_number":9},{"id":"81b04d49edbf8cefd30d084813802a3b8b61f2af","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-01-31 06:21:38.000000000","message":"Uploaded patch set 10.","accounts_in_message":[],"_revision_number":10},{"id":"fa0f526d091060494d4d4592b9843937f002ee66","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-01-31 07:27:19.000000000","message":"Patch Set 10: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c6e6495913634d0c892933b25165e914 : SUCCESS in 9m 11s\n- grenade https://zuul.opendev.org/t/openstack/build/c847498b746640a4874da3abd9bc402c : SUCCESS in 1h 01m 19s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/d223efe6581c4263be7acb0d71d23478 : SUCCESS in 37m 24s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/6e5cc2d5fba5427582f3222f89154971 : SUCCESS in 34m 41s\n- swift-tox-func-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/b729a9c0ec674316b937d42e5bfe5a9e : SUCCESS in 19m 31s (non-voting)\n- swift-tox-func-encryption-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/abf277390db2405787056073b2c1e3bd : SUCCESS in 23m 37s (non-voting)\n- swift-tox-func-ec-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/cf14cb9e764447e1b16af45cc0b05404 : SUCCESS in 16m 58s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/762dc4e854904967a58f2576c6dc80b2 : SUCCESS in 9m 13s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/daa239131fca415894fdf89158e7b0b2 : SUCCESS in 5m 19s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/c68c4bfd181a4a83b52166adc7dd3bdd : FAILURE in 19m 01s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/1bdeae03e53b46c7952ec3b71ed20de1 : FAILURE in 14m 28s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/cb0f89fd0c914497a4ff582cbe6f8881 : FAILURE in 15m 27s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/7c6e67da0e4c4140ab6d3b2bc5d83b23 : SUCCESS in 14m 24s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/fbb0b848c09e4ae88899a750ad6d3b2e : SUCCESS in 15m 45s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/55bf911b24824f6a9f8fd60b9953bd89 : SUCCESS in 18m 09s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/c746b15e579d4ef88a403dd37eb1609c : SUCCESS in 11m 18s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/a03ba983d708432a97d870c3c544ae3b : SUCCESS in 13m 17s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/64fcf220aa4449da869f0416b3fd96a3 : SUCCESS in 15m 15s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/efe1a38f0223459791840a605fcc9ae1 : SUCCESS in 5m 42s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/ad39206a78464e8b9de745452fa253c4 : SUCCESS in 15m 47s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/87a9f91597494dae94c1f732ca0d76ac : SUCCESS in 36m 32s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/e5b22203ded349dcaf4340f39dd1288a : SUCCESS in 45m 48s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/e5cd61aa46774010a93807ae7658c42f : SUCCESS in 35m 59s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/5cbc699651484b54b910177f6c0e6e75 : SUCCESS in 26m 49s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/52ffea52dcab4b659a62e32acdfb7aae : FAILURE in 21m 56s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/a30daffe42a348caae0a1e3e6ee04c95 : SUCCESS in 4m 45s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/a995fcd8b34a42e28aa5574725d6f94a : SUCCESS in 23m 50s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/1e2a884185a2425d863d07ad7c96ac9e : SUCCESS in 38m 09s","accounts_in_message":[],"_revision_number":10},{"id":"326161874aadc818fa73a67668fc411f7653a300","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-01-31 09:29:55.000000000","message":"Patch Set 10:\n\nBuild failed (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/a4fb6944984246e48f73fb7c0ae453f2 : FAILURE in 55m 37s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/af2a32a0cfec4b629b93e577da099a14 : TIMED_OUT in 1h 31m 10s\n- swift-probetests-centos-8-stream-arm64 https://zuul.opendev.org/t/openstack/build/6b73f58b45044ce2bc7fc46dc5ef4345 : TIMED_OUT in 3h 01m 29s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/139c6d8a401c4dbc90a327c95b184755 : SUCCESS in 28m 51s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/863fd10964e44fa0abdbc87ebe49fefb : SUCCESS in 26m 58s","accounts_in_message":[],"_revision_number":10},{"id":"4aad8025030a88f7c677db2e02db43538b8be2ed","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-02-01 02:20:46.000000000","message":"Uploaded patch set 11.","accounts_in_message":[],"_revision_number":11},{"id":"eab49e3987ebd8f314a484ad7f99758927824627","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-01 04:01:17.000000000","message":"Patch Set 11:\n\nBuild failed (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/e38cce22226c49e9b0ec1b6256465e79 : TIMED_OUT in 1h 31m 50s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/25a4e2abd924403f8ac0840d687af42c : SUCCESS in 1h 04m 54s\n- swift-probetests-centos-8-stream-arm64 https://zuul.opendev.org/t/openstack/build/57182db1a1e4468e85909eb9086ec672 : RETRY_LIMIT in 6m 11s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/66e64568ff8c4475b44feeb67b3796e3 : SUCCESS in 1h 04m 17s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/08ab310a640e4d1d979c64eecaedc947 : SUCCESS in 1h 26m 29s","accounts_in_message":[],"_revision_number":11},{"id":"4fd63046b49e950f9acb587d708fb3123d1d9cf3","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-01 04:05:10.000000000","message":"Patch Set 11: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/1a75a98a0c954a2483f900737ed2fc17 : SUCCESS in 12m 21s\n- grenade https://zuul.opendev.org/t/openstack/build/c2aa2ce485924426ac899b8063403ae1 : SUCCESS in 1h 04m 05s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/8bd351b7224443ec8facda85d5ba2946 : SUCCESS in 46m 20s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/74f2625c834142cdb0643de36ec7c7cb : SUCCESS in 48m 35s\n- swift-tox-func-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/306d19e06fe74ec281e16c9866e2531c : RETRY_LIMIT in 6m 27s (non-voting)\n- swift-tox-func-encryption-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/3b8fd738612d45219eb7230bce362263 : RETRY_LIMIT in 7m 15s (non-voting)\n- swift-tox-func-ec-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/72e8985bb4244c44b832b5b4a5714d33 : RETRY_LIMIT in 7m 37s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/06cdc775f3dc4856b64dcdfa7154131f : SUCCESS in 8m 32s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/e5be8383f68b496c9123e02fc0197316 : SUCCESS in 6m 50s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/e0ed85341b1249cfa14e24b09e6bc99f : SUCCESS in 16m 40s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/da778bd35034490caae1c25212b5e1d3 : SUCCESS in 16m 24s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/4595a300c4144ae693ce403238827c3d : TIMED_OUT in 1h 31m 26s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/2109eb4cf5df4e09856127c7d115cb80 : SUCCESS in 20m 05s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/468b000386934fc393640b6fcf136c9b : SUCCESS in 19m 43s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/80de642855604aafa7609d8c5c67f0d6 : SUCCESS in 19m 22s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/375a1623d5d240d4b0a7b49feb648408 : SUCCESS in 18m 15s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/b11ac86e1bcc467ea04f8c60c3c27598 : SUCCESS in 18m 46s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/70f07298e88b46188e0924c095abd81d : SUCCESS in 18m 33s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/8e794a722a3242ffbc8e4eef8fe8b8c3 : SUCCESS in 7m 59s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/df1d241c39c14fc7bed8686de106ee20 : SUCCESS in 27m 54s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/3e8f90e66268456dad5086c408f780b0 : SUCCESS in 57m 18s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/6bbe0accdeac4b05b3c8c682e6acad4d : RETRY_LIMIT in 3m 28s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/5d834f316a234f7cab1c7ead892d058c : SUCCESS in 40m 18s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/2585157fe50e45a1a6a9628665476cbd : SUCCESS in 42m 04s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/3b1f721d867941c88b4e7d47de6e971b : SUCCESS in 17m 34s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2304b1d38bdc42ce9616389a99be8d4d : SUCCESS in 7m 40s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/a966aa06e205444c8c3bf2d38cd2b673 : SUCCESS in 32m 27s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/554487702588437fa7c7f0a68ec14267 : SUCCESS in 52m 02s","accounts_in_message":[],"_revision_number":11},{"id":"3b4fea8afd0964bf8f7585b7ae3acdda781b1d60","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-02-02 05:15:59.000000000","message":"Patch Set 11:\n\n(2 comments)","accounts_in_message":[],"_revision_number":11},{"id":"b780e9d90ec543b6be6360994d4579c7e808af92","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-02-02 06:08:00.000000000","message":"Uploaded patch set 12.","accounts_in_message":[],"_revision_number":12},{"id":"dfdffecee7441c3ee763330a4adb4cc670815dfd","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-02-02 06:21:25.000000000","message":"Patch Set 12:\n\n(9 comments)","accounts_in_message":[],"_revision_number":12},{"id":"f989f809acdc44ceeb6b601a764410f62057ad9e","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-02 07:07:36.000000000","message":"Patch Set 12: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/02af09c9b06e47dc8a076c1470ab4ceb : SUCCESS in 8m 52s\n- grenade https://zuul.opendev.org/t/openstack/build/548a613ee3054fe8923b5896e78d1f9b : SUCCESS in 44m 36s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/3b79c9b1ba1d4579907d9c155fcb8cec : SUCCESS in 54m 33s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/800a8eb73ba04480bc195ead320fc9db : SUCCESS in 44m 01s\n- swift-tox-func-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/04d066e0855a4d9cb7f8e493b3169b5a : RETRY_LIMIT in 5m 49s (non-voting)\n- swift-tox-func-encryption-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/3dedd1b85b9d4e40b90b7b52b9449ca9 : RETRY_LIMIT in 5m 29s (non-voting)\n- swift-tox-func-ec-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/9d196283452e4d19ac04bad0190384d5 : RETRY_LIMIT in 5m 29s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/f27fa2bf42644bf5a500e2d29d09a0e7 : SUCCESS in 8m 25s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/6e93c4d552a443e68383f197e51cd8f5 : SUCCESS in 5m 40s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/7904c4e5123049fcba5b2336f0453dc0 : SUCCESS in 16m 42s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/c01eedeb63754fe1bb26c4eee38a65dc : SUCCESS in 16m 55s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/d459e8926253448f96da34f86fd42723 : SUCCESS in 16m 33s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/a1769fe5b9f64719a9eaa43e1a382436 : SUCCESS in 17m 47s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/9f019ad774aa47c896415bbb2c97ba98 : SUCCESS in 17m 06s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/ab9b78b152eb4c3e983cdffd9bf26e84 : SUCCESS in 16m 47s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/0213a6919326436c83e9046c07ffe2a1 : SUCCESS in 16m 40s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/8f6c7b3b28294f489d0e6f6050fabe0a : SUCCESS in 16m 44s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/fff6aa670df24f26908ee8942dc59fbd : SUCCESS in 16m 41s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/ac4b89f3f60d4a8183f92a7407be2412 : SUCCESS in 6m 04s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/59fa36cf86f9437a80d4a6d6f724a8fd : SUCCESS in 20m 09s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/5afab123f62c446b96325201699a9db7 : SUCCESS in 47m 50s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/0c2614af44a54e5eb439d72d9787892c : RETRY_LIMIT in 3m 53s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/dc79ed2ff57a468288cab9e6a698d6fb : SUCCESS in 41m 22s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/bf12a752fb8a48f8b908fc6f17d0bde7 : SUCCESS in 33m 19s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/e324c49c9feb45dd9d4b741d9660d765 : SUCCESS in 16m 36s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/c5b837ade71e4206a472e6cb80ba4663 : FAILURE in 5m 47s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/f05b50c7e76243fca317e88dbc2f4ecb : SUCCESS in 19m 25s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/f87960f637b948f6b9330d211be45fb3 : SUCCESS in 41m 13s","accounts_in_message":[],"_revision_number":12},{"id":"f8a224dbb95d252e4cfc1dd8c6aada07d8cc07cb","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-02 07:32:07.000000000","message":"Patch Set 12:\n\nBuild failed (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/a2288fafc5b841a08e29d7c2f7b674a9 : SUCCESS in 29m 00s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/5c506b01efe74e889cba28b49f3ae815 : SUCCESS in 1h 10m 33s\n- swift-probetests-centos-8-stream-arm64 https://zuul.opendev.org/t/openstack/build/4f086c13839d4c1d8df6e725fe204f93 : RETRY_LIMIT in 3m 50s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/2415fbabe2434f298022bb1dc4e76e05 : SUCCESS in 54m 02s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/c4a6de80f384410db52c1a7fca95804d : SUCCESS in 1h 21m 55s","accounts_in_message":[],"_revision_number":12},{"id":"a6011d58e3f053cce8d9b0c3b74a471146581581","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2022-02-04 05:34:34.000000000","message":"Patch Set 12:\n\n(1 comment)","accounts_in_message":[],"_revision_number":12},{"id":"b1fd50ae04418dc828e3e5b5588ee108e44fec9b","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-04 07:16:03.000000000","message":"Patch Set 12:\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/3a7ad04ba7614e6b96f342877cd7a8f2 : SUCCESS in 9m 51s\n- grenade https://zuul.opendev.org/t/openstack/build/5f9f8b840437429bb6c3344d3958dfc8 : SUCCESS in 54m 42s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/a5a07074c6634fe48748f43385bf8a99 : SUCCESS in 36m 06s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/42e9f967c9ef45089ca9b0bb648e6a2c : SUCCESS in 44m 09s\n- swift-tox-func-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/35161e9835524342a57f6e07bbdec82a : SUCCESS in 23m 03s (non-voting)\n- swift-tox-func-encryption-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/075857e877b34c8c95cc9ee1f6097daf : SUCCESS in 22m 17s (non-voting)\n- swift-tox-func-ec-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/375a3b8b66a147848ced5a45f28a628d : SUCCESS in 22m 06s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/1cada5193f044a53b7027040426b15b2 : SUCCESS in 8m 44s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/d0c3200aa3a44e44a081c70d682ecb49 : SUCCESS in 7m 23s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/6b8bade7c6be4ed0b9610a73d98874a8 : SUCCESS in 13m 05s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/ffcea1810b3d4cd087eabd26213a7aea : SUCCESS in 12m 28s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/82fcd56f1ee64a598419736999d11241 : SUCCESS in 22m 20s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/7b146c29f8984efd8752fa34e67783af : SUCCESS in 18m 34s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/bfea9f08d63a4dc39fe86c5a5a751b0d : SUCCESS in 24m 59s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/2b9aafec29e647feb6e89a52adc244e3 : SUCCESS in 14m 56s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/8e87738981ef4533bf2c948441d6e285 : SUCCESS in 17m 04s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/3febd78824964a22bdaac2c85a71af94 : SUCCESS in 18m 39s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/53f979f2bc9d42c9a67f6f48cc2ab540 : SUCCESS in 18m 46s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/19ccc69246b9401b8d73f782b6477be0 : SUCCESS in 6m 42s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/1eb89c972c4349449a3bafb644e6492c : SUCCESS in 23m 56s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/70294aee0dc541e2b6a7227593fc48a8 : SUCCESS in 1h 01m 02s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/b6360a16408f40939508791fd8f1c276 : SUCCESS in 56m 22s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/58781a50c19745deb009fcfe3b4b76e8 : SUCCESS in 41m 03s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/d6b1ae04230b4e2caf7fd1678caec2ac : SUCCESS in 40m 34s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/6d594aef384d46daae80e25213917e84 : SUCCESS in 17m 14s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/a45a1a2ae638414c9ad15cce1c94cd2d : FAILURE in 5m 52s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/a26710dadd704bf5b78b3c21c242b33c : SUCCESS in 28m 53s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/b9462cf5eb424a9f8bfa394d59a43734 : SUCCESS in 48m 38s","accounts_in_message":[],"_revision_number":12},{"id":"341ae3cb9ff7917a529c1259899519e9030600d2","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-04 08:41:10.000000000","message":"Patch Set 12:\n\nBuild failed (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/166c9ae2014e479d9e6af34bf22c0ac8 : SUCCESS in 59m 23s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/93eab04e0dab4d939b3b0d44b6f60796 : SUCCESS in 1h 13m 53s\n- swift-probetests-centos-8-stream-arm64 https://zuul.opendev.org/t/openstack/build/8983a916f7d84af098d925989e81ceb8 : TIMED_OUT in 3h 01m 31s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/bf25735937794f7bad6741d784244b78 : SUCCESS in 1h 19m 57s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/9a25403c42af473bad66dc4b2a7e34a1 : SUCCESS in 52m 02s","accounts_in_message":[],"_revision_number":12},{"id":"1ea7736c21d0d16c98540946743b70731b327863","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-02-07 05:10:03.000000000","message":"Uploaded patch set 13.","accounts_in_message":[],"_revision_number":13},{"id":"3d4627f8f5fb070236e82039a081fa84b7e67b57","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-07 06:42:55.000000000","message":"Patch Set 13: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/c51161f61e5a4189911964fa0209fa9d : SUCCESS in 10m 25s\n- grenade https://zuul.opendev.org/t/openstack/build/b1060ab43d7141979eba1792d1ba6c65 : SUCCESS in 48m 52s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/ae2c6771db9b44898aeaff43e9fc498c : SUCCESS in 1h 26m 07s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/b0fdf722be05438c8b4a19a5ae3103b3 : SUCCESS in 49m 46s\n- swift-tox-func-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/7ed15276f6d646ae8ccf903516d01ff7 : SUCCESS in 21m 03s (non-voting)\n- swift-tox-func-encryption-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/d85889632bb64d85a171b6648c3dfb60 : SUCCESS in 27m 36s (non-voting)\n- swift-tox-func-ec-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/8193e09ecc524c4d850bdc835431c3d6 : SUCCESS in 30m 50s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/7a36e585cd1e48fc97139849a073ab31 : SUCCESS in 10m 54s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/db96ea2630a94e1ca7160efb4909a0ad : SUCCESS in 6m 18s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/409ba9af954448d995d34bef87caadce : SUCCESS in 16m 55s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/07bae754d86c4516b29d0bd51fc726b5 : SUCCESS in 15m 25s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/3bdd16a7645b46fcacbbdb7b36a33c50 : TIMED_OUT in 1h 31m 14s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/3a88557ae54c441b8d792b5240cf5154 : SUCCESS in 18m 36s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/256c7ca59e3c459db6bcf506be1e163f : SUCCESS in 13m 32s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/96a2622a98584b9a833f14325435481f : SUCCESS in 17m 21s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/f75bba1994be4ffda0b245981c103716 : SUCCESS in 19m 40s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/c182f5a04a2848b691472ca0503b126e : SUCCESS in 12m 08s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/361390ff54dd48c1acd5277933af0533 : SUCCESS in 16m 36s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/c5433ebd1a824eaca74794488742e764 : SUCCESS in 6m 55s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/522a46bfc2334382b817cdc986d4367d : SUCCESS in 25m 02s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/95718a22796243328326b9875bd46660 : SUCCESS in 51m 53s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/9fbb8dbcc94745659a3a78bedfb1b19b : SUCCESS in 56m 28s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/e6de831fcf47453c983ac4212dbadf57 : SUCCESS in 52m 21s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/741a60ee0c2c4ab5b26bbe35029ee0c3 : SUCCESS in 34m 36s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/af32adfb72654f158aaaab67c5a7841d : SUCCESS in 19m 24s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/d561f2b50fa3431cb7e4a41b706bf020 : SUCCESS in 4m 51s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/64630f65b755411eae4d082f6f92b78f : SUCCESS in 20m 25s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/8c6289e65a294c1fbea41fa59dcfc0c6 : SUCCESS in 43m 09s","accounts_in_message":[],"_revision_number":13},{"id":"27600e997ceddb6e94d71435f25093f80d42bd04","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-07 08:13:51.000000000","message":"Patch Set 13:\n\nBuild failed (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/5f58df59e048483aa64db2a568d5738a : SUCCESS in 43m 20s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/42d61821f6e74deb8fa4d82ee441d9db : SUCCESS in 52m 12s\n- swift-probetests-centos-8-stream-arm64 https://zuul.opendev.org/t/openstack/build/0344a5ba933f490c94c430f76995e835 : TIMED_OUT in 3h 01m 22s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/89dfe397d6624578bc5ac58bd41ed9b8 : SUCCESS in 1h 03m 59s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/3effc48d4ec541eaa5d876dc39750cde : SUCCESS in 1h 02m 45s","accounts_in_message":[],"_revision_number":13},{"id":"197f6a370c8d65af99ddc09db5f445606a58b2af","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-02-07 23:29:32.000000000","message":"Patch Set 13:\n\n(1 comment)","accounts_in_message":[],"_revision_number":13},{"id":"68e82cfb837289aac6f3a6f4f386803643f6e273","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-08 00:37:24.000000000","message":"Patch Set 13: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/ff991a8f60364712aa2953a39cbd0cac : SUCCESS in 8m 58s\n- grenade https://zuul.opendev.org/t/openstack/build/79c00b2cd40f454fa2f07859ab053b2d : SUCCESS in 1h 01m 13s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/cd963054dfbf4f69b6fa429d1aa9af55 : SUCCESS in 51m 41s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/10cd722c9ad04808a9a0d34ad07f3d8a : SUCCESS in 46m 28s\n- swift-tox-func-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/d501b1f69eb948b08463204c6d1e3d87 : SUCCESS in 20m 37s (non-voting)\n- swift-tox-func-encryption-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/02f30799ed734ea5980a7a269852389b : SUCCESS in 19m 07s (non-voting)\n- swift-tox-func-ec-py36-centos-8-stream-fips https://zuul.opendev.org/t/openstack/build/47509dd6270f4252956ca1abe9cf2d5b : SUCCESS in 18m 42s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/96671a50a13d43febd5ab6116999578d : SUCCESS in 7m 57s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/6c373ec55ae5449aac6a67b5e35d723a : SUCCESS in 6m 04s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/a922c62d028b4ccdb1afb98aa1055c0e : SUCCESS in 15m 30s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/fc5767c8c2db4f2fa589c6dff523fb58 : SUCCESS in 15m 01s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/02052969f4da483dace34dc8a0a72d4f : SUCCESS in 17m 18s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/056100c10f5041f68e5aaed117bcb713 : SUCCESS in 15m 32s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/227411f2e0154d28b5d250895e344b65 : SUCCESS in 16m 20s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/e42e1dc4e75140ec9076c580ba55be6d : SUCCESS in 17m 02s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/ab411c7fd5a042be8ef6cedbef60292f : SUCCESS in 16m 19s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/9d1895a8895c46aebea11a4c613bc3a8 : SUCCESS in 15m 08s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/37fd53f3843041cf8e69ab96e5319219 : SUCCESS in 12m 23s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/4f6b8dd0a747415a8e832a0ff63ce336 : SUCCESS in 6m 01s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/323426da0f4045ba9ef8a7a5cb2aab2a : SUCCESS in 16m 32s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/26bac466ee884ed5b79d5a0a60a28a0b : SUCCESS in 51m 01s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/9cc24d45c26346748b8ffd12deccb27e : SUCCESS in 50m 24s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/975d8ee6a98b4e51a5d76e08d59393b4 : SUCCESS in 38m 22s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/eb88cdd8d8724799b95e1cd224c2087a : SUCCESS in 27m 30s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/f1b8944d99e0420a8a2112148e5c6009 : SUCCESS in 15m 17s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/983012a27dbb4239b6633f1c00eb9ea7 : SUCCESS in 5m 54s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/9d2aeeb9156e40bf8d2c1891612c7288 : SUCCESS in 17m 12s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/cfe02237bcf94e2a83fd1e37398c0da9 : SUCCESS in 53m 13s","accounts_in_message":[],"_revision_number":13},{"id":"c31514d215b509b8fc5b65d909d66cfe213de0f0","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2022-02-08 02:34:09.000000000","message":"Patch Set 13:\n\nBuild failed (ARM64 pipeline).\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/02e1a683cad349adb5a98caec5fc95bd : SUCCESS in 47m 24s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/90d623016f5d4306848d265fb4691e43 : SUCCESS in 46m 48s\n- swift-probetests-centos-8-stream-arm64 https://zuul.opendev.org/t/openstack/build/6aa694c966fe4fa98587fc943de51ad1 : TIMED_OUT in 3h 01m 34s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/63784ee9756f4243ae6029000ab02519 : SUCCESS in 1h 00m 25s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/224274b40dab4cc390fa6953ef7e39c0 : SUCCESS in 1h 00m 38s","accounts_in_message":[],"_revision_number":13},{"id":"d242a19bc814f6463b65101369fff8172b29ef0b","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2022-05-02 02:11:26.000000000","message":"Patch Set 13:\n\n(1 comment)","accounts_in_message":[],"_revision_number":13}],"current_revision_number":13,"current_revision":"48b3fa1dd7c9e17b2dcda7eb1248354290b1cc2e","revisions":{"673b0479c42b1002cc37706a32f80eca8e335a5a":{"kind":"REWORK","_number":1,"created":"2021-09-30 03:53:02.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/1"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"subject":"container-updater: demonstrate account stat blockage with probe test","message":"container-updater: demonstrate account stat blockage with probe test\n\nWe have seen account stats become distorted: in worst case an account with\nonly one container drops to reporting zero bytes and zero object count\ndespite the container having objects.\nAnd can remain stuck until another object is PUT or DELETEd into the\ncontainer.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1, so the handoff db has zero\n     object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more updates to account\n  6. container replication leaves primary put timestamp unchanged because\n     handoff put time is older than primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes used until an\nobject is put or deleted at which point a primary will update the account\nstats.\n\nThis patch rolls all this up in a probe test so we have something to\nfix.\n\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/673b0479c42b1002cc37706a32f80eca8e335a5a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/673b0479c42b1002cc37706a32f80eca8e335a5a"}]},"branch":"refs/heads/master"},"d83303bd88f24965d903af552ea61314dd139452":{"kind":"REWORK","_number":2,"created":"2021-10-01 07:34:02.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/2"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-01 07:20:31.000000000","tz":600},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d83303bd88f24965d903af552ea61314dd139452"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d83303bd88f24965d903af552ea61314dd139452"}]},"branch":"refs/heads/master"},"5b53a1d494d45ac02dbfe8d998a4b1ad2c40cef5":{"kind":"REWORK","_number":3,"created":"2021-10-03 22:27:29.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/3"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-03 22:27:15.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5b53a1d494d45ac02dbfe8d998a4b1ad2c40cef5"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5b53a1d494d45ac02dbfe8d998a4b1ad2c40cef5"}]},"branch":"refs/heads/master"},"f09cbb097fa746d0ff2de7f8974a1064991cbbd9":{"kind":"REWORK","_number":4,"created":"2021-10-04 00:07:34.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/4"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-04 00:07:19.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/f09cbb097fa746d0ff2de7f8974a1064991cbbd9"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/f09cbb097fa746d0ff2de7f8974a1064991cbbd9"}]},"branch":"refs/heads/master"},"e112cf809584d3336d188daa8afbe446a6c0694b":{"kind":"REWORK","_number":5,"created":"2021-10-04 05:38:20.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/5"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-04 05:38:08.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/e112cf809584d3336d188daa8afbe446a6c0694b"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/e112cf809584d3336d188daa8afbe446a6c0694b"}]},"branch":"refs/heads/master"},"6241759637f1f852efd0820937cb800abbee3b74":{"kind":"REWORK","_number":6,"created":"2021-10-05 08:58:37.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/6"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-05 08:54:55.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs). So a new\noption now existing in the container updater, all_update.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/6241759637f1f852efd0820937cb800abbee3b74"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/6241759637f1f852efd0820937cb800abbee3b74"}]},"branch":"refs/heads/master"},"08faa1c5b0766262dbabaa68325987ac92b43192":{"kind":"REWORK","_number":7,"created":"2021-10-06 01:26:14.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/7","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/7","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/7"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-06 01:25:59.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs). So a new\noption now existing in the container updater, all_update.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/08faa1c5b0766262dbabaa68325987ac92b43192"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/08faa1c5b0766262dbabaa68325987ac92b43192"}]},"branch":"refs/heads/master"},"9bc4d675c1cc794c7a17a552b256440fa4b61b87":{"kind":"REWORK","_number":8,"created":"2021-10-13 08:48:08.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/8","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/8","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/8"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-13 08:45:44.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs).\nTo solve this the container-updater now loads the raw container ring\nmetadata (only) to grab the number of replicas. If it\u0027s only a single\nreplica ring then the old behaviour stands, that is any container will\nupdate the account.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/9bc4d675c1cc794c7a17a552b256440fa4b61b87"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/9bc4d675c1cc794c7a17a552b256440fa4b61b87"}]},"branch":"refs/heads/master"},"5f144a4aa4c8a568c5c56e6a8d447b6bf8c4945f":{"kind":"REWORK","_number":9,"created":"2021-10-13 22:28:09.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/9","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/9","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/9 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/9 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/9 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/9"}}},"commit":{"parents":[{"commit":"029e57679c639edda87816482685b8f48de67dd9","subject":"Merge \"Clean up extra spaces\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/029e57679c639edda87816482685b8f48de67dd9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-10-13 22:27:54.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs).\nTo solve this the container-updater now loads the raw container ring\nmetadata (only) to grab the number of replicas. If it\u0027s only a single\nreplica ring then the old behaviour stands, that is any container will\nupdate the account.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5f144a4aa4c8a568c5c56e6a8d447b6bf8c4945f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5f144a4aa4c8a568c5c56e6a8d447b6bf8c4945f"}]},"branch":"refs/heads/master"},"21282c3df961f3fd13667bb97a86578c76546618":{"kind":"REWORK","_number":10,"created":"2022-01-31 06:21:38.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/10","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/10","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/10 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/10 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/10 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/10"}}},"commit":{"parents":[{"commit":"3429a11c395b3dcdfb7f823ebf7f31c62a933d27","subject":"Merge \"tests: Clean up some dangling timeouts\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/3429a11c395b3dcdfb7f823ebf7f31c62a933d27"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2022-01-31 06:21:19.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs).\nTo solve this the container-updater now loads the raw container ring\nmetadata (only) to grab the number of replicas. If it\u0027s only a single\nreplica ring then the old behaviour stands, that is any container will\nupdate the account.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/21282c3df961f3fd13667bb97a86578c76546618"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/21282c3df961f3fd13667bb97a86578c76546618"}]},"branch":"refs/heads/master"},"59c2c1fd967c6a83bdf39854e09402d08e26f478":{"kind":"REWORK","_number":11,"created":"2022-02-01 02:20:46.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/11","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/11","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/11 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/11 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/11 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/11"}}},"commit":{"parents":[{"commit":"3429a11c395b3dcdfb7f823ebf7f31c62a933d27","subject":"Merge \"tests: Clean up some dangling timeouts\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/3429a11c395b3dcdfb7f823ebf7f31c62a933d27"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2022-02-01 02:20:32.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db that git a replica after a rebalance, in this\n     case the other \"old\" replicas have yet talked with it so\n     again it doesn\u0027t have a good enough view of the world yet.\n     But after 1 replication sync it\u0027s defintely got a much\n     better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs).\nTo solve this the container-updater now loads the raw container ring\nmetadata (only) to grab the number of replicas. If it\u0027s only a single\nreplica ring then the old behaviour stands, that is any container will\nupdate the account.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/59c2c1fd967c6a83bdf39854e09402d08e26f478"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/59c2c1fd967c6a83bdf39854e09402d08e26f478"}]},"branch":"refs/heads/master"},"4ba904d5253b6dc8d47a56be674884eda1bd5ad1":{"kind":"REWORK","_number":12,"created":"2022-02-02 06:08:00.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/12","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/12","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/12 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/12 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/12 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/12"}}},"commit":{"parents":[{"commit":"6e0ec0ad9cb5f561246c92c6f4b83a39e05dfd4c","subject":"Merge \"s3api: Fix non-ascii MPUs\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/6e0ec0ad9cb5f561246c92c6f4b83a39e05dfd4c"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2022-02-02 06:03:58.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db on a replica after a rebalance, in this\n     case the other \"old\" replicas haven\u0027t yet talked with it so\n     again it doesn\u0027t have a good enough view of the world... yet.\n     But after 1 replication sync it\u0027s got a much better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs).\nTo solve this the container-updater now loads container ring\nmetadata (only) to grab the number of replicas. If it\u0027s only a single\nreplica ring then the old behaviour stands, that is any container will\nupdate the account.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4ba904d5253b6dc8d47a56be674884eda1bd5ad1"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4ba904d5253b6dc8d47a56be674884eda1bd5ad1"}]},"branch":"refs/heads/master"},"48b3fa1dd7c9e17b2dcda7eb1248354290b1cc2e":{"kind":"REWORK","_number":13,"created":"2022-02-07 05:10:03.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/33/811833/13","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/33/811833/13","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/13 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/13 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/33/811833/13 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/33/811833/13"}}},"commit":{"parents":[{"commit":"6e0ec0ad9cb5f561246c92c6f4b83a39e05dfd4c","subject":"Merge \"s3api: Fix non-ascii MPUs\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/6e0ec0ad9cb5f561246c92c6f4b83a39e05dfd4c"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-09-30 03:45:14.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2022-02-07 05:09:18.000000000","tz":660},"subject":"container-updater: no incoming syncs no account update","message":"container-updater: no incoming syncs no account update\n\nWe have seen account stats become distorted: in worst case\nan account with only one container drops to reporting zero\nbytes and zero object count despite the container having\nobjects. And can remain stuck until another object is PUT\nor DELETEd into the container.\n\nIn general the edgecase goes:\n  1. a container PUT lands on a handoff at t1,\n     so the handoff db has zero object count.\n  2. subsequent PUTs land on primaries, say at t2\n  3. primaries update account with valid obj count and bytes\n  4. handoff update account with zero obj count and bytes\n  5. no more objects are PUT so primaries make no more\n     updates to account\n  6. container replication leaves primary put timestamp\n     unchanged because handoff put time is older than\n     primary -\u003e no primary updates to account\n\nThen the account remains stuck with zero obj count and bytes\nused until an object is put or deleted at which point a\nprimary will update the account stats.\n\nTo solve this problem this patch adds an additional check\nto the container-updater. If the broker has no incoming-syncs\nthen don\u0027t update. Missing incoming syncs could mean 3 things:\n\n  1. It could be a handoff database, a primary wont replicate\n     with it.\n     - If this is the case then it doesn\u0027t have a good view of\n       the stats to update the account. So we don\u0027t want to\n       report.\n  2. Its a new db on a replica after a rebalance, in this\n     case the other \"old\" replicas haven\u0027t yet talked with it so\n     again it doesn\u0027t have a good enough view of the world... yet.\n     But after 1 replication sync it\u0027s got a much better one.\n  3. Its a brand new container, after the initial PUT but before\n     the replicators have had a chance to run. In this case we\n     loose nothing in waiting until one replica has replicated\n     with the others.\n\nTo make things a little more complicated, in order to support single\nreplica container rings we need the ability to all containers to be\nable to undate (single replica will have no incoming_syncs).\nTo solve this the container-updater now loads container ring\nmetadata (only) to grab the number of replicas. If it\u0027s only a single\nreplica ring then the old behaviour stands, that is any container will\nupdate the account.\n\nCloses-Bug: #1945595\nChange-Id: Idacc0d80ec1a73bad56696056463deeb075c2171\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/48b3fa1dd7c9e17b2dcda7eb1248354290b1cc2e"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/48b3fa1dd7c9e17b2dcda7eb1248354290b1cc2e"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY"},{"label":"Workflow","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}