)]}'
{"id":"openstack%2Fswift~791885","triplet_id":"openstack%2Fswift~master~I48d855bdbb7dca46e2b1fd6054f9b5017c331685","project":"openstack/swift","branch":"master","topic":"p-sharder-unify-conf","hashtags":[],"change_id":"I48d855bdbb7dca46e2b1fd6054f9b5017c331685","subject":"sharding: Add auto mode to rows_per_shard","status":"NEW","created":"2021-05-18 04:43:35.000000000","updated":"2021-05-26 06:32:19.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":18,"unresolved_comment_count":11,"has_review_started":true,"meta_rev_id":"292fe0efee1cc6f9dfd45061cb04d0cb75ae5fa7","_number":791885,"virtual_id_number":791885,"owner":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"actions":{},"labels":{"Verified":{"disliked":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"tag":"autogenerated:zuul:check","value":-1,"date":"2021-05-25 08:31:00.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":-1,"default_value":0,"optional":true},"Code-Review":{"disliked":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"all":[{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"value":-1,"date":"2021-05-25 14:40:42.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","value":-1,"default_value":0,"optional":true},"Workflow":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2021-05-18 05:41:54.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2021-05-18 14:31:32.000000000","updated_by":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"reviewer":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"state":"CC"},{"updated":"2021-05-25 14:40:42.000000000","updated_by":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"reviewer":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"state":"REVIEWER"}],"messages":[{"id":"afacd8512da195143666f2803178d39e7203c39b","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-18 04:43:35.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"22668b576055fa329da9279add06ce80130288f7","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-18 04:48:07.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"37b73913acb4361ac8b31dc4f6ae000b83fd0b6c","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-18 04:49:43.000000000","message":"Patch Set 1: Workflow-1\n\nstill needs tests.","accounts_in_message":[],"_revision_number":1},{"id":"9277b40e2bfe3eb472f8c7fa9a5bffce703588fe","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-05-18 05:41:54.000000000","message":"Patch Set 1: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/bb5887e2311545ea9b3ed734d85c3dea : SUCCESS in 8m 54s\n- grenade https://zuul.opendev.org/t/openstack/build/55b3d7e13f4e47c1b16aee989148f8ab : SUCCESS in 57m 26s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/3c5428699d4a46c18e87cadfb49e0e57 : SUCCESS in 33m 26s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/366ac36aa4f74f57b9423aa0e33b269b : SUCCESS in 7m 52s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/dcd6b07cc96a432fa93ec39eb7549626 : SUCCESS in 6m 01s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/3d15f06b992d49a09c060e2d1d89a9b1 : FAILURE in 38m 47s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/0e1b7451d90e482d8a24b6a2e8d60ba0 : FAILURE in 15m 06s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/fc936f6eef1646a5872d9f96fcccc384 : FAILURE in 13m 51s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/31bebfb8b1d64910b8c13739c25337a3 : FAILURE in 14m 38s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/a77a8f44727c4609a15c8ff3461e64a0 : FAILURE in 16m 03s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/3fe60e0cb59d42969fe208f40cea692a : SUCCESS in 14m 39s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/da55b26299624a80a50dab98043cbab3 : SUCCESS in 15m 08s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/ae70bc3ebdc84be0b4ee7a9a10323d07 : SUCCESS in 15m 40s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/b01645cbc1a6420c8fccd263a3bc01b3 : SUCCESS in 15m 17s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/68f3401ebc024bafb9e3d12213c689c4 : SUCCESS in 19m 15s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/cb0ca3dd2af9426a82c404e5f8bdf160 : SUCCESS in 16m 21s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/e10691806e8340d3a05c2fe7c1bb5bd7 : SUCCESS in 9m 08s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/e2b6d7d8aa3643fab692227955b3d68a : SUCCESS in 19m 03s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/7b7b601cc9ab41f6bdd57de7f421fe75 : FAILURE in 44m 50s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/fc5a9cf226d940339ffa0260733a4f47 : FAILURE in 41m 38s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/3fdc175fac104725a403cd94a632770f : SUCCESS in 27m 16s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/a943a071a0cf45f09c69c81e5797e348 : SUCCESS in 36m 55s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/61c3777f768b4b9d83d435afb8b29565 : FAILURE in 16m 35s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2e6cfdeede8948b2b4b2332223718e97 : FAILURE in 5m 00s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/3d67ce2e5d9541c29109d2839c4f3fd7 : SUCCESS in 29m 37s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/b9048cd415ff4738ab009963907eab2e : SUCCESS in 44m 43s\n\nWarning:\n  Comments left for invalid file test/unit/container/test_sharder.py","accounts_in_message":[],"_revision_number":1},{"id":"98ad5eee416994214dc18a904d68f05b0092f219","author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"date":"2021-05-18 14:31:32.000000000","message":"Patch Set 1:\n\n(2 comments)\n\nSeems like we have two new ideas going on here and in the previous patch:\n\n1. A priori make an estimate of the best shard size to carve the container into equal sized shards e.g. 500k +/- some adjustment.\n\n2. Recognising that the result of 1 may be stale by the time we get to the end of finding shards, check that the final shard is \"big enough\".\n\nWe could just do 2 and stick with shard_size \u003d 500k, so we might get:\n\n[500k, 500k, ...., 501k] or [500k, 500k, ...., 300k] but not [500k, 500k, ...., 1k]\n\nor we could do both 1 and 2, so we might get :\n\n[510k, 510k, 520k, 520k, ...., 490k, 491k]\n\nIf we do 1, it seems reasonable to expect any final shard to either be small (some objects were added while we were searching for uppers) or just under shard_size (some objects were deleted). So it might be that for 2 a simple check that the final shard isn\u0027t \u0027too small\u0027 is sufficient without needing to get the exact count.","accounts_in_message":[],"_revision_number":1},{"id":"a4f5f22ea0c8241e4e9e44d77650ea23eeb36c74","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-19 06:29:48.000000000","message":"Patch Set 1:\n\n(2 comments)","accounts_in_message":[],"_revision_number":1},{"id":"ca623c391d28bbcaf46b528f3e807ed87817931f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-21 07:28:48.000000000","message":"Uploaded patch set 2: Patch Set 1 was rebased.","accounts_in_message":[],"_revision_number":2},{"id":"82a02a552da18616294dd1fcfbbed53fa864fd90","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-05-21 08:33:20.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/ce23c30a51664c22adc955cd9ad10722 : SUCCESS in 9m 06s\n- grenade https://zuul.opendev.org/t/openstack/build/b908d3c490c542919dbe033d49eb58e5 : SUCCESS in 1h 02m 46s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/e0f02742a16043259e1dbaaa062f696d : SUCCESS in 43m 17s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/6b87bfb9b91e4f12a00177a7a181660f : SUCCESS in 9m 08s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/fafd6bfce33a4b8584918899c8ea1595 : SUCCESS in 6m 50s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/9937ee1c176e473a97b7ca4c0efc78ee : FAILURE in 21m 59s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/ab2d74e78af84c22aa25da509ce98f16 : FAILURE in 17m 34s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/cee5c8132aa54396adf1dc20a74a9d6a : FAILURE in 22m 09s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/2a7219e5acfd435393911937bc96f3b1 : FAILURE in 16m 28s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/30361968b48b4dea86a284f7330d5a3f : FAILURE in 20m 58s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/1635137d1e1d446fb0158cdf6aa7cd0d : SUCCESS in 17m 36s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/888b7b96539845b1b558431320450ca0 : SUCCESS in 23m 28s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/43b79d0cc4a4403689180054ef3f78c5 : SUCCESS in 18m 31s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/a7bba65e34dd41ee833bbb9fa27edc99 : SUCCESS in 21m 13s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/c68a3082c87e471798c7270072f46c64 : SUCCESS in 19m 06s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/3c0d6fd825c64653b0115d908f65ae76 : SUCCESS in 23m 04s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/3f49d193f8164a958a4c1d27e2a383ec : SUCCESS in 6m 41s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/549b514988b845a9ac2effb0c663bf76 : SUCCESS in 24m 33s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/5fd1dea4427b4008a7480fe7f43bbb29 : FAILURE in 50m 17s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/d942b50e0e7d478cbc6301059aa66f65 : FAILURE in 52m 49s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/be5e758778ac4eac8a25ed2a591aa415 : SUCCESS in 40m 57s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/d8a21e6ac0ba4ed9ab87232cf234b15c : SUCCESS in 38m 55s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/d86b36b5b48f416e899035a6499c4f0d : FAILURE in 17m 34s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/488846046b0e44b9828b32ac7307894a : SUCCESS in 6m 40s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/d05fab95bfef43cd8bd238175abe6c95 : SUCCESS in 23m 46s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/119e6339e47e4da3b77faf3346a118f7 : SUCCESS in 46m 48s","accounts_in_message":[],"_revision_number":2},{"id":"48f362488b306714040e5a28246426dc04dc286d","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-25 05:55:14.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"3814541dd6b3b0b894a96372e5f65966f5a57ae0","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-25 07:18:19.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"867c5e5e0dea5dabed4d51d5e1e6d4b2f415f16a","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-25 07:23:40.000000000","message":"Patch Set 4:\n\n(1 comment)\n\nThis is just a reorganise of what I had up before. Just flipped the order and pulled the auto stuff into it.\n\nHaven\u0027t looked into the grabbing object_count and then again to make sure things haven\u0027t changed yet.. because might be able to use that to do calculation rather then doing a count of what\u0027s left, as object_count is keep up to day by a trigger.\nHaving said that, the count is only called when we know we\u0027re at the end _and_ it only hits the name,delete index, so also not too expensive.\n\nCan look into though if we decide it\u0027s necessary.","accounts_in_message":[],"_revision_number":4},{"id":"32b4ca897c012a10f65d23dfa24000f1049f426d","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-05-25 08:31:00.000000000","message":"Patch Set 4: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/59467c2be8b545349a4ca5f918612481 : SUCCESS in 8m 05s\n- grenade https://zuul.opendev.org/t/openstack/build/597c79595db1459ca51f338049fae889 : SUCCESS in 1h 06m 00s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/1cb0bb8fbdd041a4a3014bb70b1e5e0d : SUCCESS in 47m 10s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/4b3181b65c984baeaf8ca9d8ae2ac40c : SUCCESS in 9m 09s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/91b13dab182a474ab44c0997d19f72b6 : SUCCESS in 7m 07s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/fba97d7e5e6a4f9ba2314f595c17220b : SUCCESS in 15m 57s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/46f102bd446541f4b14090e9119d2773 : SUCCESS in 16m 20s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/683c1da8365246c6baf4a9f4f13cecb5 : SUCCESS in 19m 00s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/25378dace3b04f59ba8b28d4fdabfe1a : SUCCESS in 21m 59s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/5d3cfae2901d4023916ab8853c03bf5e : SUCCESS in 16m 27s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/c7c59d127dae4b66a636dd2e50766e17 : SUCCESS in 18m 39s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/d71be01b95744c73a062325ab08d9c99 : SUCCESS in 21m 22s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/6a28da7b52d74d5986b5933464952510 : SUCCESS in 19m 57s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/ab2a66d583e04647902dee5e6645973f : SUCCESS in 22m 59s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/4e060b6e018040cfbf6301167bdd92aa : SUCCESS in 19m 29s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/4937c6956b5a4abfa286f70b5d273d65 : SUCCESS in 27m 24s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/a3732112f6934770b3564b9135c52c65 : SUCCESS in 6m 47s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/212b1af2541346f99088ae4e145670cc : SUCCESS in 23m 06s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/ca849c35c5db4837b75b2017785e7d4d : FAILURE in 48m 14s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/a5322e30d2ae4a4ebd43798b815cfda7 : FAILURE in 49m 44s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/4a6f06fd451d4dcdb918680bbda8dbb6 : SUCCESS in 41m 19s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/0fd6f06cacda424394f24994c4f057a2 : SUCCESS in 40m 59s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/22dbfba8af3c40d08a9ac8e87f012cbf : SUCCESS in 17m 09s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/8c0ca02479274fb6a99cdf3c8f62b60d : SUCCESS in 6m 34s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/0638b1588de24cfd8d5e2da992c3e063 : SUCCESS in 30m 54s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/43b7b88c354f46a9ae986fe429ce79d4 : SUCCESS in 48m 39s","accounts_in_message":[],"_revision_number":4},{"id":"99c0d800b56a492971a752a5bdb25e2e8c7795f8","author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"date":"2021-05-25 14:40:42.000000000","message":"Patch Set 4: Code-Review-1\n\n(7 comments)\n\nI think we should and could avoid the repetitive extra db queries in the backend, and aim for just one extra query. But doing so means tidying up the final shard range in the callers, not in the backend. I\u0027m OK with that - it just needs a helper method that sharder and s-m-s-r can call, and it avoids burying complexity (that we might want to make configurable) down in the broker.\n\nWhile playing with this I realised that rows_per_shard \u003d auto is not as flexible as I had first imagined, because it relies on a default shard_container_threshold. So I played with an alternative --minimum-shard-size option.\n\nI explored but did not complete both ideas in https://review.opendev.org/c/openstack/swift/+/792986\n\nThere doesn\u0027t seem to be any tests here other than for the option passing - I also put a fledgling test in https://review.opendev.org/c/openstack/swift/+/792986","accounts_in_message":[],"_revision_number":4},{"id":"292fe0efee1cc6f9dfd45061cb04d0cb75ae5fa7","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2021-05-26 06:32:19.000000000","message":"Patch Set 4:\n\n(5 comments)\n\n\u003e Patch Set 4: Code-Review-1\n\u003e \n\u003e (7 comments)\n\u003e \n\u003e I think we should and could avoid the repetitive extra db queries in the backend, and aim for just one extra query. But doing so means tidying up the final shard range in the callers, not in the backend. I\u0027m OK with that - it just needs a helper method that sharder and s-m-s-r can call, and it avoids burying complexity (that we might want to make configurable) down in the broker.\n\u003e \n\u003e While playing with this I realised that rows_per_shard \u003d auto is not as flexible as I had first imagined, because it relies on a default shard_container_threshold. So I played with an alternative --minimum-shard-size option.\n\u003e \n\u003e I explored but did not complete both ideas in https://review.opendev.org/c/openstack/swift/+/792986\n\u003e \n\u003e There doesn\u0027t seem to be any tests here other than for the option passing - I also put a fledgling test in https://review.opendev.org/c/openstack/swift/+/792986\n\nYeah, this was origianlly the second patch in the series, and hadn\u0027t had much test love.\n\nGreat thoughts, originally I thought about just having it always on and working in the broker. So not triggered by auto and no correct_tail. Just start doing the right thing. In current sharding we dont guarantee the last ShardRange is shard_size size (because it most likely will be smaller). So maybe having the last 2 potentually not shard_size doesn\u0027t seem too much of a stretch.\n\nBut having said that, if we ever want to pass in the overload factor it needs to be plumbed quite far down.\n\nSorry lost alot of my day on some other tasks. But will try and get a look at your other approaches before I call it a day.","accounts_in_message":[],"_revision_number":4}],"current_revision_number":4,"current_revision":"787fc75d9d7075bbaa2fc70ec3157a253d7f594d","revisions":{"0c6188fb179f14e74e83ebf398da0e1d4c9252ce":{"kind":"REWORK","_number":1,"created":"2021-05-18 04:43:35.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/85/791885/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/85/791885/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/85/791885/1"}}},"commit":{"parents":[{"commit":"4075ecc13d76e3cc6a34c8c4f608d1f1c8a07ef9","subject":"WIP: sharding: Allow rows_per_shard to be \u0027auto\u0027","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4075ecc13d76e3cc6a34c8c4f608d1f1c8a07ef9"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-18 04:31:29.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-18 04:31:29.000000000","tz":600},"subject":"sharding: make broker\u0027s _get_next_shard_range_upper smarter","message":"sharding: make broker\u0027s _get_next_shard_range_upper smarter\n\nCurrently when we ask a container broker to give us the next shard range\nupper if just reads the next `shard_size` objects from the last and\neither return it or if it\u0027s beyond return None.\n\nBut this can lead to tiny shard ranges at the end of a sharding\ncontainer. It would be better to read ahead an extra range and adjust\naccordingly.\nIt does this by:\n - Get the next upper (first_upper). If this is none return it as we\u0027re at\n   the end.\n - Get the second upper. If this exists then we can just return the\n   first.\n - If there isn\u0027t a second upper we need to get how many objs are left\n   and:\n  ~ if the objs left \u003c some value. Lump the rest of the objects in the\n    same shard, that is return None as we\u0027re at the end.\n  - If the objs left \u003e some value. Find a new next upper that\u0027ll make 2\n    better sized shards:\n\n  return next_upper(last_upper, (shard_size + objs_left) // 2)\n\nChange-Id: I48d855bdbb7dca46e2b1fd6054f9b5017c331685\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/0c6188fb179f14e74e83ebf398da0e1d4c9252ce"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/0c6188fb179f14e74e83ebf398da0e1d4c9252ce"}]},"branch":"refs/heads/master"},"b357ed01dd52395399b92c55493bde673acf01c9":{"kind":"TRIVIAL_REBASE","_number":2,"created":"2021-05-21 07:28:48.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/85/791885/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/85/791885/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/85/791885/2"}}},"commit":{"parents":[{"commit":"c9a1912910a921312e1f928731454c9468ded415","subject":"sharding: Allow rows_per_shard to be \u0027auto\u0027","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/c9a1912910a921312e1f928731454c9468ded415"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-18 04:31:29.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-21 07:28:26.000000000","tz":600},"subject":"sharding: make broker\u0027s _get_next_shard_range_upper smarter","message":"sharding: make broker\u0027s _get_next_shard_range_upper smarter\n\nCurrently when we ask a container broker to give us the next shard range\nupper if just reads the next `shard_size` objects from the last and\neither return it or if it\u0027s beyond return None.\n\nBut this can lead to tiny shard ranges at the end of a sharding\ncontainer. It would be better to read ahead an extra range and adjust\naccordingly.\nIt does this by:\n - Get the next upper (first_upper). If this is none return it as we\u0027re at\n   the end.\n - Get the second upper. If this exists then we can just return the\n   first.\n - If there isn\u0027t a second upper we need to get how many objs are left\n   and:\n  ~ if the objs left \u003c some value. Lump the rest of the objects in the\n    same shard, that is return None as we\u0027re at the end.\n  - If the objs left \u003e some value. Find a new next upper that\u0027ll make 2\n    better sized shards:\n\n  return next_upper(last_upper, (shard_size + objs_left) // 2)\n\nChange-Id: I48d855bdbb7dca46e2b1fd6054f9b5017c331685\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/b357ed01dd52395399b92c55493bde673acf01c9"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/b357ed01dd52395399b92c55493bde673acf01c9"}]},"branch":"refs/heads/master"},"c25f7f662f44b26d5b87fc9484772e1b6cb4d605":{"kind":"REWORK","_number":3,"created":"2021-05-25 05:55:14.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/85/791885/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/85/791885/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/85/791885/3"}}},"commit":{"parents":[{"commit":"18f20daf389e3db42c7bfb56a520913e416caaf7","subject":"Add absolute values for shard shrinking config options","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/18f20daf389e3db42c7bfb56a520913e416caaf7"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-18 04:31:29.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-25 05:34:22.000000000","tz":600},"subject":"sharding: Add auto mode to rows_per_shard","message":"sharding: Add auto mode to rows_per_shard\n\nWhen setting rows_per_shard to \u0027auto\u0027, the new default, the sharder will\nadjust the last 2 shard_ranges to make sure the last shards isn\u0027t too\nsmall. It does this by making broker\u0027s _get_next_shard_range_upper smarter\n\nCurrently when we ask a container broker to give us the next shard range\nupper if just reads the next `shard_size` objects from the last and\neither return it or if it\u0027s beyond return None.\n\nBut this can lead to tiny shard ranges at the end of a sharding\ncontainer. It would be better to read ahead an extra range and adjust\naccordingly. To make this happen the broker\u0027s find_shard_ranges has\ngrown a correct_tail flag, that when set:\n - Get the next upper (first_upper). If this is none return it as we\u0027re at\n   the end.\n - Get the second upper. If this exists then we can just return the\n   first.\n - If there isn\u0027t a second upper we need to get how many objs are left\n   and:\n  ~ if the objs left \u003c some value. Lump the rest of the objects in the\n    same shard, that is return None as we\u0027re at the end.\n  - If the objs left \u003e some value. Find a new next upper that\u0027ll make 2\n    better sized shards:\n\n  return next_upper(last_upper, (shard_size + objs_left) // 2)\n\nChange-Id: I48d855bdbb7dca46e2b1fd6054f9b5017c331685\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/c25f7f662f44b26d5b87fc9484772e1b6cb4d605"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/c25f7f662f44b26d5b87fc9484772e1b6cb4d605"}]},"branch":"refs/heads/master"},"787fc75d9d7075bbaa2fc70ec3157a253d7f594d":{"kind":"REWORK","_number":4,"created":"2021-05-25 07:18:19.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/85/791885/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/85/791885/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/85/791885/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/85/791885/4"}}},"commit":{"parents":[{"commit":"18f20daf389e3db42c7bfb56a520913e416caaf7","subject":"Add absolute values for shard shrinking config options","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/18f20daf389e3db42c7bfb56a520913e416caaf7"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-18 04:31:29.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-05-25 07:17:42.000000000","tz":600},"subject":"sharding: Add auto mode to rows_per_shard","message":"sharding: Add auto mode to rows_per_shard\n\nWhen setting rows_per_shard to \u0027auto\u0027, the new default, the sharder will\nadjust the last 2 shard_ranges to make sure the last shards isn\u0027t too\nsmall. It does this by making broker\u0027s _get_next_shard_range_upper smarter\n\nCurrently when we ask a container broker to give us the next shard range\nupper if just reads the next `shard_size` objects from the last and\neither return it or if it\u0027s beyond return None.\n\nBut this can lead to tiny shard ranges at the end of a sharding\ncontainer. It would be better to read ahead an extra range and adjust\naccordingly. To make this happen the broker\u0027s find_shard_ranges has\ngrown a correct_tail flag, that when set:\n - Get the next upper (first_upper). If this is none return it as we\u0027re at\n   the end.\n - Get the second upper. If this exists then we can just return the\n   first.\n - If there isn\u0027t a second upper we need to get how many objs are left\n   and:\n  ~ if the objs left \u003c some value. Lump the rest of the objects in the\n    same shard, that is return None as we\u0027re at the end.\n  - If the objs left \u003e some value. Find a new next upper that\u0027ll make 2\n    better sized shards:\n\n  return next_upper(last_upper, (shard_size + objs_left) // 2)\n\nChange-Id: I48d855bdbb7dca46e2b1fd6054f9b5017c331685\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/787fc75d9d7075bbaa2fc70ec3157a253d7f594d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/787fc75d9d7075bbaa2fc70ec3157a253d7f594d"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY","applied_by":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"}},{"label":"Workflow","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}