)]}'
{"id":"openstack%2Fswift~571917","triplet_id":"openstack%2Fswift~master~I1f323e0086baea1387b2e6c70b808bc7e137f4da","project":"openstack/swift","branch":"master","topic":"objUpdaterEquality","hashtags":[],"change_id":"I1f323e0086baea1387b2e6c70b808bc7e137f4da","subject":"Manage async_pendings priority per containers","status":"ABANDONED","created":"2018-06-02 03:27:37.000000000","updated":"2022-02-25 21:30:54.000000000","total_comment_count":14,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"a8dabb8d2af43a5b7f0b9389e4e940605e103ef9","_number":571917,"virtual_id_number":571917,"owner":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"tag":"autogenerated:zuul:check","value":1,"date":"2020-05-04 13:26:20.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"date":"2020-05-04 12:14:35.000000000","_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":0,"permitted_voting_range":{"min":-2,"max":2},"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"permitted_voting_range":{"min":-1,"max":0},"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-04-10 21:31:28.000000000","updated_by":{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},"reviewer":{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},"state":"REVIEWER"},{"updated":"2020-04-30 21:37:03.000000000","updated_by":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"reviewer":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"state":"REVIEWER"},{"updated":"2020-05-01 04:11:48.000000000","updated_by":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"reviewer":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"state":"REVIEWER"},{"updated":"2020-05-04 13:26:20.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"536d830033bfaf83d1aeee7e788d8ca77fc6f37f","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2018-06-02 03:27:37.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"67707ed7bdd4763b60d1e2de4afbe98c78fc1543","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2018-06-02 04:59:26.000000000","message":"Patch Set 1: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- build-openstack-sphinx-docs http://logs.openstack.org/17/571917/1/check/build-openstack-sphinx-docs/2f67ae8/html/ : SUCCESS in 4m 10s\n- openstack-tox-pep8 http://logs.openstack.org/17/571917/1/check/openstack-tox-pep8/79f80b8/ : FAILURE in 3m 28s\n- legacy-swift-dsvm-functional http://logs.openstack.org/17/571917/1/check/legacy-swift-dsvm-functional/c1bb273/ : SUCCESS in 27m 29s\n- legacy-swift-dsvm-functional-identity-v3-only http://logs.openstack.org/17/571917/1/check/legacy-swift-dsvm-functional-identity-v3-only/c802f52/ : SUCCESS in 20m 15s (non-voting)\n- legacy-tempest-dsvm-neutron-full http://logs.openstack.org/17/571917/1/check/legacy-tempest-dsvm-neutron-full/8ed868c/ : SUCCESS in 1h 26m 46s\n- neutron-grenade http://logs.openstack.org/17/571917/1/check/neutron-grenade/7027d1f/ : SUCCESS in 48m 51s\n- swift-tox-py27 http://logs.openstack.org/17/571917/1/check/swift-tox-py27/5b52504/ : FAILURE in 10m 06s\n- swift-tox-py35 http://logs.openstack.org/17/571917/1/check/swift-tox-py35/4d7ee3e/ : SUCCESS in 4m 21s\n- swift-tox-func http://logs.openstack.org/17/571917/1/check/swift-tox-func/fad5479/ : SUCCESS in 8m 45s\n- swift-tox-func-encryption http://logs.openstack.org/17/571917/1/check/swift-tox-func-encryption/9711db9/ : SUCCESS in 7m 51s\n- swift-tox-func-domain-remap-staticweb http://logs.openstack.org/17/571917/1/check/swift-tox-func-domain-remap-staticweb/5cf0cda/ : SUCCESS in 9m 46s\n- swift-tox-func-ec http://logs.openstack.org/17/571917/1/check/swift-tox-func-ec/db9821e/ : SUCCESS in 7m 48s\n- swift-tox-func-s3api http://logs.openstack.org/17/571917/1/check/swift-tox-func-s3api/c525557/ : SUCCESS in 10m 32s\n- swift-probetests-centos-7 http://logs.openstack.org/17/571917/1/check/swift-probetests-centos-7/6601110/ : SUCCESS in 26m 39s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"acd7b553a0fd1b353187e81353337d45b8c771e4","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-04-10 20:53:52.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"f09c9d7ea56bd6effdbfbef054955e0207cb77a4","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-04-10 21:28:37.000000000","message":"Patch Set 2:\n\n(3 comments)\n\nA few first thoughts -- definitely like the idea!","accounts_in_message":[],"_revision_number":2},{"id":"1fcb6a4e41dffe3bb5dd79254f58b2775f9cd5d1","author":{"_account_id":28499,"name":"Romain de Joux","email":"romain.de-joux@ovhcloud.com","username":"rdejoux"},"date":"2020-04-10 21:31:28.000000000","message":"Patch Set 2:\n\n(4 comments)\n\nSome comment about naming and folders cleaning","accounts_in_message":[],"_revision_number":2},{"id":"948099adbd236cceb87e507f980f81d1086dc756","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-04-10 22:10:22.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/5d1f2d73878a48eeba6e969466105002 : SUCCESS in 7m 21s\n- grenade-py3 https://zuul.opendev.org/t/openstack/build/64e952b1fefc4dec9198fc9805450be8 : SUCCESS in 1h 10m 08s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/06fb30f7b21a47b99e957870d4704d15 : SUCCESS in 48m 51s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/4a97ae1b117447fdabab46eaddc84691 : SUCCESS in 9m 35s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/3071aac722bd4b6d8baa41ce05d7123a : SUCCESS in 11m 17s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/fa1876d643b24536a8bf13a4970e32e0 : FAILURE in 28m 15s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/abcf97bd3ab940fca6e2554363545da0 : FAILURE in 12m 21s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/5c8f8475406d47c7b6bb78c0cfe39184 : FAILURE in 28m 49s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/95dc9c59b03849f9ad27180a219c6f02 : FAILURE in 25m 08s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/4f3a014ac5a24c01a708b6a86ed5b3bb : SUCCESS in 9m 51s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/320a5bed6ccf4c53a7452b43dc013cbb : SUCCESS in 30m 03s\n- swift-tox-func-domain-remap-staticweb-py27 https://zuul.opendev.org/t/openstack/build/26a7be02ce5b4171aa03642db86d05c1 : SUCCESS in 22m 59s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/cea55300e937410e8f0d212419008ef6 : SUCCESS in 10m 15s\n- swift-tox-func-s3api-py27 https://zuul.opendev.org/t/openstack/build/8854cf8684ee424082f85d4c09e84dfc : SUCCESS in 14m 41s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/77bc9525aa4342268470cad635488165 : SUCCESS in 26m 31s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/1f31e3f56a1846788b563632cd9285a6 : SUCCESS in 23m 37s\n- swift-tox-func-domain-remap-staticweb-py37 https://zuul.opendev.org/t/openstack/build/758224abc0b34343b3aa50cf0ab966ad : SUCCESS in 10m 48s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/f1b8459a0c0640ae9454220346d52b9d : SUCCESS in 10m 38s\n- swift-tox-func-s3api-py37 https://zuul.opendev.org/t/openstack/build/40325b24bf6e4b87841d73b1371ef777 : SUCCESS in 16m 29s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/0ae2535a772f4163a817d2a0f6e2807d : TIMED_OUT in 41m 33s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/7d7f2efdc1864092b665cd67e528c8c2 : SUCCESS in 52m 42s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/106a5133f0b34708b2e858b602940817 : SUCCESS in 28m 55s\n- swift-dsvm-functional-py3 https://zuul.opendev.org/t/openstack/build/73fb3a20af2f459fa153177459490a17 : SUCCESS in 27m 04s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/e24ba5f272a54b91a6217d2848ff9cc4 : SUCCESS in 29m 37s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/b65d1e60d3a54874980d8aa9a0d9f966 : FAILURE in 34m 10s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/da3d2f5885904f6580419b4e8e9820eb : SUCCESS in 6m 02s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/2d5d2a1522b9400cb03a82ddda30b247 : FAILURE in 9m 43s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/863f34bb1a34462ab7e2c5a19c470326 : SUCCESS in 1h 10m 56s","accounts_in_message":[],"_revision_number":2},{"id":"7ebfcf8bc9e328c5b4ef2e1a9b806123d616bc08","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-04-17 18:40:42.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"49dd1eb8d7266a6e58fd87d22b2fbf191bf62c32","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-04-23 18:20:37.000000000","message":"Patch Set 2:\n\nInstead of doing this in the filesystem layout [1] I think we should switch to producer/consumer and force all async pending updates from this specific node that are aimed at a specific container go through a single serialized queue.\n\nIf the worker handling the queue decides to errorlimit/timeout a specific bad DB it can just turn into a sink or signal to the producer it can skip them and try again next interval.","accounts_in_message":[],"_revision_number":2},{"id":"784e588aaee5ff0ff20a2deb151c691cfee46d5f","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-04-23 18:21:02.000000000","message":"Patch Set 2:\n\nforgot the reference:\n\n1. https://gist.github.com/clayg/249c5d3ff580032a0d40751fc3f9f24b","accounts_in_message":[],"_revision_number":2},{"id":"33f5bfedb6dbc3747b527330ba0d72217d661339","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-04-28 23:33:04.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"24cb874477beec138ca54e84b3bfcf3f58ddea8a","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-04-28 23:40:37.000000000","message":"Patch Set 2:\n\n(6 comments)\n\n\u003e Instead of doing this in the filesystem layout [1] I think we\n \u003e should switch to producer/consumer and force all async pending\n \u003e updates from this specific node that are aimed at a specific\n \u003e container go through a single serialized queue.\n \u003e \n \u003e If the worker handling the queue decides to errorlimit/timeout a\n \u003e specific bad DB it can just turn into a sink or signal to the\n \u003e producer it can skip them and try again next interval.\n\nI chose a different approach. My goal was that no container in trouble can get multiple greenthread working on it. So I wrote an iterator that iterate all directory, picking one async-pending from a directory, then moving to the next until all are empty. It\u0027s not perfect (eg: if there is only one directory, all greenthreads will work on it), but it\u0027s an improvement imho.","accounts_in_message":[],"_revision_number":2},{"id":"7b17d89d770833312734d9f8ed75074f0ee265c9","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-04-29 00:47:34.000000000","message":"Patch Set 3: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/13ad35b5dab747bcab3500d742e3a772 : SUCCESS in 9m 31s\n- grenade https://zuul.opendev.org/t/openstack/build/3a360996447648feb7eaa308290a6bec : SUCCESS in 1h 00m 25s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/e7e203d6671c45e082a1325afffe54f7 : SUCCESS in 51m 34s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/13ea564733164f738961aba7fad55d8c : SUCCESS in 10m 52s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/4d917bf4094d4202b29de43d87b4651f : SUCCESS in 10m 15s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/90b54c47800f478fb2d150bd6ef37e36 : SUCCESS in 12m 36s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/133f85405a564749a9f0335d7b76661e : SUCCESS in 13m 33s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/851b229fd8394379ba74d1ebe1d5d1bb : SUCCESS in 13m 04s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/ecc00c843fd84185871caf6898422e3d : SUCCESS in 13m 11s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/8931acdaea53416bbbedeac5b2d934ba : SUCCESS in 13m 20s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/452e9814751d474f83c5577341037a2e : SUCCESS in 16m 21s\n- swift-tox-func-domain-remap-staticweb-py27 https://zuul.opendev.org/t/openstack/build/4baea06181a94b6bb69180b4bdb10318 : SUCCESS in 13m 16s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/4a693243fae74dc79cd50550a66c1e6a : SUCCESS in 11m 13s\n- swift-tox-func-s3api-py27 https://zuul.opendev.org/t/openstack/build/b8e0b65aecd7414bab8fca73f49ef2fa : SUCCESS in 15m 05s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/327fb2d1f2664541b0ab859069a431ec : SUCCESS in 13m 49s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/46964aab51a140a19c8a65f0fbf387bf : SUCCESS in 12m 10s\n- swift-tox-func-domain-remap-staticweb-py37 https://zuul.opendev.org/t/openstack/build/30e9926a4e9b40a6a7745d5e38f896d4 : SUCCESS in 14m 13s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/f34525ad65ea41cd81ec0df4c5e8d102 : SUCCESS in 13m 49s\n- swift-tox-func-s3api-py37 https://zuul.opendev.org/t/openstack/build/132d842c61844672a110818b4f004ca2 : SUCCESS in 21m 05s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/03aed8452e53438e88dc073b06a4f8e4 : SUCCESS in 27m 37s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/1b1ece8928344559bec4c6e06c53fdef : FAILURE in 47m 56s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/eb5872743d44495dbd915ba1460af82a : SUCCESS in 27m 50s\n- swift-dsvm-functional-py3 https://zuul.opendev.org/t/openstack/build/552a78bedb884f198c3e63cfa8e490ad : SUCCESS in 33m 17s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/55a31451a5ec498fa319e45e5254e4c1 : SUCCESS in 30m 33s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/ef33797f2d6241bea3cee3eb90284380 : SUCCESS in 23m 24s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/c9437c5a175e4dc5ac6ab5a13af33f37 : SUCCESS in 9m 01s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/c30c06d8c3374f1cade93ad00161cdba : FAILURE in 27m 12s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/2553147936574e97991ed873d61a5e41 : SUCCESS in 56m 37s\n- grenade-py3 https://zuul.opendev.org/t/openstack/build/804707c21f7b486eb6950b92a19339da : SUCCESS in 1h 07m 24s","accounts_in_message":[],"_revision_number":3},{"id":"11a7d064bd6c959496ee58575447ebed0336d104","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2020-04-30 21:37:03.000000000","message":"Patch Set 3:\n\nObviously we need a general solution, but I\u0027m still not sold exploding the filesystem is the best way for the updater workers to coordinate prioritization.\n\nI have this for now:\n\nhttps://gist.github.com/clayg/c3d31a62eba590eebd5f5d257c24a297\n\nOne reason I don\u0027t want to change the filesystem layout is I already have external tools that depend on it:\n\nhttps://gist.github.com/clayg/249c5d3ff580032a0d40751fc3f9f24b\n\nAlso the on-disk layout seems entirely orthogonal to the problem at hand.","accounts_in_message":[],"_revision_number":3},{"id":"0e424d611402f1a20281568ee51b95269678c7a3","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-04-30 23:16:04.000000000","message":"Patch Set 3:\n\n\u003e Obviously we need a general solution, but I\u0027m still not sold\n \u003e exploding the filesystem is the best way for the updater workers to\n \u003e coordinate prioritization.\n \u003e \n \u003e I have this for now:\n \u003e \n \u003e https://gist.github.com/clayg/c3d31a62eba590eebd5f5d257c24a297\n\n\nThe thing is, if you don\u0027t change the way async-pendings are organized, you will have to read all of them (possibly tens/hundreds of thousands) to find the very few ones that can be treated.\n\nEither you burn your disk I/O (and CPU) or, if you enabled ratelimiting, it will take days/weeks.\n\nI don\u0027t see any alternative to changing the organization. We could stop using filesystem for a real queuing mecanism, it would help with the I/O, but only marginally.","accounts_in_message":[],"_revision_number":3},{"id":"090b18bb2eb7c27b0b3f58bb71196fdaea0ffcc8","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-04-30 23:28:48.000000000","message":"Patch Set 3:\n\n\u003e We could stop using filesystem for a real queuing mecanism, it would help with the I/O, but only marginally.\n\nAnswer to myself: we don\u0027t need a \"real queuing\" system, it won\u0027t fit the way we want to avoid sending obsolete updates.\n\nMaybe a sqlite would do it? It would at least avoid many fsync at each update creation, and could probably be faster to read.\n\nIt would indeed look like the container server DB schema. We could even maybe reuse the same code (like to clean an object entry when a new one is created). I dunno, just throwing what\u0027s going in my head right now...","accounts_in_message":[],"_revision_number":3},{"id":"e36d21163407590818f516025f8155a49cd2f5d1","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2020-05-01 04:11:48.000000000","message":"Patch Set 3:\n\n\u003e Maybe a sqlite would do it? It would at least avoid many fsync at each update creation, and could probably be faster to read.\n\u003e\n\u003e It would indeed look like the container server DB schema. We could even maybe reuse the same code (like to clean an object entry when a new one is created).\n\nMakes me wonder whether it\u0027d be worth making a local handoff then replicating *that*, similar to what sharding does...\n\nOr alternatively, maybe let the updater go a few nodes into handoffs? The PUT will already pop the container into existence if there\u0027s no DB on disk...\n\nAsync pendings are on my mind lately... my cluster\u0027s not exactly in a good place right now :-(","accounts_in_message":[],"_revision_number":3},{"id":"a01e589cd0901421c418f7d3c02029ed760abb91","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-05-03 23:11:35.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"81d362e622c910414f3b58f92e2ac7296114138e","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-05-03 23:12:43.000000000","message":"Patch Set 5: Patch Set 4 was rebased","accounts_in_message":[],"_revision_number":5},{"id":"99e37b0601a7a8454e2baea75103a2fe82c64398","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-05-04 00:41:29.000000000","message":"Patch Set 5: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/533e98f8ffe149348c64bf783f69eccd : SUCCESS in 9m 31s\n- grenade https://zuul.opendev.org/t/openstack/build/0221d74e2ede4f009e832f8b1febbd8a : SUCCESS in 1h 27m 08s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/812b8dab7d874a1b82a4979a334d6793 : SUCCESS in 1h 11m 15s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/15798202f0e648c28359a7954a880004 : SUCCESS in 10m 35s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/2681b4c338974d84ba6a90834dbc87b1 : SUCCESS in 11m 25s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/4a1cca1da3494d1abe755968a4ac7fb1 : TIMED_OUT in 41m 09s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/80196baab7da4b25aaf3b04400bb140a : TIMED_OUT in 41m 16s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/7cf1eb7c8f9f4a5998d2b4063a012481 : TIMED_OUT in 41m 11s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/f65635dcefc74994b4a04b4c8def0a60 : TIMED_OUT in 41m 16s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/b329711b0cb64674858a4008b0f418bd : TIMED_OUT in 41m 14s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/9869bb735f89449b90908259e774ee69 : TIMED_OUT in 41m 10s\n- swift-tox-func-domain-remap-staticweb-py27 https://zuul.opendev.org/t/openstack/build/02aae239db3d4d8b90f551ab6c34472d : TIMED_OUT in 41m 06s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/c3f15f65b86e4e0587b29fd96ce1e121 : TIMED_OUT in 41m 06s\n- swift-tox-func-s3api-py27 https://zuul.opendev.org/t/openstack/build/7e24ba54d1154723ab690602d4c362e0 : TIMED_OUT in 41m 21s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/9355805164df401d9e651ef0ffff37d6 : TIMED_OUT in 41m 25s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/2c07c6ea74be4a9ebf930c1da4e5f5a3 : TIMED_OUT in 40m 59s\n- swift-tox-func-domain-remap-staticweb-py37 https://zuul.opendev.org/t/openstack/build/01e47145d68c427cbc802676202b9bd0 : SUCCESS in 9m 47s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/a77cfc027ff74a3cb7f4aa9746677fb4 : SUCCESS in 24m 50s\n- swift-tox-func-s3api-py37 https://zuul.opendev.org/t/openstack/build/06724fe4222840f6b4795ac0856b1570 : SUCCESS in 22m 36s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/133e4572340740a4b872a376a0a5bb52 : SUCCESS in 18m 45s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/c4506653534a4314ac77481428b0a07e : SUCCESS in 42m 00s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/004cc6a178524a80b6e52cec4e40d537 : SUCCESS in 39m 40s\n- swift-dsvm-functional-py3 https://zuul.opendev.org/t/openstack/build/89e11c77c32c4c07bf9b97340fa70d86 : SUCCESS in 31m 25s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/dfea1198df3b431db514e577d52f69be : SUCCESS in 29m 58s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/355be0aecafd4360b403f92fd9453e2f : SUCCESS in 12m 16s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/5c324d0738314e788223ae39698e22ac : SUCCESS in 5m 37s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/0c6ae0c25dba4b9e8274c401808bb2a8 : FAILURE in 19m 50s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/738316f1c8564a6aac918a9128eeec38 : SUCCESS in 1h 01m 54s\n- grenade-py3 https://zuul.opendev.org/t/openstack/build/9382ba9f70fd4839bc91dbff44f650c7 : SUCCESS in 52m 59s","accounts_in_message":[],"_revision_number":5},{"id":"f9372ad92948a7ddce58d629e2a4247692cc11e9","author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"date":"2020-05-04 12:14:35.000000000","message":"Patch Set 5:\n\nrecheck","accounts_in_message":[],"_revision_number":5},{"id":"fb19bfa652f21c8a9f622177a1f493eaddf02d42","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-05-04 13:26:20.000000000","message":"Patch Set 5: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/34878a55b0ba44289141696ebacfee13 : SUCCESS in 11m 59s\n- grenade https://zuul.opendev.org/t/openstack/build/dc781ed68c794c59b30c5a1331077cec : SUCCESS in 1h 06m 07s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/5d43e3fcf28045fd98e3a26df1ea5014 : SUCCESS in 44m 11s\n- swift-build-image https://zuul.opendev.org/t/openstack/build/81f51227fd1a459abc53c3ab96ce4341 : SUCCESS in 8m 51s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/36f8272b8ae64e44be11bc9a0f141a96 : SUCCESS in 9m 37s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/8d588430517a492ba8b281e364e13c38 : SUCCESS in 12m 11s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/da21e56db48e4011a4cef37de46fa533 : SUCCESS in 16m 28s\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/8bbe60175c234f4d8223fce0148ef60d : SUCCESS in 15m 23s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/eff166ea27ef4626b845ce3166ea8687 : SUCCESS in 14m 14s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/54115e7ddf044668becaaa43a0754d39 : SUCCESS in 13m 02s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/b466eea78fa94352a29df9038830822b : SUCCESS in 15m 22s\n- swift-tox-func-domain-remap-staticweb-py27 https://zuul.opendev.org/t/openstack/build/77baa49c91a94bf7853e6f3eeae2017e : SUCCESS in 10m 04s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/df70e45576a745218ebc926f2f5fc62b : SUCCESS in 12m 37s\n- swift-tox-func-s3api-py27 https://zuul.opendev.org/t/openstack/build/7ec30e1234fd4d2c983b8b20192c372d : SUCCESS in 18m 19s\n- swift-tox-func-py37 https://zuul.opendev.org/t/openstack/build/17dd5842cf0b466aafa465f0c7c28556 : SUCCESS in 10m 02s\n- swift-tox-func-encryption-py37 https://zuul.opendev.org/t/openstack/build/7169b80321fb4b09af9a4dc4d5f70f56 : SUCCESS in 16m 34s\n- swift-tox-func-domain-remap-staticweb-py37 https://zuul.opendev.org/t/openstack/build/31255fbf499246adbfa82dc2f7e69a17 : SUCCESS in 12m 41s\n- swift-tox-func-ec-py37 https://zuul.opendev.org/t/openstack/build/102b5a110fad4057bc23b3156b89e89b : SUCCESS in 11m 47s\n- swift-tox-func-s3api-py37 https://zuul.opendev.org/t/openstack/build/0977ae87a56f4f46a2e7d5ce9f1e6ce8 : SUCCESS in 15m 42s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/b322f08bf2cc469c81cb52552fa1d534 : FAILURE in 4m 56s (non-voting)\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/5607db539c244575874f4b160d6eeb68 : SUCCESS in 46m 18s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/8f41fc0cc23743a59e6408c85d6351c0 : SUCCESS in 29m 39s\n- swift-dsvm-functional-py3 https://zuul.opendev.org/t/openstack/build/9f6aebe81e024edbb6ff84ed49eade7d : SUCCESS in 34m 39s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/fcab4d5533264cf39a34daa955972c7f : SUCCESS in 29m 49s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/493957943b2245efab41d5358c2d0f57 : SUCCESS in 17m 03s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/6aea0cd71d004c03806ee142af643328 : SUCCESS in 9m 13s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/68152a6f5a8840ffb185e5185e0839ff : FAILURE in 9m 10s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/fc5dc09441954e1693ea86d9dc85da8c : SUCCESS in 1h 02m 51s\n- grenade-py3 https://zuul.opendev.org/t/openstack/build/9c72d13a8553482b94ffce18d12832a2 : SUCCESS in 1h 04m 11s","accounts_in_message":[],"_revision_number":5},{"id":"a8dabb8d2af43a5b7f0b9389e4e940605e103ef9","tag":"autogenerated:gerrit:abandon","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2022-02-25 21:30:54.000000000","message":"Abandoned\n\nAddressed via https://review.opendev.org/c/openstack/swift/+/820962/ instead.","accounts_in_message":[],"_revision_number":5}],"current_revision_number":5,"current_revision":"cf27c4b45c740f8a1d4ea53155d77a2cb01268db","revisions":{"62f3f8e81b5ce58d79407dbdaf8b3a0d00b90cfc":{"kind":"REWORK","_number":1,"created":"2018-06-02 03:27:37.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/17/571917/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/17/571917/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/17/571917/1"}}},"commit":{"parents":[{"commit":"7f12d608beb7b6e24aa2094d1fed23f94a69761c","subject":"Merge \"Verify diff stat is unchanged when syncing only shard ranges\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/7f12d608beb7b6e24aa2094d1fed23f94a69761c"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2018-06-02 03:13:12.000000000","tz":120},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2018-06-02 03:27:09.000000000","tz":120},"subject":"WIP: Manage async_pendings priority per containers","message":"WIP: Manage async_pendings priority per containers\n\nIf, for any reasons (e.g.: big unsharded container), a container starts to\nrefuse all updates from object-servers, a lot of async_pendings will be\ncreated on devices. All these async_pendings will rarely succeed,\nslowing down the entire object-updater daemon. The issue is that\nasync_pendings not related to this problematic container will get treated\noccasionaly, because they are lost in the crowd.\n\nThe fix is to change the path so that async_pendings are created in a\ndirectory per container. The patch also allow to give a time-limit and\nerror-limit per container so that a problematic containers does not\nconsume all the time of the object-updater\n\nTODO: write a test to check the time-limit/error-limit\n\nChange-Id: I1f323e0086baea1387b2e6c70b808bc7e137f4da\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/62f3f8e81b5ce58d79407dbdaf8b3a0d00b90cfc"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/62f3f8e81b5ce58d79407dbdaf8b3a0d00b90cfc"}]},"branch":"refs/heads/master"},"4e37d157d732ba37b1dfe50969f9345475f4b8ab":{"kind":"REWORK","_number":2,"created":"2020-04-10 20:53:52.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/17/571917/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/17/571917/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/17/571917/2"}}},"commit":{"parents":[{"commit":"a495f1e327fac17246a296721fb1eda8470e4773","subject":"Merge \"pep8: Turn on E305\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/a495f1e327fac17246a296721fb1eda8470e4773"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-04-10 20:07:21.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-04-10 20:53:45.000000000","tz":-240},"subject":"Manage async_pendings priority per containers","message":"Manage async_pendings priority per containers\n\nIf, for any reasons (e.g.: big unsharded container), a container starts to\nrefuse all updates from object-servers, a lot of async_pendings will be\ncreated on devices. All these async_pendings will rarely succeed,\nslowing down the entire object-updater daemon. The issue is that\nasync_pendings not related to this problematic container will get treated\noccasionaly, because they are lost in the crowd.\n\nThe fix is to change the path so that async_pendings are created in a\ndirectory per container. The patch also allow to give a time-limit and\nerror-limit per container so that a problematic containers does not\nconsume all the time of the object-updater\n\nChange-Id: I1f323e0086baea1387b2e6c70b808bc7e137f4da\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4e37d157d732ba37b1dfe50969f9345475f4b8ab"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4e37d157d732ba37b1dfe50969f9345475f4b8ab"}]},"branch":"refs/heads/master"},"1975f05aef2cc0492e4b467185c97fc79c31179f":{"kind":"REWORK","_number":3,"created":"2020-04-28 23:33:04.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/17/571917/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/17/571917/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/17/571917/3"}}},"commit":{"parents":[{"commit":"a495f1e327fac17246a296721fb1eda8470e4773","subject":"Merge \"pep8: Turn on E305\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/a495f1e327fac17246a296721fb1eda8470e4773"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-04-10 20:07:21.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-04-28 23:30:02.000000000","tz":-240},"subject":"Manage async_pendings priority per containers","message":"Manage async_pendings priority per containers\n\nIf, for any reasons (e.g.: big unsharded container), a container starts to\nrefuse all updates from object-servers, a lot of async_pendings will be\ncreated on devices. All these async_pendings will rarely succeed,\nslowing down the entire object-updater daemon. The issue is that\nasync_pendings not related to this problematic container will get treated\noccasionaly, because they are lost in the crowd.\n\nThe fix is to change the path so that async_pendings are created in a\ndirectory per container(-shard). The patch also allow to give a\ntime-limit and error-limit per container(-shard) so that a problematic\ncontainers does not consume all the time of the object-updater.\n\nThe directories are all processed in parallels, one async-pending\nper directory at a time, up to the concurrency limit. The goal is to avoid\nspawning all the concurrency on a single container that is already in\ntrouble and have a many async-pendings on disk.\n\nFailures and time limit are not enforced on legacy suffix directories.\n\nChange-Id: I1f323e0086baea1387b2e6c70b808bc7e137f4da\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/1975f05aef2cc0492e4b467185c97fc79c31179f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/1975f05aef2cc0492e4b467185c97fc79c31179f"}]},"branch":"refs/heads/master"},"4e5d3e9061786cd32f0082136bd5aa86d9083fb0":{"kind":"REWORK","_number":4,"created":"2020-05-03 23:11:35.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/17/571917/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/17/571917/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/17/571917/4"}}},"commit":{"parents":[{"commit":"a495f1e327fac17246a296721fb1eda8470e4773","subject":"Merge \"pep8: Turn on E305\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/a495f1e327fac17246a296721fb1eda8470e4773"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-04-10 20:07:21.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-05-03 23:07:56.000000000","tz":-240},"subject":"Manage async_pendings priority per containers","message":"Manage async_pendings priority per containers\n\nIf, for any reasons (e.g.: big unsharded container), a container starts to\nrefuse all updates from object-servers, a lot of async_pendings will be\ncreated on devices. All these async_pendings will rarely succeed,\nslowing down the entire object-updater daemon. The issue is that\nasync_pendings not related to this problematic container will get treated\noccasionaly, because they are lost in the crowd.\n\nThe fix is to change the path so that async_pendings are created in a\ndirectory per container(-shard). The patch also allow to give a\ntime-limit and error-limit per container(-shard) so that a problematic\ncontainers does not consume all the time of the object-updater.\n\nThe directories are all processed in parallels, one async-pending\nper directory at a time, up to the concurrency limit. The goal is to avoid\nspawning all the concurrency on a single container that is already in\ntrouble and have a many async-pendings on disk.\n\nFailures and time limit are not enforced on legacy suffix directories.\n\nChange-Id: I1f323e0086baea1387b2e6c70b808bc7e137f4da\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4e5d3e9061786cd32f0082136bd5aa86d9083fb0"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4e5d3e9061786cd32f0082136bd5aa86d9083fb0"}]},"branch":"refs/heads/master"},"cf27c4b45c740f8a1d4ea53155d77a2cb01268db":{"kind":"TRIVIAL_REBASE","_number":5,"created":"2020-05-03 23:12:43.000000000","uploader":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"ref":"refs/changes/17/571917/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/17/571917/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/17/571917/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/17/571917/5"}}},"commit":{"parents":[{"commit":"207e08c64bc933afbfd5f5c4ddca9a75a1c7d07f","subject":"Merge \"sharding: Add probe test that exercises swift-manage-shard-ranges\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/207e08c64bc933afbfd5f5c4ddca9a75a1c7d07f"}]}],"author":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-04-10 20:07:21.000000000","tz":-240},"committer":{"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","date":"2020-05-03 23:12:43.000000000","tz":0},"subject":"Manage async_pendings priority per containers","message":"Manage async_pendings priority per containers\n\nIf, for any reasons (e.g.: big unsharded container), a container starts to\nrefuse all updates from object-servers, a lot of async_pendings will be\ncreated on devices. All these async_pendings will rarely succeed,\nslowing down the entire object-updater daemon. The issue is that\nasync_pendings not related to this problematic container will get treated\noccasionaly, because they are lost in the crowd.\n\nThe fix is to change the path so that async_pendings are created in a\ndirectory per container(-shard). The patch also allow to give a\ntime-limit and error-limit per container(-shard) so that a problematic\ncontainers does not consume all the time of the object-updater.\n\nThe directories are all processed in parallels, one async-pending\nper directory at a time, up to the concurrency limit. The goal is to avoid\nspawning all the concurrency on a single container that is already in\ntrouble and have a many async-pendings on disk.\n\nFailures and time limit are not enforced on legacy suffix directories.\n\nChange-Id: I1f323e0086baea1387b2e6c70b808bc7e137f4da\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/cf27c4b45c740f8a1d4ea53155d77a2cb01268db"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/cf27c4b45c740f8a1d4ea53155d77a2cb01268db"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}