)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1eb7167b0a6f2336ac8527c927745659b95322a1","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Limit node_iter on EC HEAD"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"EC HEADs shouldn\u0027t check 2*replica count to find metadata. Every"},{"line_number":10,"context_line":"fragment in EC stores a copy of the object metadata, so we only ever"},{"line_number":11,"context_line":"need to visit any fragment to answer a HEAD response. Replica_count"},{"line_number":12,"context_line":"would be plenty."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"6ad471f1_3e393b88","line":9,"updated":"2025-02-03 05:40:37.000000000","message":"Without saying that for the case of 404, this sentence is kind of confusing. For the normal case, EC HEADs only need to talk to one object-server to return metadata.","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ca7cad0ba5ee24c466614009b7ec596cfc9185af","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Limit node_iter on EC HEAD"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"EC HEADs shouldn\u0027t check 2*replica count to find metadata. Every"},{"line_number":10,"context_line":"fragment in EC stores a copy of the object metadata, so we only ever"},{"line_number":11,"context_line":"need to visit any fragment to answer a HEAD response. Replica_count"},{"line_number":12,"context_line":"would be plenty."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"d89e57d6_02e60a1b","line":9,"in_reply_to":"6ad471f1_3e393b88","updated":"2025-02-03 08:40:42.000000000","message":"Good call!","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Limit node_iter on EC HEAD"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"EC HEADs shouldn\u0027t check 2*replica count to find metadata. Every"},{"line_number":10,"context_line":"fragment in EC stores a copy of the object metadata, so we only ever"},{"line_number":11,"context_line":"need to visit any fragment to answer a HEAD response. Replica_count"},{"line_number":12,"context_line":"would be plenty."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"78a44e3e_0a71821b","line":9,"in_reply_to":"d89e57d6_02e60a1b","updated":"2025-08-29 19:53:48.000000000","message":"Done","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f55b357b5f19cab137b25fba548d6c14f0b2bf19","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"0ddd8743_f6c0b012","updated":"2025-01-17 05:31:17.000000000","message":"Added a follow up with some initial tests. 1 to test we can override with policy overrides, and 1 to show we can change the number of HEAD requests sent: https://review.opendev.org/c/openstack/swift/+/939512","commit_id":"947d00a914cd671e28662669efe2751a71980bdd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b504cb8471cbcdd005d3e41eee7e68b1c0eaee15","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"7ceb55be_c9f8605d","updated":"2025-01-16 20:00:03.000000000","message":"if we want to go with this approach, I think the \"style\" of the tests established over in 939408: sq? moar tests for per-policy request_node_count | https://review.opendev.org/c/openstack/swift/+/939408 might help us explore the space of the new \"experimental\" config option some.","commit_id":"947d00a914cd671e28662669efe2751a71980bdd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c909220b2c6312728b78453487f936972239aeb3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"5d393047_be885973","updated":"2025-01-16 17:07:09.000000000","message":"needs tests","commit_id":"947d00a914cd671e28662669efe2751a71980bdd"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1eb7167b0a6f2336ac8527c927745659b95322a1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"1883305a_8d666478","updated":"2025-02-03 05:40:37.000000000","message":"After looking into the previous patch, I agree the new approach is much direct to limit the number of object nodes on 404 EC head, while the previous proposed ``request_node_count`` setting affects all object-related methods GET/HEAD/POST/PUT/DELETE. Also, ``one replica`` should be good enough for the 404 EC, even considering the availability of object stored.\n\n``-1`` here simply because ProxyOverrideOptions::__eq__() misses the new option name.","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a3d6f2808e833f7c9f908b482f15c85d477ff83b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"6bc05f6a_2f13075e","updated":"2025-01-30 17:48:08.000000000","message":"this seems fine to me, and the tests are pretty good [1]\n\nBut I\u0027m worried Matt and I are in a bubble - I\u0027d really like a fresh set of eyes on the idea - see if there\u0027s not something better/cleaner we might be missing.  I don\u0027t think I would advocate we should *merge* this idea with \"experimental_ec_head_limit\" as the config option if we can carry it and test it and we like it.\n\n1. when we had per-policy-request-node-count most of the tests in were just to demonstrate the weird stuff that happens when you start screwing with GETs node_iter: https://review.opendev.org/c/openstack/swift/+/939408/1/test/unit/proxy/controllers/test_obj.py - the HEAD tests were all super obvious and straight forward!","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ca7cad0ba5ee24c466614009b7ec596cfc9185af","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"8b853ab0_c715fda6","updated":"2025-02-03 08:40:42.000000000","message":"Thanks Jian, great stopping.\n\nAnd yeah I agree. I also don\u0027t expect this patch to land as is either, not with an option starting with experimantal... but we want an option we can use for tuning while we carry and test this out (at least that\u0027s my understanding).","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"e216b7a01974ed407b2fb726d6b15bcafd7f35cd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"d8f7a9a7_771b0d04","in_reply_to":"8b853ab0_c715fda6","updated":"2025-02-03 08:41:44.000000000","message":"* great spotting, apparently I can\u0027t type :P","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ada4e7bc7ea18e0e71c6e0b21766886d5c0eca58","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"532ad59e_2739e45b","updated":"2025-08-12 17:17:57.000000000","message":"we seem to still be using this:\n\n```\n[root@stgss0237 ~]# /opt/ss/bin/swift-config /etc/swift/proxy-server.conf.d | grep experimental_ec_head_limit\nexperimental_ec_head_limit \u003d 8\n```\n\n^ that\u0027s an 8+4 schema\n\nIn prod we use auto.","commit_id":"ed1abb0a077a421e1d7d3f7411d3a792e29241cf"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"be45b39f6f84d0ba43af754b7c71709ab7ded40c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"295b8690_6ae77f4f","updated":"2025-08-20 17:06:15.000000000","message":"recheck\n\nall the failed jobs say \"This build does not provide any results\"","commit_id":"ddbe7b4fbae76dee7ded79dc49fde05e0cd56e87"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"ef523c94_4f7f45f3","updated":"2025-08-29 19:53:48.000000000","message":"I think we\u0027ve addressed all the blockers.","commit_id":"a72c112402a8405f0575e71dc8390cb813e11329"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3b8e50c992403b95f6b3b5e368f4c19006c684e4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":15,"id":"648c8281_18534a08","updated":"2025-12-12 15:41:10.000000000","message":"I reverted some test changes following the merge conflict with patchset 10, to prefer what was on master.","commit_id":"d280f5cff87e4ea7857c9fc9d0be8cd0224989cb"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"44351efe1af200f88a17d3643a02c7cc6d65114c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":15,"id":"599eeaca_89438c91","updated":"2026-05-11 06:06:33.000000000","message":"Well I still think this patch is a win on valid EC HEAD 404s. Maybe we could throw a shuffle in and then merge it?","commit_id":"d280f5cff87e4ea7857c9fc9d0be8cd0224989cb"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f55b357b5f19cab137b25fba548d6c14f0b2bf19","unresolved":true,"context_lines":[{"line_number":2832,"context_line":"            concurrency \u003d policy.ec_ndata \\"},{"line_number":2833,"context_line":"                if policy_options.concurrent_gets else 1"},{"line_number":2834,"context_line":"            # every frag-archive has all metadata so we only need one!"},{"line_number":2835,"context_line":"            head_nodes \u003d policy.object_ring.get_part_nodes(partition)"},{"line_number":2836,"context_line":"            if policy_options.experimental_ec_head_limit:"},{"line_number":2837,"context_line":"                head_nodes \u003d itertools.islice("},{"line_number":2838,"context_line":"                    head_nodes, policy_options.experimental_ec_head_limit)"}],"source_content_type":"text/x-python","patch_set":1,"id":"a875be40_de09636b","line":2835,"updated":"2025-01-17 05:31:17.000000000","message":"So the default wont be `2 * replica` it\u0027ll instead be EC replica count (ndata + nparity).\n\nWell in reality, it\u0027ll be `2 * replicas` for nodes to check, but we\u0027re only giving them replica count of nodes in the iter. So that\u0027s all NodeIter can check, thereby limiting.","commit_id":"947d00a914cd671e28662669efe2751a71980bdd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":2832,"context_line":"            concurrency \u003d policy.ec_ndata \\"},{"line_number":2833,"context_line":"                if policy_options.concurrent_gets else 1"},{"line_number":2834,"context_line":"            # every frag-archive has all metadata so we only need one!"},{"line_number":2835,"context_line":"            head_nodes \u003d policy.object_ring.get_part_nodes(partition)"},{"line_number":2836,"context_line":"            if policy_options.experimental_ec_head_limit:"},{"line_number":2837,"context_line":"                head_nodes \u003d itertools.islice("},{"line_number":2838,"context_line":"                    head_nodes, policy_options.experimental_ec_head_limit)"}],"source_content_type":"text/x-python","patch_set":1,"id":"0f73a390_e148e935","line":2835,"in_reply_to":"a875be40_de09636b","updated":"2025-08-29 19:53:48.000000000","message":"yeah it\u0027s just `1 * replica` default - that\u0027s all we got.","commit_id":"947d00a914cd671e28662669efe2751a71980bdd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"1b7589f2d80e1b4ef56e4bf035625ec67ecf4dc9","unresolved":true,"context_lines":[{"line_number":2835,"context_line":"            head_nodes \u003d policy.object_ring.get_part_nodes(partition)"},{"line_number":2836,"context_line":"            if policy_options.experimental_ec_head_limit:"},{"line_number":2837,"context_line":"                head_nodes \u003d itertools.islice("},{"line_number":2838,"context_line":"                    head_nodes, policy_options.experimental_ec_head_limit)"},{"line_number":2839,"context_line":"            node_iter \u003d NodeIter("},{"line_number":2840,"context_line":"                \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"},{"line_number":2841,"context_line":"                req, policy\u003dpolicy, node_iter\u003dhead_nodes)"}],"source_content_type":"text/x-python","patch_set":1,"id":"f9572538_daf8da15","line":2838,"updated":"2025-01-17 00:20:33.000000000","message":"I wonder if we should throw a shuffle in here so we\u0027re not alway hitting the same replicas in the same order for heads for partitions.","commit_id":"947d00a914cd671e28662669efe2751a71980bdd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b76ba94eac9dd659b4a753a595d32c10edf62c2a","unresolved":false,"context_lines":[{"line_number":2835,"context_line":"            head_nodes \u003d policy.object_ring.get_part_nodes(partition)"},{"line_number":2836,"context_line":"            if policy_options.experimental_ec_head_limit:"},{"line_number":2837,"context_line":"                head_nodes \u003d itertools.islice("},{"line_number":2838,"context_line":"                    head_nodes, policy_options.experimental_ec_head_limit)"},{"line_number":2839,"context_line":"            node_iter \u003d NodeIter("},{"line_number":2840,"context_line":"                \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"},{"line_number":2841,"context_line":"                req, policy\u003dpolicy, node_iter\u003dhead_nodes)"}],"source_content_type":"text/x-python","patch_set":1,"id":"c3d2da25_dacde34f","line":2838,"in_reply_to":"f9572538_daf8da15","updated":"2025-01-17 00:29:13.000000000","message":"opps, nodeiter shuffles the primary nodes, it\u0027s why it pulls them off. it calls the self.app.sort_nodes so all good. Just a brain fart.","commit_id":"947d00a914cd671e28662669efe2751a71980bdd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a3d6f2808e833f7c9f908b482f15c85d477ff83b","unresolved":true,"context_lines":[{"line_number":954,"context_line":"            self.account_name, self.container_name, self.object_name)"},{"line_number":955,"context_line":"        node_iter \u003d NodeIter("},{"line_number":956,"context_line":"            \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"},{"line_number":957,"context_line":"            req, policy\u003dpolicy)"},{"line_number":958,"context_line":"        concurrency \u003d self.app.get_object_ring(policy.idx).replica_count \\"},{"line_number":959,"context_line":"            if self.app.get_policy_options(policy).concurrent_gets else 1"},{"line_number":960,"context_line":"        resp \u003d self.GETorHEAD_base("}],"source_content_type":"text/x-python","patch_set":2,"id":"c1bf7da4_07fcb2e6","line":957,"updated":"2025-01-30 17:48:08.000000000","message":"hrmm.. so for EC GET this is the same","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a3d6f2808e833f7c9f908b482f15c85d477ff83b","unresolved":true,"context_lines":[{"line_number":2838,"context_line":"                    head_nodes, policy_options.experimental_ec_head_limit)"},{"line_number":2839,"context_line":"            node_iter \u003d NodeIter("},{"line_number":2840,"context_line":"                \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"},{"line_number":2841,"context_line":"                req, policy\u003dpolicy, node_iter\u003dhead_nodes)"},{"line_number":2842,"context_line":"            resp \u003d self.GETorHEAD_base("},{"line_number":2843,"context_line":"                req, \u0027Object\u0027, node_iter, partition,"},{"line_number":2844,"context_line":"                req.swift_entity_path, concurrency, policy)"}],"source_content_type":"text/x-python","patch_set":2,"id":"57951b45_6d58c8a5","line":2841,"updated":"2025-01-30 17:48:08.000000000","message":"but it makes sense that the ec/repl Controller\u0027s `_get_or_head_response` need to be responsible for NodeIter construction because stupid EC now wants to make a *different* NodeIter depending on if it\u0027s a GET or HEAD!","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a3d6f2808e833f7c9f908b482f15c85d477ff83b","unresolved":true,"context_lines":[{"line_number":2848,"context_line":"        # GET request"},{"line_number":2849,"context_line":"        node_iter \u003d NodeIter("},{"line_number":2850,"context_line":"            \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"},{"line_number":2851,"context_line":"            req, policy\u003dpolicy)"},{"line_number":2852,"context_line":"        orig_range \u003d None"},{"line_number":2853,"context_line":"        range_specs \u003d []"},{"line_number":2854,"context_line":"        if req.range:"}],"source_content_type":"text/x-python","patch_set":2,"id":"bfba721d_d32de62a","line":2851,"updated":"2025-01-30 17:48:08.000000000","message":"ok so this was pulled \"down\" from the shared `GETorHEAD`","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1eb7167b0a6f2336ac8527c927745659b95322a1","unresolved":true,"context_lines":[{"line_number":2813,"context_line":"                # got a stop"},{"line_number":2814,"context_line":"                break"},{"line_number":2815,"context_line":""},{"line_number":2816,"context_line":"    def _get_or_head_response(self, req, policy):"},{"line_number":2817,"context_line":"        \"\"\""},{"line_number":2818,"context_line":"        EC GETorHEAD concurrency/node_iter handling"},{"line_number":2819,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"20cc74d5_e7206cf5","line":2816,"updated":"2025-02-03 05:40:37.000000000","message":"this is for EC only, will we consider to apply number of ``one replica`` nodes to talk to for HEAD requests of 3-replica policy as well? maybe 3 nodes under 3-replica policy are not enough to support the case of 404?\n\nif we consider that, we should use different ``node_iter`` for 3-R HEAD as well.","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"135318e91c82a9128bbdcb840ab7d4cab1895383","unresolved":false,"context_lines":[{"line_number":2813,"context_line":"                # got a stop"},{"line_number":2814,"context_line":"                break"},{"line_number":2815,"context_line":""},{"line_number":2816,"context_line":"    def _get_or_head_response(self, req, policy):"},{"line_number":2817,"context_line":"        \"\"\""},{"line_number":2818,"context_line":"        EC GETorHEAD concurrency/node_iter handling"},{"line_number":2819,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"0795faeb_af093853","line":2816,"in_reply_to":"1cb79f89_d57eb60f","updated":"2025-02-03 19:03:56.000000000","message":"Acknowledged","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ca7cad0ba5ee24c466614009b7ec596cfc9185af","unresolved":true,"context_lines":[{"line_number":2813,"context_line":"                # got a stop"},{"line_number":2814,"context_line":"                break"},{"line_number":2815,"context_line":""},{"line_number":2816,"context_line":"    def _get_or_head_response(self, req, policy):"},{"line_number":2817,"context_line":"        \"\"\""},{"line_number":2818,"context_line":"        EC GETorHEAD concurrency/node_iter handling"},{"line_number":2819,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"1cb79f89_d57eb60f","line":2816,"in_reply_to":"20cc74d5_e7206cf5","updated":"2025-02-03 08:40:42.000000000","message":"Nah I think the 2 * replica default makes sense in a replica world. We\u0027re only checking 6 nodes max for a 3 replica policy (which is usually less then 1 replica count for EC).","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1eb7167b0a6f2336ac8527c927745659b95322a1","unresolved":true,"context_lines":[{"line_number":2827,"context_line":"            # information about the object."},{"line_number":2828,"context_line":"            policy_options \u003d self.app.get_policy_options(policy)"},{"line_number":2829,"context_line":"            concurrency \u003d policy.ec_ndata \\"},{"line_number":2830,"context_line":"                if policy_options.concurrent_gets else 1"},{"line_number":2831,"context_line":"            # every frag-archive has all metadata so we only need one!"},{"line_number":2832,"context_line":"            head_nodes \u003d iter(policy.object_ring.get_part_nodes(partition))"},{"line_number":2833,"context_line":"            if policy_options.experimental_ec_head_limit:"}],"source_content_type":"text/x-python","patch_set":3,"id":"4dda3dc8_2d92858e","line":2830,"updated":"2025-02-03 05:40:37.000000000","message":"if we simply increase ``policy_options.concurrent_gets`` to ``2 * Replica``, will we see performance improvement in terms of request latency? was EC 404 slow because that ``concurrency \u003d\u003d 1`` so that the next request will only be send out after the previous one returns?","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"135318e91c82a9128bbdcb840ab7d4cab1895383","unresolved":true,"context_lines":[{"line_number":2827,"context_line":"            # information about the object."},{"line_number":2828,"context_line":"            policy_options \u003d self.app.get_policy_options(policy)"},{"line_number":2829,"context_line":"            concurrency \u003d policy.ec_ndata \\"},{"line_number":2830,"context_line":"                if policy_options.concurrent_gets else 1"},{"line_number":2831,"context_line":"            # every frag-archive has all metadata so we only need one!"},{"line_number":2832,"context_line":"            head_nodes \u003d iter(policy.object_ring.get_part_nodes(partition))"},{"line_number":2833,"context_line":"            if policy_options.experimental_ec_head_limit:"}],"source_content_type":"text/x-python","patch_set":3,"id":"4bb75abc_7cf57146","line":2830,"in_reply_to":"29675000_90d5716c","updated":"2025-02-03 19:03:56.000000000","message":"yes, the number of requests will be reduced by half. That\u0027ll be great if we can reduce the EC 404 HEAD latency as well.\n\nwhen ``policy_options.concurrent_gets`` is enabled, HEAD ``concurrency`` will be ``policy.ec_ndata`` which is 8 in case of 8+4 EC. so before this patch, EC 404 HEAD will need to talk to 24 nodes in 3 concurrent request batches (24 / 8); with the proposed ``experimental_ec_head_limit \u003d 1 * replica``, EC 404 HEAD would need to talk to 12 nodes in 2 concurrent request batches (12 / 8). If we increase the HEAD ``concurrency`` to 12, that\u0027ll be only 1 concurrent request round-trip.","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":2827,"context_line":"            # information about the object."},{"line_number":2828,"context_line":"            policy_options \u003d self.app.get_policy_options(policy)"},{"line_number":2829,"context_line":"            concurrency \u003d policy.ec_ndata \\"},{"line_number":2830,"context_line":"                if policy_options.concurrent_gets else 1"},{"line_number":2831,"context_line":"            # every frag-archive has all metadata so we only need one!"},{"line_number":2832,"context_line":"            head_nodes \u003d iter(policy.object_ring.get_part_nodes(partition))"},{"line_number":2833,"context_line":"            if policy_options.experimental_ec_head_limit:"}],"source_content_type":"text/x-python","patch_set":3,"id":"56f0a804_165ec97d","line":2830,"in_reply_to":"4bb75abc_7cf57146","updated":"2025-08-29 19:53:48.000000000","message":"\u003e was EC 404 slow because that concurrency \u003d\u003d 1 \n\nno, we weren\u0027t in that case:\n\n```\n[root@s8k-sjc11-b25-pxy-02 ~]# /opt/ss/bin/swift-config /etc/swift/proxy-server.conf.d -s proxy-server:policy:1 | grep concurrent\nconcurrent_gets \u003d True\nconcurrent_ec_extra_requests \u003d 1\n```\n\n\u003e However, because of the distributed nature of authors and reviewers it’s imperative that you try your best to answer your own questions as part of your review.\n\nhttps://docs.openstack.org/swift/latest/contributor/review_guidelines.html#leave-comments\n\n\u003e in 3 concurrent request batches (24 / 8)\n\nI think that\u0027s assuming all responses take equal time - concurrency is just the size of the greenpool.\n\n\u003e If we increase the HEAD concurrency to 12, that\u0027ll be only 1 concurrent request round-trip.\n\nI don\u0027t think that\u0027s what we should do.  We don\u0027t need to keep looking for frags if the first ndata primary respond 404.","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ca7cad0ba5ee24c466614009b7ec596cfc9185af","unresolved":true,"context_lines":[{"line_number":2827,"context_line":"            # information about the object."},{"line_number":2828,"context_line":"            policy_options \u003d self.app.get_policy_options(policy)"},{"line_number":2829,"context_line":"            concurrency \u003d policy.ec_ndata \\"},{"line_number":2830,"context_line":"                if policy_options.concurrent_gets else 1"},{"line_number":2831,"context_line":"            # every frag-archive has all metadata so we only need one!"},{"line_number":2832,"context_line":"            head_nodes \u003d iter(policy.object_ring.get_part_nodes(partition))"},{"line_number":2833,"context_line":"            if policy_options.experimental_ec_head_limit:"}],"source_content_type":"text/x-python","patch_set":3,"id":"29675000_90d5716c","line":2830,"in_reply_to":"4dda3dc8_2d92858e","updated":"2025-02-03 08:40:42.000000000","message":"I think in the 404 case it wasn\u0027t just latency, although yeah that was really bad, it was also expensive (from a number of connections / requestions POV).","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"135318e91c82a9128bbdcb840ab7d4cab1895383","unresolved":true,"context_lines":[{"line_number":2832,"context_line":"            head_nodes \u003d iter(policy.object_ring.get_part_nodes(partition))"},{"line_number":2833,"context_line":"            if policy_options.experimental_ec_head_limit:"},{"line_number":2834,"context_line":"                head_nodes \u003d itertools.islice("},{"line_number":2835,"context_line":"                    head_nodes, policy_options.experimental_ec_head_limit)"},{"line_number":2836,"context_line":"            node_iter \u003d NodeIter("},{"line_number":2837,"context_line":"                \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"},{"line_number":2838,"context_line":"                req, policy\u003dpolicy, node_iter\u003dhead_nodes)"}],"source_content_type":"text/x-python","patch_set":3,"id":"c0d3a5c3_5a8beed7","line":2835,"updated":"2025-02-03 19:03:56.000000000","message":"how about we also set ``concurrency`` to be one ``replica`` at here? so two round-trips to finish all 12 requests would become only one.\n\n```\n            if policy_options.experimental_ec_head_limit:\n                head_nodes \u003d itertools.islice(\n                    head_nodes, policy_options.experimental_ec_head_limit)\n                concurrency \u003d policy_options.experimental_ec_head_limit\n```","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":2832,"context_line":"            head_nodes \u003d iter(policy.object_ring.get_part_nodes(partition))"},{"line_number":2833,"context_line":"            if policy_options.experimental_ec_head_limit:"},{"line_number":2834,"context_line":"                head_nodes \u003d itertools.islice("},{"line_number":2835,"context_line":"                    head_nodes, policy_options.experimental_ec_head_limit)"},{"line_number":2836,"context_line":"            node_iter \u003d NodeIter("},{"line_number":2837,"context_line":"                \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"},{"line_number":2838,"context_line":"                req, policy\u003dpolicy, node_iter\u003dhead_nodes)"}],"source_content_type":"text/x-python","patch_set":3,"id":"a4aa6933_b0eb99c4","line":2835,"in_reply_to":"c0d3a5c3_5a8beed7","updated":"2025-08-29 19:53:48.000000000","message":"The reason to limit concurrency to ndata (same as on GET default) is because any one *success* means we don\u0027t have to send any more requests.\n\nBy sending \"only\" ndata *concurrent* requests - we eliminate un-needed IO in the happy path.\n\n```\n# By default on a EC GET request swift will connect to a minimum number of\n# storage nodes in a minimum number of threads - for erasure coded data, ndata\n# requests to primary nodes are started at the same time.  When greater than\n# zero this option provides additional robustness and may reduce first byte\n# latency by starting additional requests - up to as many as nparity.\n# concurrent_ec_extra_requests \u003d 0\n```\n\nHowever it\u0027s probably an existing limitation/criticism that HEAD concurrency doesn\u0027t respect the `concurrent_ec_extra_requests` option:\n\n```\n        ec_request_count \u003d policy.ec_ndata\n        head_request_count \u003d policy_options.ec_head_node_count_fn(len(part_nodes))\n        if policy_options.concurrent_gets:\n            head_request_count +\u003d policy_options.concurrent_ec_extra_requests\n```","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":true,"context_lines":[{"line_number":2839,"context_line":"            # node_count_fn from returning a value that will blow-up"},{"line_number":2840,"context_line":"            # itertools.islice"},{"line_number":2841,"context_line":"            node_count \u003d policy_options.ec_head_node_count_fn(len(part_nodes))"},{"line_number":2842,"context_line":"            head_nodes \u003d itertools.islice(node_chain, node_count)"},{"line_number":2843,"context_line":"            # N.B. GetOrHeadHandler will wrap GreenthreadSafeIterator for us"},{"line_number":2844,"context_line":"            node_iter \u003d NodeIter("},{"line_number":2845,"context_line":"                \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"}],"source_content_type":"text/x-python","patch_set":7,"id":"20ede7c0_fc7b36f9","line":2842,"updated":"2025-08-29 19:53:48.000000000","message":"hrmm... so doing this *before* applying app.sort_nodes means with non-default `ec_head_node_count \u003c 1 * replicas` you may be artificially truncating some arbitrary subset of parity nodes.\n\nIt\u0027s *probably* not a big deal - but if some proxy is consistently closer to parity frags in a multi-region setup it might be annoying that you have to stick with the default.\n\nIdeally I think NodeIter just wouldn\u0027t work the way it does and we could just \"inject\" a `request_node_count_fn`","commit_id":"ddbe7b4fbae76dee7ded79dc49fde05e0cd56e87"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"44351efe1af200f88a17d3643a02c7cc6d65114c","unresolved":true,"context_lines":[{"line_number":2839,"context_line":"            # node_count_fn from returning a value that will blow-up"},{"line_number":2840,"context_line":"            # itertools.islice"},{"line_number":2841,"context_line":"            node_count \u003d policy_options.ec_head_node_count_fn(len(part_nodes))"},{"line_number":2842,"context_line":"            head_nodes \u003d itertools.islice(node_chain, node_count)"},{"line_number":2843,"context_line":"            # N.B. GetOrHeadHandler will wrap GreenthreadSafeIterator for us"},{"line_number":2844,"context_line":"            node_iter \u003d NodeIter("},{"line_number":2845,"context_line":"                \u0027object\u0027, self.app, policy.object_ring, partition, self.logger,"}],"source_content_type":"text/x-python","patch_set":7,"id":"e5417535_29184d42","line":2842,"in_reply_to":"20ede7c0_fc7b36f9","updated":"2026-05-11 06:06:33.000000000","message":"I guess we could shuffle part_nodes before adding them to the chain on line 2977 can\u0027t hurt.","commit_id":"ddbe7b4fbae76dee7ded79dc49fde05e0cd56e87"}],"swift/proxy/server.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1eb7167b0a6f2336ac8527c927745659b95322a1","unresolved":true,"context_lines":[{"line_number":190,"context_line":"            \u0027rebalance_missing_suppression_count\u0027,"},{"line_number":191,"context_line":"            \u0027concurrent_gets\u0027,"},{"line_number":192,"context_line":"            \u0027concurrency_timeout\u0027,"},{"line_number":193,"context_line":"            \u0027concurrent_ec_extra_requests\u0027,"},{"line_number":194,"context_line":"        ))"},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"d72d40dd_e15bb66d","line":193,"updated":"2025-02-03 05:40:37.000000000","message":"needs to add ``experimental_ec_head_limit`` at here.","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":190,"context_line":"            \u0027rebalance_missing_suppression_count\u0027,"},{"line_number":191,"context_line":"            \u0027concurrent_gets\u0027,"},{"line_number":192,"context_line":"            \u0027concurrency_timeout\u0027,"},{"line_number":193,"context_line":"            \u0027concurrent_ec_extra_requests\u0027,"},{"line_number":194,"context_line":"        ))"},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"489f3758_f2221c3b","line":193,"in_reply_to":"a4ee224c_344342f2","updated":"2025-08-29 19:53:48.000000000","message":"well spotted!","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ca7cad0ba5ee24c466614009b7ec596cfc9185af","unresolved":true,"context_lines":[{"line_number":190,"context_line":"            \u0027rebalance_missing_suppression_count\u0027,"},{"line_number":191,"context_line":"            \u0027concurrent_gets\u0027,"},{"line_number":192,"context_line":"            \u0027concurrency_timeout\u0027,"},{"line_number":193,"context_line":"            \u0027concurrent_ec_extra_requests\u0027,"},{"line_number":194,"context_line":"        ))"},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"a4ee224c_344342f2","line":193,"in_reply_to":"d72d40dd_e15bb66d","updated":"2025-02-03 08:40:42.000000000","message":"oh yeah, nice missed that one.\n\nWhile writing the tests I wondered how much effort I shouhld be making seeing as I don\u0027t even know if the option will become an option at all, and the very minimum it wont start with \"experimental_*\"","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1eb7167b0a6f2336ac8527c927745659b95322a1","unresolved":true,"context_lines":[{"line_number":161,"context_line":"        self.concurrent_ec_extra_requests \u003d int(get("},{"line_number":162,"context_line":"            \u0027concurrent_ec_extra_requests\u0027, 0))"},{"line_number":163,"context_line":"        self.experimental_ec_head_limit \u003d config_auto_int_value("},{"line_number":164,"context_line":"            get(\u0027experimental_ec_head_limit\u0027, \u0027auto\u0027), None)"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"    def __repr__(self):"},{"line_number":167,"context_line":"        return \u0027%s({}, {%s}, app)\u0027 % ("}],"source_content_type":"text/x-python","patch_set":3,"id":"b4064428_5e7f5a73","line":164,"updated":"2025-02-03 05:40:37.000000000","message":"Will this new option ``experimental_ec_head_limit`` become foot gun as well, as comments mentioned in previous implementation? https://review.opendev.org/c/openstack/swift/+/937667/comments/abc1a4f1_1330e1a3\n\nI feel a fixed value will be better. After some calculation and testing, let\u0027s say some number (like one Replica) is a good balance between availability and performance, then let\u0027s use that number instead of an option.","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ca7cad0ba5ee24c466614009b7ec596cfc9185af","unresolved":true,"context_lines":[{"line_number":161,"context_line":"        self.concurrent_ec_extra_requests \u003d int(get("},{"line_number":162,"context_line":"            \u0027concurrent_ec_extra_requests\u0027, 0))"},{"line_number":163,"context_line":"        self.experimental_ec_head_limit \u003d config_auto_int_value("},{"line_number":164,"context_line":"            get(\u0027experimental_ec_head_limit\u0027, \u0027auto\u0027), None)"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"    def __repr__(self):"},{"line_number":167,"context_line":"        return \u0027%s({}, {%s}, app)\u0027 % ("}],"source_content_type":"text/x-python","patch_set":3,"id":"c3335045_b40b71e0","line":164,"in_reply_to":"b4064428_5e7f5a73","updated":"2025-02-03 08:40:42.000000000","message":"Yup totally agree, which is why clay used `experimental_ec_head_limit` I don\u0027t expect it to keep this name or even stay around as an option at all. Initially, for carrying purposes we wanted to have a value to use to find a good solution.\n\nI could imagine replica_count or maybe even ndata would be a good possible value. Or maybe the option with jsut be a binary option between the 2, who knows.","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":161,"context_line":"        self.concurrent_ec_extra_requests \u003d int(get("},{"line_number":162,"context_line":"            \u0027concurrent_ec_extra_requests\u0027, 0))"},{"line_number":163,"context_line":"        self.experimental_ec_head_limit \u003d config_auto_int_value("},{"line_number":164,"context_line":"            get(\u0027experimental_ec_head_limit\u0027, \u0027auto\u0027), None)"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"    def __repr__(self):"},{"line_number":167,"context_line":"        return \u0027%s({}, {%s}, app)\u0027 % ("}],"source_content_type":"text/x-python","patch_set":3,"id":"b097ded1_2b927d18","line":164,"in_reply_to":"c3335045_b40b71e0","updated":"2025-08-29 19:53:48.000000000","message":"I went with a request_node_count option b/c I thought that was the easiest to explain.  The value we use in prod is replica_count and that gets most of the benefit and works well.  We could try smaller values later and change the default if we felt this was still too conservative for most deployments.","commit_id":"fa56156814c4f8fa34800fd8a77a28bea12c5c0f"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a3d6f2808e833f7c9f908b482f15c85d477ff83b","unresolved":true,"context_lines":[{"line_number":3334,"context_line":"                resp \u003d req.get_response(self.app)"},{"line_number":3335,"context_line":"            self.assertEqual(resp.status_int, 404)"},{"line_number":3336,"context_line":""},{"line_number":3337,"context_line":"        for head_limit in range(2, self.policy.object_ring.replicas + 1):"},{"line_number":3338,"context_line":"            do_test(head_limit)"},{"line_number":3339,"context_line":""},{"line_number":3340,"context_line":"    def test_GET_simple(self):"}],"source_content_type":"text/x-python","patch_set":2,"id":"2d8698a1_7057e289","line":3337,"updated":"2025-01-30 17:48:08.000000000","message":"is this range limited for a reason?\n\nDoes anything interesting happen if you set it `\u003c2` or `\u003e2*replicas()`","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":3334,"context_line":"                resp \u003d req.get_response(self.app)"},{"line_number":3335,"context_line":"            self.assertEqual(resp.status_int, 404)"},{"line_number":3336,"context_line":""},{"line_number":3337,"context_line":"        for head_limit in range(2, self.policy.object_ring.replicas + 1):"},{"line_number":3338,"context_line":"            do_test(head_limit)"},{"line_number":3339,"context_line":""},{"line_number":3340,"context_line":"    def test_GET_simple(self):"}],"source_content_type":"text/x-python","patch_set":2,"id":"0d305b4b_edadcac9","line":3337,"in_reply_to":"2d8698a1_7057e289","updated":"2025-08-29 19:53:48.000000000","message":"yes, \u003c1 and you get 503 \u003erequest_node_count and it\u0027s truncated","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"}],"test/unit/proxy/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a3d6f2808e833f7c9f908b482f15c85d477ff83b","unresolved":true,"context_lines":[{"line_number":1874,"context_line":"                              \"write_affinity\": \"r2\","},{"line_number":1875,"context_line":"                              \"write_affinity_node_count_fn\": 3,"},{"line_number":1876,"context_line":"                              \"write_affinity_handoff_delete_count\": 2,"},{"line_number":1877,"context_line":"                              \"experimental_ec_head_limit\": 4},"},{"line_number":1878,"context_line":"                       POLICIES[0]: {\"read_affinity\": \"r1\u003d100\","},{"line_number":1879,"context_line":"                                     \"sorting_method\": \"affinity\","},{"line_number":1880,"context_line":"                                     \"write_affinity\": \"r1\","}],"source_content_type":"text/x-python","patch_set":2,"id":"64893022_13920e38","line":1877,"updated":"2025-01-30 17:48:08.000000000","message":"nice.","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"518a8c73967b6583d2fd8f85818484f597d085bd","unresolved":false,"context_lines":[{"line_number":1874,"context_line":"                              \"write_affinity\": \"r2\","},{"line_number":1875,"context_line":"                              \"write_affinity_node_count_fn\": 3,"},{"line_number":1876,"context_line":"                              \"write_affinity_handoff_delete_count\": 2,"},{"line_number":1877,"context_line":"                              \"experimental_ec_head_limit\": 4},"},{"line_number":1878,"context_line":"                       POLICIES[0]: {\"read_affinity\": \"r1\u003d100\","},{"line_number":1879,"context_line":"                                     \"sorting_method\": \"affinity\","},{"line_number":1880,"context_line":"                                     \"write_affinity\": \"r1\","}],"source_content_type":"text/x-python","patch_set":2,"id":"0d4ba8e2_c38ffc34","line":1877,"in_reply_to":"64893022_13920e38","updated":"2025-08-29 19:53:48.000000000","message":"Acknowledged","commit_id":"dab922a01583d1e756c7b98c85945a92281dbe66"}]}