)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":true,"context_lines":[{"line_number":10,"context_line":"container GET that explicitly requests \u0027shard\u0027 or \u0027object\u0027 record"},{"line_number":11,"context_line":"type. A request for \u0027shard\u0027 record type may specify \u0027namespace\u0027"},{"line_number":12,"context_line":"format, but this request is unrelated to container listings or object"},{"line_number":13,"context_line":"updates and passes directly to the backend."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This patch also removes unnecessary JSON serialisation and"},{"line_number":16,"context_line":"de-serialisation of namespaces within the proxy GET path when a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":19,"id":"e19a66fa_7adc72bb","line":13,"updated":"2024-01-16 06:35:44.000000000","message":"Makes sense, when we are asking for shards it\u0027s usually the sharder and yes we want the latest, so yeah goto the backend!","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"771d8dd6ac5c49dd35748b47acc396431c173b60","unresolved":false,"context_lines":[{"line_number":10,"context_line":"container GET that explicitly requests \u0027shard\u0027 or \u0027object\u0027 record"},{"line_number":11,"context_line":"type. A request for \u0027shard\u0027 record type may specify \u0027namespace\u0027"},{"line_number":12,"context_line":"format, but this request is unrelated to container listings or object"},{"line_number":13,"context_line":"updates and passes directly to the backend."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This patch also removes unnecessary JSON serialisation and"},{"line_number":16,"context_line":"de-serialisation of namespaces within the proxy GET path when a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":19,"id":"b3c52127_9200aa02","line":13,"in_reply_to":"e19a66fa_7adc72bb","updated":"2024-01-16 10:20:40.000000000","message":"Acknowledged","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":true,"context_lines":[{"line_number":15,"context_line":"This patch also removes unnecessary JSON serialisation and"},{"line_number":16,"context_line":"de-serialisation of namespaces within the proxy GET path when a"},{"line_number":17,"context_line":"sharded object listing is being built. The final response body will"},{"line_number":18,"context_line":"container a list of objects so there is no need to write intermediate"},{"line_number":19,"context_line":"response bodies with list of namespaces."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Requests that explicitly specify record type of \u0027shard\u0027 will of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":19,"id":"fe38403b_a7ac1007","line":18,"range":{"start_line":18,"start_character":0,"end_line":18,"end_character":9},"updated":"2024-01-16 06:35:44.000000000","message":"contain\n\nI do this all the time too 😊","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"771d8dd6ac5c49dd35748b47acc396431c173b60","unresolved":false,"context_lines":[{"line_number":15,"context_line":"This patch also removes unnecessary JSON serialisation and"},{"line_number":16,"context_line":"de-serialisation of namespaces within the proxy GET path when a"},{"line_number":17,"context_line":"sharded object listing is being built. The final response body will"},{"line_number":18,"context_line":"container a list of objects so there is no need to write intermediate"},{"line_number":19,"context_line":"response bodies with list of namespaces."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Requests that explicitly specify record type of \u0027shard\u0027 will of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":19,"id":"b164931e_bac742df","line":18,"range":{"start_line":18,"start_character":0,"end_line":18,"end_character":9},"in_reply_to":"fe38403b_a7ac1007","updated":"2024-01-16 10:20:40.000000000","message":"Done","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":true,"context_lines":[{"line_number":12,"context_line":"format, but this request is unrelated to container listings or object"},{"line_number":13,"context_line":"updates and passes directly to the backend."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This patch also removes unnecessary JSON serialisation and"},{"line_number":16,"context_line":"de-serialisation of namespaces within the proxy GET path when a"},{"line_number":17,"context_line":"sharded object listing is being built. The final response body will"},{"line_number":18,"context_line":"contain a list of objects so there is no need to write intermediate"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":24,"id":"6b50f2d0_678beb1f","line":15,"updated":"2024-01-25 00:32:03.000000000","message":"s/serialisation/serialization","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"34caab6883c358d1ed4e85fff848dd36a4eeaecd","unresolved":false,"context_lines":[{"line_number":12,"context_line":"format, but this request is unrelated to container listings or object"},{"line_number":13,"context_line":"updates and passes directly to the backend."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This patch also removes unnecessary JSON serialisation and"},{"line_number":16,"context_line":"de-serialisation of namespaces within the proxy GET path when a"},{"line_number":17,"context_line":"sharded object listing is being built. The final response body will"},{"line_number":18,"context_line":"contain a list of objects so there is no need to write intermediate"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":24,"id":"527f809e_3c38b5e3","line":15,"in_reply_to":"6b50f2d0_678beb1f","updated":"2024-01-25 17:26:59.000000000","message":"depends which version of English you speak :) https://dictionary.cambridge.org/dictionary/english/serialization","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":19,"id":"24dcacbb_b4869be3","updated":"2024-01-16 06:35:44.000000000","message":"I\u0027ve done a deeper parse on this and looking great Al! I really like the GET refactor.\n\nJust a comment typo, and now just want to run it for a while in my VSAIO and make sure we haven\u0027t missed anything 😊","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"53032e1f16204cdbd3d471d13dbc3022972704cb","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":19,"id":"4c76f31f_240dc22b","updated":"2024-01-11 13:35:56.000000000","message":"recheck \n\n```FAILED test/probe/test_container_merge_policy_index.py::TestReservedNamespaceMergePolicyIndex::test_reconciler_move_object_twice```\n\nhttps://bugs.launchpad.net/swift/+bug/2028175","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"8b0cc3b1cc559565441ae0c4171ff4308337c822","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"fd0d3ed5_c6296a17","updated":"2024-01-18 06:33:44.000000000","message":"Been running it on a vsaio, and even have a sharded container. So far so good. I\u0027ll keep some swift-bench running and sharding and double check in the morning. But things seems to be running great.","commit_id":"0c6ccf2abe8c4dc3d98a210c20c647c750118f81"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9aa8847cbfb25b19dcf6355434b53ef2910919c8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"298f821f_4a579b35","updated":"2024-01-18 05:40:04.000000000","message":"Not only this patch improve the container listing path to correctly use caching, it also does a lot of good refactoring. But I feel the refactoring is only half done, _GET_auto() could be in a better shape as well. I would suggest to break up _GET_auto() into different sub-functions to handle different cases.","commit_id":"0c6ccf2abe8c4dc3d98a210c20c647c750118f81"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"98414533bc5427e485f02858232c01fb55cf6731","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":21,"id":"2b2042c3_327cac37","updated":"2024-01-23 05:18:19.000000000","message":"Do have a bit of a NIT. But I don\u0027t think it\u0027s a blocker. I love this clean up. Nice one Al! And think anything else we can fix in follow ups. This already makes the code _SO_ much easier to maintain!","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"286ca9f9_89426f0b","updated":"2024-01-24 06:40:51.000000000","message":"sorry I am kind of slow, production code looks great to me, still got some questions on the tests.","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"f7b0fd51_f12e668f","updated":"2024-01-25 00:32:03.000000000","message":"Great work, Al! This patch not only fixes the caching usages when handling container GET, optimize out the unnecessary JSON serialization, also refactors the whole proxy container GET path, readability and maintainability are much improved.","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ec2fda12747bea153fe0755548f625ece7c4ca81","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":25,"id":"b40f5ccc_0fd01e4f","updated":"2024-01-29 16:04:03.000000000","message":"There\u0027s not any new test methods in here, but some are moved in the file so that they can be grouped under a new TestGetExplicitRecordType[Legacy] class.\n\nThere\u0027s a more renaming of shard range -\u003e namespace.\n\nI added more helpers to create a Namespace, or for legacy, a ShardRange, when we use filler ranges or own shard range.","commit_id":"b5869c93d360d5ff6109ab6a2048445758df7a65"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6605914faf566271ab59a2ed7d1f01f8deee5a40","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":25,"id":"9e0c38d1_cb843caf","updated":"2024-01-29 17:07:22.000000000","message":"my last comment was meant for the follow on patch https://review.opendev.org/c/openstack/swift/+/907084/1?usp\u003drelated-change","commit_id":"b5869c93d360d5ff6109ab6a2048445758df7a65"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"34caab6883c358d1ed4e85fff848dd36a4eeaecd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":25,"id":"ef7d8a2f_6779f57c","updated":"2024-01-25 17:26:59.000000000","message":"recheck \n\nFAILED test/probe/test_container_merge_policy_index.py::TestContainerMergePolicyIndex::test_reconciler_move_object_twice\n\nhttps://bugs.launchpad.net/swift/+bug/2028175","commit_id":"b5869c93d360d5ff6109ab6a2048445758df7a65"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"08eec64b8f6f7134465ae4d47f913a54d0f066a9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"1948b121_db68d994","updated":"2024-02-01 09:12:48.000000000","message":"Love this one. I\u0027ve been testing it with swift-bench to put load on it, with sharding and all other daemons. Everything seems to be working. I haven\u0027t noticed any errors or stacktraces.\n\nLet\u0027s land this!","commit_id":"252f0d36b7c97d1e3261fd4cf475b362ee985abe"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9675d9252059db09a8fea09bf7fe5319ea220531","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"f1197fc9_b5b3eca9","updated":"2024-01-31 16:46:45.000000000","message":"The squashed test patch has been reviewed.","commit_id":"252f0d36b7c97d1e3261fd4cf475b362ee985abe"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"172a4d591fb94e0d1eb8bcbcfbf9f4b0fb8a1157","unresolved":true,"context_lines":[{"line_number":259,"context_line":"                # don\u0027t know if the container was sharded, or the case that the"},{"line_number":260,"context_line":"                # sharding state in metadata indicates the container was"},{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"99e576fb_618392b3","line":262,"updated":"2023-11-20 14:48:10.000000000","message":"Clay\u0027s comment from https://review.opendev.org/c/openstack/swift/+/900350/comment/99d9c5f0_9d0a2b47/\n\n\"\nsome additional differentiation might be useful in this case\n\n... but it looks like this part of diff is only (mostly?) carrying over existing behavior from _GET_using_cache but inlined/indented.\n\"","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":false,"context_lines":[{"line_number":259,"context_line":"                # don\u0027t know if the container was sharded, or the case that the"},{"line_number":260,"context_line":"                # sharding state in metadata indicates the container was"},{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"23e9586a_692cbca3","line":262,"in_reply_to":"51099d6e_bd6582f4","updated":"2024-01-24 16:43:48.000000000","message":"Acknowledged","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":true,"context_lines":[{"line_number":259,"context_line":"                # don\u0027t know if the container was sharded, or the case that the"},{"line_number":260,"context_line":"                # sharding state in metadata indicates the container was"},{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"51099d6e_bd6582f4","line":262,"in_reply_to":"672a2ce3_4ea7e2c3","updated":"2024-01-16 06:35:44.000000000","message":"yup, more granulatity can come later. This patch is already adding cleaning up a bunch, and we\u0027re not loosing any cache_state\u0027s so a +1 from me.","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5b7df78897d7242cca97041c256b345d6b62f7b5","unresolved":true,"context_lines":[{"line_number":259,"context_line":"                # don\u0027t know if the container was sharded, or the case that the"},{"line_number":260,"context_line":"                # sharding state in metadata indicates the container was"},{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"672a2ce3_4ea7e2c3","line":262,"in_reply_to":"99e576fb_618392b3","updated":"2023-11-20 16:41:50.000000000","message":"this is existing behavior - can we add granularity in a follow up?","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"172a4d591fb94e0d1eb8bcbcfbf9f4b0fb8a1157","unresolved":true,"context_lines":[{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        if namespaces:"},{"line_number":267,"context_line":"            # Namespaces found in cache so there is no need to go to backend,"}],"source_content_type":"text/x-python","patch_set":2,"id":"32589c1a_270818fd","line":264,"updated":"2023-11-20 14:48:10.000000000","message":"remove TODO: see comments https://review.opendev.org/c/openstack/swift/+/900350/comment/ede55871_8196496b/","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5b7df78897d7242cca97041c256b345d6b62f7b5","unresolved":false,"context_lines":[{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        if namespaces:"},{"line_number":267,"context_line":"            # Namespaces found in cache so there is no need to go to backend,"}],"source_content_type":"text/x-python","patch_set":2,"id":"9e935b80_a0258167","line":264,"in_reply_to":"32589c1a_270818fd","updated":"2023-11-20 16:41:50.000000000","message":"Done","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"f838e91c28b97b02bca161a433654bc683e3df34","unresolved":true,"context_lines":[{"line_number":271,"context_line":"            # GETorHEAD_base does not, so don\u0027t set it here either"},{"line_number":272,"context_line":"            headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":273,"context_line":"                            \u0027x-backend-record-shard-format\u0027: \u0027namespace\u0027,"},{"line_number":274,"context_line":"                            \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":275,"context_line":"            resp \u003d Response(request\u003dreq)"},{"line_number":276,"context_line":"            update_headers(resp, headers)"},{"line_number":277,"context_line":"            resp.last_modified \u003d Timestamp("}],"source_content_type":"text/x-python","patch_set":2,"id":"23393340_40716e6a","line":274,"updated":"2023-11-19 01:01:40.000000000","message":"I wonder is this header necessary? it seems it\u0027s used as a internal flag within this class to ask ``_get_or_head_post_check`` not to cache the namespace/container_info.","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a0fa4dd98b141c700dfe30c6083eee515a583fb4","unresolved":true,"context_lines":[{"line_number":271,"context_line":"            # GETorHEAD_base does not, so don\u0027t set it here either"},{"line_number":272,"context_line":"            headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":273,"context_line":"                            \u0027x-backend-record-shard-format\u0027: \u0027namespace\u0027,"},{"line_number":274,"context_line":"                            \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":275,"context_line":"            resp \u003d Response(request\u003dreq)"},{"line_number":276,"context_line":"            update_headers(resp, headers)"},{"line_number":277,"context_line":"            resp.last_modified \u003d Timestamp("}],"source_content_type":"text/x-python","patch_set":2,"id":"5df3e85f_46893bf6","line":274,"in_reply_to":"23393340_40716e6a","updated":"2024-01-18 11:22:09.000000000","message":"Yes the header is used to communicate that the container info was loaded from cache and therefore shouldn\u0027t be written to cache. I think it is necessary.","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"72560586573075d85ce92b342da54a6887cc3e4c","unresolved":false,"context_lines":[{"line_number":271,"context_line":"            # GETorHEAD_base does not, so don\u0027t set it here either"},{"line_number":272,"context_line":"            headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":273,"context_line":"                            \u0027x-backend-record-shard-format\u0027: \u0027namespace\u0027,"},{"line_number":274,"context_line":"                            \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":275,"context_line":"            resp \u003d Response(request\u003dreq)"},{"line_number":276,"context_line":"            update_headers(resp, headers)"},{"line_number":277,"context_line":"            resp.last_modified \u003d Timestamp("}],"source_content_type":"text/x-python","patch_set":2,"id":"e18f9deb_01aca483","line":274,"in_reply_to":"5df3e85f_46893bf6","updated":"2024-01-22 06:48:45.000000000","message":"Acknowledged","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"172a4d591fb94e0d1eb8bcbcfbf9f4b0fb8a1157","unresolved":true,"context_lines":[{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("},{"line_number":318,"context_line":"                \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":319,"context_line":"            complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":320,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":321,"context_line":"            if resp_record_type \u003d\u003d \u0027shard\u0027:"},{"line_number":322,"context_line":"                data \u003d self._parse_listing_response(req, resp)"},{"line_number":323,"context_line":"                namespaces \u003d self._parse_namespaces(req, data, resp)"}],"source_content_type":"text/x-python","patch_set":2,"id":"281aacab_cd43b01a","line":320,"updated":"2023-11-20 14:48:10.000000000","message":"Clay\u0027s comment from https://review.opendev.org/c/openstack/swift/+/900350/comment/f271d1e0_31d58b17/\n\n\"\nthis seems like a weird way to decide this? test\u0027s stub responses may not be consistent with the backend implementation.\n\nI\u0027m looking at test_GET_sharded_container_sharding_shard - we enter under _get_from_shards twice, but never _set_listing_namespace_in_cache\n\nWould it be better/suffciient to parse namespaces and make sure ns[0].lower \u003d\u003d ns[-1].upper \u003d\u003d \u0027\u0027?\n\"","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5b7df78897d7242cca97041c256b345d6b62f7b5","unresolved":true,"context_lines":[{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("},{"line_number":318,"context_line":"                \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":319,"context_line":"            complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":320,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":321,"context_line":"            if resp_record_type \u003d\u003d \u0027shard\u0027:"},{"line_number":322,"context_line":"                data \u003d self._parse_listing_response(req, resp)"},{"line_number":323,"context_line":"                namespaces \u003d self._parse_namespaces(req, data, resp)"}],"source_content_type":"text/x-python","patch_set":2,"id":"cd79574d_9b30ddc0","line":320,"in_reply_to":"281aacab_cd43b01a","updated":"2023-11-20 16:41:50.000000000","message":"We could infer \u0027complete_listing\u0027 for requests to the root, but for recursive requests to shards we\u0027d need to know the shard\u0027s namespace. We do know that in _get_from_shards so we could bury that in the request environ and refer to it here....but on the other hand, the backend tells us explicitly that we have the complete listing.\n\ntest_GET_sharded_container_sharding_shard does not have memcache enabled so \u0027X-Backend-Override-Shard-Name-Filter\u0027 isn\u0027t sent or received from backend.\n\nI\u0027ve added test_GET_sharded_container_sharding_shard_with_memcache, but still only the root namespaces will be cached because the shard is in *sharding* state.\n\nAlso added test_GET_sharded_container_sharded_shard_with_memcache, so the root and shard namespaces will be cached.","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":false,"context_lines":[{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("},{"line_number":318,"context_line":"                \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":319,"context_line":"            complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":320,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":321,"context_line":"            if resp_record_type \u003d\u003d \u0027shard\u0027:"},{"line_number":322,"context_line":"                data \u003d self._parse_listing_response(req, resp)"},{"line_number":323,"context_line":"                namespaces \u003d self._parse_namespaces(req, data, resp)"}],"source_content_type":"text/x-python","patch_set":2,"id":"de46ada5_f9279d09","line":320,"in_reply_to":"cd79574d_9b30ddc0","updated":"2024-01-24 16:43:48.000000000","message":"Done","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"172a4d591fb94e0d1eb8bcbcfbf9f4b0fb8a1157","unresolved":true,"context_lines":[{"line_number":333,"context_line":"                        req, namespaces)"},{"line_number":334,"context_line":"                    namespaces \u003d self._filter_complete_listing(req, namespaces)"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"        if cache_state:"},{"line_number":337,"context_line":"            resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":338,"context_line":"            self._record_shard_listing_cache_metrics("},{"line_number":339,"context_line":"                cache_state, resp, resp_record_type, info)"}],"source_content_type":"text/x-python","patch_set":2,"id":"cdbae48d_1937b97f","line":336,"updated":"2023-11-20 14:48:10.000000000","message":"Clay\u0027s comment from https://review.opendev.org/c/openstack/swift/+/900350/comment/16360e85_20cb659a/\n\n\"\nI don\u0027t see when cache_state is falsy\n\n```\ndiff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py\nindex bea9a2312..84a21a1fb 100644\n--- a/swift/proxy/controllers/container.py\n+++ b/swift/proxy/controllers/container.py\n@@ -337,6 +337,8 @@ class ContainerController(Controller):\n             resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)\n             self._record_shard_listing_cache_metrics(\n                 cache_state, resp, resp_record_type, info)\n+        else:\n+            asdf\n\n         if namespaces is not None:\n             # we got namespaces, so the container must be sharded; now build\n```\n... seems to pass at least test.unit.proxy.controller.test_container\n\"","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5b7df78897d7242cca97041c256b345d6b62f7b5","unresolved":false,"context_lines":[{"line_number":333,"context_line":"                        req, namespaces)"},{"line_number":334,"context_line":"                    namespaces \u003d self._filter_complete_listing(req, namespaces)"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"        if cache_state:"},{"line_number":337,"context_line":"            resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":338,"context_line":"            self._record_shard_listing_cache_metrics("},{"line_number":339,"context_line":"                cache_state, resp, resp_record_type, info)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1aa67881_084cdca7","line":336,"in_reply_to":"cdbae48d_1937b97f","updated":"2023-11-20 16:41:50.000000000","message":"you\u0027re right! on master there was a condition when cache state might be None, ... when we went down the GET_using_cache path but not for listing shard ranges - we don\u0027t do that anymore 😊\n\ngood catch!","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5b7df78897d7242cca97041c256b345d6b62f7b5","unresolved":false,"context_lines":[{"line_number":336,"context_line":"        if cache_state:"},{"line_number":337,"context_line":"            resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":338,"context_line":"            self._record_shard_listing_cache_metrics("},{"line_number":339,"context_line":"                cache_state, resp, resp_record_type, info)"},{"line_number":340,"context_line":""},{"line_number":341,"context_line":"        if namespaces is not None:"},{"line_number":342,"context_line":"            # we got namespaces, so the container must be sharded; now build"}],"source_content_type":"text/x-python","patch_set":2,"id":"84cec1c8_af85ccdc","line":339,"updated":"2023-11-20 16:41:50.000000000","message":"resp_record_type doesn\u0027t need to be passed as an arg - resp has it in its headers","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"f838e91c28b97b02bca161a433654bc683e3df34","unresolved":false,"context_lines":[{"line_number":603,"context_line":"    @public"},{"line_number":604,"context_line":"    @delay_denial"},{"line_number":605,"context_line":"    @cors_validation"},{"line_number":606,"context_line":"    def HEAD(self, req):"},{"line_number":607,"context_line":"        \"\"\"Handler for HTTP HEAD requests.\"\"\""},{"line_number":608,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":609,"context_line":"        if aresp:"}],"source_content_type":"text/x-python","patch_set":2,"id":"8f4a221b_89f9a6e4","line":606,"updated":"2023-11-19 01:01:40.000000000","message":"Before the refactoring, shard range GET, object listing GET and HEAD all shared one main code path. Very glad to see GET and HEAD are separated now.","commit_id":"c447e9adc7bcf2a56f34e61d3831f6f4acb51350"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":true,"context_lines":[{"line_number":425,"context_line":"            # itself while recursively building a listing from shards."},{"line_number":426,"context_line":"            resp \u003d self._GET_auto(req)"},{"line_number":427,"context_line":"            resp.headers.pop(\u0027X-Backend-Record-Type\u0027, None)"},{"line_number":428,"context_line":"            resp.headers.pop(\u0027X-Backend-Record-Shard-Format\u0027, None)"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":431,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"e3fe54ff_c062ecfb","line":428,"updated":"2024-01-16 06:35:44.000000000","message":"I LOVE this new simplfied GET function. So much easier to read and grok what\u0027s happening.\n\nI also can\u0027t think of anything off the top of my head that when specifying a record_type, where we don\u0027t want it to go directly to the backend.\nclient traffic from the proxy will always be auto, and that\u0027s where we need the caching. If something internal it hitting a backend too hard, then we need to do better ourselves 😊","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"771d8dd6ac5c49dd35748b47acc396431c173b60","unresolved":false,"context_lines":[{"line_number":425,"context_line":"            # itself while recursively building a listing from shards."},{"line_number":426,"context_line":"            resp \u003d self._GET_auto(req)"},{"line_number":427,"context_line":"            resp.headers.pop(\u0027X-Backend-Record-Type\u0027, None)"},{"line_number":428,"context_line":"            resp.headers.pop(\u0027X-Backend-Record-Shard-Format\u0027, None)"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":431,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"c97cd720_22b81bc3","line":428,"in_reply_to":"e3fe54ff_c062ecfb","updated":"2024-01-16 10:20:40.000000000","message":"Acknowledged","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":true,"context_lines":[{"line_number":520,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":521,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":522,"context_line":"            else:"},{"line_number":523,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":524,"context_line":"            if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":525,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":526,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"86346593_78324c83","line":523,"updated":"2024-01-16 06:35:44.000000000","message":"Yup make sure new calls have auto to make sure they\u0027re passed through the auto path, nice.","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"72560586573075d85ce92b342da54a6887cc3e4c","unresolved":true,"context_lines":[{"line_number":520,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":521,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":522,"context_line":"            else:"},{"line_number":523,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":524,"context_line":"            if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":525,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":526,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"a5d9eec4_8c8b0c59","line":523,"in_reply_to":"86346593_78324c83","updated":"2024-01-22 06:48:45.000000000","message":"if this part of code doesn\u0027t have ``headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027`` set, calling ``ContainerController._get_container_listing``  and recursively calling ``ContainerController.GET()`` again will call ``self._GET_auto(req)`` by default, but adding this line will be more explicit.\n\nwhen ``ContainerController._get_container_listing`` recursively builds the container listing, it need go to the ``self._GET_auto(req)`` path (except the loop case) because it doesn\u0027t know if a shard container is sharded or unsharded? since a shard container could shard itself as well?","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"732064dcdf65268fa1f21dfeea8c4c3b6c626ec9","unresolved":false,"context_lines":[{"line_number":520,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":521,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":522,"context_line":"            else:"},{"line_number":523,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":524,"context_line":"            if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":525,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":526,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"b3a2a0d3_ca5b2f5c","line":523,"in_reply_to":"99882fe2_5450e08e","updated":"2024-01-23 05:39:18.000000000","message":"Acknowledged","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e5692de260d797d17b2539c317267e1483cabb09","unresolved":true,"context_lines":[{"line_number":520,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":521,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":522,"context_line":"            else:"},{"line_number":523,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":524,"context_line":"            if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":525,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":526,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"99882fe2_5450e08e","line":523,"in_reply_to":"a5d9eec4_8c8b0c59","updated":"2024-01-22 09:46:28.000000000","message":"\u003e recursively calling ContainerController.GET() again will call self._GET_auto(req) by default, but adding this line will be more explicit.\n\n@Jianjian - you are correct. The \u0027auto\u0027 head is not strictly necessary because that is the default path anyway, but I felt it made it more obvious, particularly with respect to the alternative \u0027object\u0027 case.\n\n\u003e it need go to the self._GET_auto(req) path (except the loop case) because it doesn\u0027t know if a shard container is sharded or unsharded? since a shard container could shard itself as well?\n\ncorrect, shards can be sharded or sharding but not yet updated the root container, so the state of the shard is only discovered when we recurse to it.","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":true,"context_lines":[{"line_number":613,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":614,"context_line":"        if aresp:"},{"line_number":615,"context_line":"            return aresp"},{"line_number":616,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":617,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":19,"id":"87ea697c_a3f828d7","line":616,"updated":"2024-01-16 06:35:44.000000000","message":"When we do container_info\u0027s we store the result in cache.. so I do wonder if we do want to support getting head from cache at some point in the future rather the _always_ going to back end :thiking:\n\nJust a thought, and something to experiement with later. This is completely out of scope of this change 😊","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"98414533bc5427e485f02858232c01fb55cf6731","unresolved":false,"context_lines":[{"line_number":613,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":614,"context_line":"        if aresp:"},{"line_number":615,"context_line":"            return aresp"},{"line_number":616,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":617,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":19,"id":"6ebfca14_2c65ec7e","line":616,"in_reply_to":"03584bd5_400ff163","updated":"2024-01-23 05:18:19.000000000","message":"Acknowledged","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9aa8847cbfb25b19dcf6355434b53ef2910919c8","unresolved":true,"context_lines":[{"line_number":613,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":614,"context_line":"        if aresp:"},{"line_number":615,"context_line":"            return aresp"},{"line_number":616,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":617,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":19,"id":"03584bd5_400ff163","line":616,"in_reply_to":"87ea697c_a3f828d7","updated":"2024-01-18 05:40:04.000000000","message":"I had a patch in the past\nhttps://review.opendev.org/c/openstack/swift/+/883638\n\nafter we have optimized container HEAD so much, maybe reading it from cache is not that helpful anymore.","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9aa8847cbfb25b19dcf6355434b53ef2910919c8","unresolved":true,"context_lines":[{"line_number":225,"context_line":"                self.logger, self.server_type.lower(), \u0027shard_listing\u0027,"},{"line_number":226,"context_line":"                cache_state, resp)"},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"    def _GET_auto(self, req):"},{"line_number":229,"context_line":"        # This is an object listing but the backend may be sharded."},{"line_number":230,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":231,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":20,"id":"5bc4ce8b_c8ab849a","line":228,"updated":"2024-01-18 05:40:04.000000000","message":"this single new function handles so many different cases: sharded, unsharded, cache_enabled, cache_disable, cache hit, cache miss... I would strongly advocate for splitting it into a few sub-functions and each of them handles only one/two case. Like: _GET_auto() only collect container info and memcache, and then call into get_sharded_cached_enabled(), get_sharded_cached_disabled(), get_unsharded_cached_enabled()... and create some helper functions for those duplicate code among sub-functions.","commit_id":"0c6ccf2abe8c4dc3d98a210c20c647c750118f81"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a0fa4dd98b141c700dfe30c6083eee515a583fb4","unresolved":true,"context_lines":[{"line_number":225,"context_line":"                self.logger, self.server_type.lower(), \u0027shard_listing\u0027,"},{"line_number":226,"context_line":"                cache_state, resp)"},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"    def _GET_auto(self, req):"},{"line_number":229,"context_line":"        # This is an object listing but the backend may be sharded."},{"line_number":230,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":231,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":20,"id":"c62c4710_2a53b0fb","line":228,"in_reply_to":"5bc4ce8b_c8ab849a","updated":"2024-01-18 11:22:09.000000000","message":"I have separated some pieces (in next patchset) but I\u0027m not seeing much more opportunity to split the method up. Let me know if you have specific suggestions.","commit_id":"0c6ccf2abe8c4dc3d98a210c20c647c750118f81"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"72560586573075d85ce92b342da54a6887cc3e4c","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                self.logger, self.server_type.lower(), \u0027shard_listing\u0027,"},{"line_number":226,"context_line":"                cache_state, resp)"},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"    def _GET_auto(self, req):"},{"line_number":229,"context_line":"        # This is an object listing but the backend may be sharded."},{"line_number":230,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":231,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":20,"id":"090206ba_6cbc4e32","line":228,"in_reply_to":"c62c4710_2a53b0fb","updated":"2024-01-22 06:48:45.000000000","message":"new patchset looks good, I feel readability is much improved, thanks!","commit_id":"0c6ccf2abe8c4dc3d98a210c20c647c750118f81"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9aa8847cbfb25b19dcf6355434b53ef2910919c8","unresolved":false,"context_lines":[{"line_number":387,"context_line":"    @public"},{"line_number":388,"context_line":"    @delay_denial"},{"line_number":389,"context_line":"    @cors_validation"},{"line_number":390,"context_line":"    def GET(self, req):"},{"line_number":391,"context_line":"        \"\"\"Handler for HTTP GET requests.\"\"\""},{"line_number":392,"context_line":"        # early checks for request validity"},{"line_number":393,"context_line":"        validate_container_params(req)"}],"source_content_type":"text/x-python","patch_set":20,"id":"9ce51511_7f42ec12","line":390,"updated":"2024-01-18 05:40:04.000000000","message":"the main flow of this GET() is great, easy to read and follow.","commit_id":"0c6ccf2abe8c4dc3d98a210c20c647c750118f81"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"732064dcdf65268fa1f21dfeea8c4c3b6c626ec9","unresolved":true,"context_lines":[{"line_number":143,"context_line":"        skip_chance \u003d self.app.container_listing_shard_ranges_skip_cache"},{"line_number":144,"context_line":"        ns_bound_list, cache_state \u003d get_namespaces_from_cache("},{"line_number":145,"context_line":"            req, cache_key, skip_chance)"},{"line_number":146,"context_line":"        if ns_bound_list:"},{"line_number":147,"context_line":"            # namespaces can be returned from cache"},{"line_number":148,"context_line":"            namespaces \u003d ns_bound_list.get_namespaces()"},{"line_number":149,"context_line":"            self.logger.debug(\u0027Found %d shards in cache for %s\u0027,"}],"source_content_type":"text/x-python","patch_set":21,"id":"7c40643f_3d6dc0f9","line":146,"updated":"2024-01-23 05:39:18.000000000","message":"consider to use guard return here?\nsee https://review.opendev.org/c/openstack/swift/+/906338","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":false,"context_lines":[{"line_number":143,"context_line":"        skip_chance \u003d self.app.container_listing_shard_ranges_skip_cache"},{"line_number":144,"context_line":"        ns_bound_list, cache_state \u003d get_namespaces_from_cache("},{"line_number":145,"context_line":"            req, cache_key, skip_chance)"},{"line_number":146,"context_line":"        if ns_bound_list:"},{"line_number":147,"context_line":"            # namespaces can be returned from cache"},{"line_number":148,"context_line":"            namespaces \u003d ns_bound_list.get_namespaces()"},{"line_number":149,"context_line":"            self.logger.debug(\u0027Found %d shards in cache for %s\u0027,"}],"source_content_type":"text/x-python","patch_set":21,"id":"8d3bc217_cd484c94","line":146,"in_reply_to":"7c40643f_3d6dc0f9","updated":"2024-01-24 16:43:48.000000000","message":"Done","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":false,"context_lines":[{"line_number":143,"context_line":"        skip_chance \u003d self.app.container_listing_shard_ranges_skip_cache"},{"line_number":144,"context_line":"        ns_bound_list, cache_state \u003d get_namespaces_from_cache("},{"line_number":145,"context_line":"            req, cache_key, skip_chance)"},{"line_number":146,"context_line":"        if ns_bound_list:"},{"line_number":147,"context_line":"            # namespaces can be returned from cache"},{"line_number":148,"context_line":"            namespaces \u003d ns_bound_list.get_namespaces()"},{"line_number":149,"context_line":"            self.logger.debug(\u0027Found %d shards in cache for %s\u0027,"}],"source_content_type":"text/x-python","patch_set":21,"id":"f33d4757_7d1bd791","line":146,"in_reply_to":"7c40643f_3d6dc0f9","updated":"2024-01-24 06:40:51.000000000","message":"Done","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"732064dcdf65268fa1f21dfeea8c4c3b6c626ec9","unresolved":false,"context_lines":[{"line_number":160,"context_line":"            headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":161,"context_line":"                            \u0027x-backend-record-shard-format\u0027: \u0027namespace\u0027,"},{"line_number":162,"context_line":"                            \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":163,"context_line":"            resp \u003d Response(request\u003dreq)"},{"line_number":164,"context_line":"            update_headers(resp, headers)"},{"line_number":165,"context_line":"            resp.last_modified \u003d Timestamp("},{"line_number":166,"context_line":"                headers[\u0027x-put-timestamp\u0027]).ceil()"}],"source_content_type":"text/x-python","patch_set":21,"id":"a801e4bd_6946d5b6","line":163,"updated":"2024-01-23 05:39:18.000000000","message":"so here, we don\u0027t encode ``Namespace`` objects into json string and attach it to be the response body anymore.","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"732064dcdf65268fa1f21dfeea8c4c3b6c626ec9","unresolved":true,"context_lines":[{"line_number":208,"context_line":"        return ns_bound_list.get_namespaces()"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    def _get_listing_namespaces_from_backend(self, req, cache_enabled):"},{"line_number":211,"context_line":"        # Instruct the backend server to \u0027automatically\u0027 return namespaces"},{"line_number":212,"context_line":"        # of shards in a \u0027listing\u0027 state if the container is sharded, and"},{"line_number":213,"context_line":"        # that the more compact \u0027namespace\u0027 format is sufficient. Older"},{"line_number":214,"context_line":"        # container servers may still respond with the \u0027full\u0027 shard range"}],"source_content_type":"text/x-python","patch_set":21,"id":"e84f5ed5_85fefdfe","line":211,"range":{"start_line":211,"start_character":0,"end_line":211,"end_character":8},"updated":"2024-01-23 05:39:18.000000000","message":"add docstrings for ``_get_listing_namespaces_from_backend``","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        return ns_bound_list.get_namespaces()"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    def _get_listing_namespaces_from_backend(self, req, cache_enabled):"},{"line_number":211,"context_line":"        # Instruct the backend server to \u0027automatically\u0027 return namespaces"},{"line_number":212,"context_line":"        # of shards in a \u0027listing\u0027 state if the container is sharded, and"},{"line_number":213,"context_line":"        # that the more compact \u0027namespace\u0027 format is sufficient. Older"},{"line_number":214,"context_line":"        # container servers may still respond with the \u0027full\u0027 shard range"}],"source_content_type":"text/x-python","patch_set":21,"id":"0a28e7dc_cd0a56c0","line":211,"range":{"start_line":211,"start_character":0,"end_line":211,"end_character":8},"in_reply_to":"e84f5ed5_85fefdfe","updated":"2024-01-24 16:43:48.000000000","message":"Done","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        return ns_bound_list.get_namespaces()"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    def _get_listing_namespaces_from_backend(self, req, cache_enabled):"},{"line_number":211,"context_line":"        # Instruct the backend server to \u0027automatically\u0027 return namespaces"},{"line_number":212,"context_line":"        # of shards in a \u0027listing\u0027 state if the container is sharded, and"},{"line_number":213,"context_line":"        # that the more compact \u0027namespace\u0027 format is sufficient. Older"},{"line_number":214,"context_line":"        # container servers may still respond with the \u0027full\u0027 shard range"}],"source_content_type":"text/x-python","patch_set":21,"id":"37e6b39c_d87a0637","line":211,"range":{"start_line":211,"start_character":0,"end_line":211,"end_character":8},"in_reply_to":"e84f5ed5_85fefdfe","updated":"2024-01-24 06:40:51.000000000","message":"Done","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"98414533bc5427e485f02858232c01fb55cf6731","unresolved":true,"context_lines":[{"line_number":333,"context_line":"                resp \u003d namespaces \u003d None"},{"line_number":334,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":335,"context_line":"        else:"},{"line_number":336,"context_line":"            resp \u003d namespaces \u003d None"},{"line_number":337,"context_line":"            cache_state \u003d \u0027disabled\u0027"},{"line_number":338,"context_line":""},{"line_number":339,"context_line":"        if not namespaces:"}],"source_content_type":"text/x-python","patch_set":21,"id":"8904d839_368e2cbc","line":336,"range":{"start_line":336,"start_character":12,"end_line":336,"end_character":36},"updated":"2024-01-23 05:18:19.000000000","message":"OK this initialisation has now been done 3 times in different branches. Surely we could just move it up and do it once?","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":false,"context_lines":[{"line_number":333,"context_line":"                resp \u003d namespaces \u003d None"},{"line_number":334,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":335,"context_line":"        else:"},{"line_number":336,"context_line":"            resp \u003d namespaces \u003d None"},{"line_number":337,"context_line":"            cache_state \u003d \u0027disabled\u0027"},{"line_number":338,"context_line":""},{"line_number":339,"context_line":"        if not namespaces:"}],"source_content_type":"text/x-python","patch_set":21,"id":"1b22ac96_9f9eed82","line":336,"range":{"start_line":336,"start_character":12,"end_line":336,"end_character":36},"in_reply_to":"8904d839_368e2cbc","updated":"2024-01-24 16:43:48.000000000","message":"LOL I had it that way, then decided to write it more explicitly so it\u0027s obvious that the vars are initialised in every case.","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2d1d60e52c61993fd6c22768ff5aef37fc503951","unresolved":false,"context_lines":[{"line_number":3005,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges("},{"line_number":3006,"context_line":"            headers\u003d{\u0027X-Newest\u0027: \u0027true\u0027},"},{"line_number":3007,"context_line":"            params\u003d{\u0027states\u0027: \u0027listing\u0027})"},{"line_number":3008,"context_line":"        self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3009,"context_line":""},{"line_number":3010,"context_line":"        # this is what the sharder requests..."},{"line_number":3011,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges("}],"source_content_type":"text/x-python","patch_set":19,"id":"6cf58fa0_ef86050d","line":3008,"updated":"2024-01-16 06:35:44.000000000","message":"Nice","commit_id":"e1dc97831575a596a568d3fe26103a236c8857e8"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":false,"context_lines":[{"line_number":3000,"context_line":""},{"line_number":3001,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges("},{"line_number":3002,"context_line":"            params\u003d{\u0027states\u0027: \u0027listing\u0027})"},{"line_number":3003,"context_line":"        self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3004,"context_line":""},{"line_number":3005,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges("},{"line_number":3006,"context_line":"            headers\u003d{\u0027X-Newest\u0027: \u0027true\u0027},"}],"source_content_type":"text/x-python","patch_set":21,"id":"408003d0_5fd01d00","line":3003,"updated":"2024-01-24 06:40:51.000000000","message":"here verifies the fixes work as expected.","commit_id":"b2a47afd4a788e499e533723a07e5755d16de0c7"}],"test/unit/proxy/controllers/test_container.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":true,"context_lines":[{"line_number":2469,"context_line":"        shard_resp_hdrs \u003d self._make_shard_resp_hdrs(sr_objs)"},{"line_number":2470,"context_line":"        shard_1_shard_resp_hdrs \u003d dict(shard_resp_hdrs[1])"},{"line_number":2471,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Record-Type\u0027] \u003d \u0027shard\u0027"},{"line_number":2472,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharded\u0027"},{"line_number":2473,"context_line":"        shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS)"},{"line_number":2474,"context_line":""},{"line_number":2475,"context_line":"        # second shard is sharding and has cleaved two out of three sub shards"}],"source_content_type":"text/x-python","patch_set":23,"id":"0b2db4dd_7d513b86","line":2472,"updated":"2024-01-24 06:40:51.000000000","message":"why this is \u0027sharded\u0027? if this is \u0027sharded\u0027, why the namespaces of shard_1 didn\u0027t get cached into memcache? https://review.opendev.org/c/openstack/swift/+/901335/23/test/unit/proxy/controllers/test_container.py#2579\n\nShould this header be set to \u0027sharding\u0027? I changed it to \u0027sharding\u0027, and test still passes.","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":true,"context_lines":[{"line_number":2469,"context_line":"        shard_resp_hdrs \u003d self._make_shard_resp_hdrs(sr_objs)"},{"line_number":2470,"context_line":"        shard_1_shard_resp_hdrs \u003d dict(shard_resp_hdrs[1])"},{"line_number":2471,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Record-Type\u0027] \u003d \u0027shard\u0027"},{"line_number":2472,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharded\u0027"},{"line_number":2473,"context_line":"        shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS)"},{"line_number":2474,"context_line":""},{"line_number":2475,"context_line":"        # second shard is sharding and has cleaved two out of three sub shards"}],"source_content_type":"text/x-python","patch_set":23,"id":"ac94a1eb_0d2982d3","line":2472,"in_reply_to":"0b2db4dd_7d513b86","updated":"2024-01-24 16:43:48.000000000","message":"This is wrong 😭 BUT, this header is returned with the second \u0027object\u0027 GET to the shard (acting as a filler range) so shard caching is not relevant here.","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"4786e8ddc68169c7da714429f4e78cb4aa9dfe2b","unresolved":false,"context_lines":[{"line_number":2469,"context_line":"        shard_resp_hdrs \u003d self._make_shard_resp_hdrs(sr_objs)"},{"line_number":2470,"context_line":"        shard_1_shard_resp_hdrs \u003d dict(shard_resp_hdrs[1])"},{"line_number":2471,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Record-Type\u0027] \u003d \u0027shard\u0027"},{"line_number":2472,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharded\u0027"},{"line_number":2473,"context_line":"        shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS)"},{"line_number":2474,"context_line":""},{"line_number":2475,"context_line":"        # second shard is sharding and has cleaved two out of three sub shards"}],"source_content_type":"text/x-python","patch_set":23,"id":"db7184eb_83d31e9b","line":2472,"in_reply_to":"0b2db4dd_7d513b86","updated":"2024-01-24 16:42:41.000000000","message":"oic, this test doesn\u0027t set ``\u0027x-backend-override-shard-name-filter\u0027`` in the response header, so the namespaces of shard_1 won\u0027t get cached.","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":false,"context_lines":[{"line_number":2469,"context_line":"        shard_resp_hdrs \u003d self._make_shard_resp_hdrs(sr_objs)"},{"line_number":2470,"context_line":"        shard_1_shard_resp_hdrs \u003d dict(shard_resp_hdrs[1])"},{"line_number":2471,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Record-Type\u0027] \u003d \u0027shard\u0027"},{"line_number":2472,"context_line":"        shard_1_shard_resp_hdrs[\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharded\u0027"},{"line_number":2473,"context_line":"        shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS)"},{"line_number":2474,"context_line":""},{"line_number":2475,"context_line":"        # second shard is sharding and has cleaved two out of three sub shards"}],"source_content_type":"text/x-python","patch_set":23,"id":"b2ffd983_534a2553","line":2472,"in_reply_to":"ac94a1eb_0d2982d3","updated":"2024-01-25 00:32:03.000000000","message":"Acknowledged","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":true,"context_lines":[{"line_number":2473,"context_line":"        shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS)"},{"line_number":2474,"context_line":""},{"line_number":2475,"context_line":"        # second shard is sharding and has cleaved two out of three sub shards"},{"line_number":2476,"context_line":"        shard_resp_hdrs[1][\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharding\u0027"},{"line_number":2477,"context_line":"        sub_shard_bounds \u003d ((\u0027ham\u0027, \u0027juice\u0027), (\u0027juice\u0027, \u0027lemon\u0027))"},{"line_number":2478,"context_line":"        sub_shard_ranges \u003d ["},{"line_number":2479,"context_line":"            ShardRange(\u0027a/c_sub_\u0027 + upper, Timestamp.now(), lower, upper)"}],"source_content_type":"text/x-python","patch_set":23,"id":"48e78cf2_678589c4","line":2476,"updated":"2024-01-24 06:40:51.000000000","message":"this is \u0027sharding\u0027, but test passes if I set it to \u0027sharded\u0027.","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":true,"context_lines":[{"line_number":2473,"context_line":"        shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS)"},{"line_number":2474,"context_line":""},{"line_number":2475,"context_line":"        # second shard is sharding and has cleaved two out of three sub shards"},{"line_number":2476,"context_line":"        shard_resp_hdrs[1][\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharding\u0027"},{"line_number":2477,"context_line":"        sub_shard_bounds \u003d ((\u0027ham\u0027, \u0027juice\u0027), (\u0027juice\u0027, \u0027lemon\u0027))"},{"line_number":2478,"context_line":"        sub_shard_ranges \u003d ["},{"line_number":2479,"context_line":"            ShardRange(\u0027a/c_sub_\u0027 + upper, Timestamp.now(), lower, upper)"}],"source_content_type":"text/x-python","patch_set":23,"id":"5de7c92e_9a1f826c","line":2476,"in_reply_to":"48e78cf2_678589c4","updated":"2024-01-24 16:43:48.000000000","message":"this header is returned for the first \u0027auto\u0027 GET to the shard, so shards are not cached.\n\nIf this is changed to ``sharded`` the shards are still not cached because the response does not have ``shard_1_shard_resp_hdrs[\u0027X-Backend-Override-Shard-Name-Filter\u0027] \u003d \u0027true\u0027``\n\nIf we add ``shard_1_shard_resp_hdrs[\u0027X-Backend-Override-Shard-Name-Filter\u0027] \u003d \u0027true\u0027`` then the test will fail, because we don\u0027t see the expected requests. That is because the header is a lie - the test is not returning a complete set of shards, and the filler range is not accurately set up 😞\n\nOnce I fixed that filler range, the test would fail with\n\n```\ntest/unit/proxy/controllers/test_container.py:2468 (TestGetShardedContainer.test_GET_sharded_container_sharding_shard_with_memcache)\n{\u0027shard-listing-v2/.shards_a/c_pie\u0027, \u0027shard-listing-v2/a/c\u0027} !\u003d {\u0027shard-listing-v2/a/c\u0027}\n\nExpected :{\u0027shard-listing-v2/a/c\u0027}\nActual   :{\u0027shard-listing-v2/.shards_a/c_pie\u0027, \u0027shard-listing-v2/a/c\u0027}\n```","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":false,"context_lines":[{"line_number":2473,"context_line":"        shard_1_shard_resp_hdrs.update(self.RESP_SHARD_FORMAT_HEADERS)"},{"line_number":2474,"context_line":""},{"line_number":2475,"context_line":"        # second shard is sharding and has cleaved two out of three sub shards"},{"line_number":2476,"context_line":"        shard_resp_hdrs[1][\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharding\u0027"},{"line_number":2477,"context_line":"        sub_shard_bounds \u003d ((\u0027ham\u0027, \u0027juice\u0027), (\u0027juice\u0027, \u0027lemon\u0027))"},{"line_number":2478,"context_line":"        sub_shard_ranges \u003d ["},{"line_number":2479,"context_line":"            ShardRange(\u0027a/c_sub_\u0027 + upper, Timestamp.now(), lower, upper)"}],"source_content_type":"text/x-python","patch_set":23,"id":"9516a9ff_9f324bfc","line":2476,"in_reply_to":"5de7c92e_9a1f826c","updated":"2024-01-25 00:32:03.000000000","message":"Acknowledged","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":true,"context_lines":[{"line_number":2511,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":2512,"context_line":"            (200, sr_objs[0], shard_resp_hdrs[0]),"},{"line_number":2513,"context_line":"            (200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs),"},{"line_number":2514,"context_line":"            (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]),"},{"line_number":2515,"context_line":"            (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]),"},{"line_number":2516,"context_line":"            (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):],"},{"line_number":2517,"context_line":"             shard_resp_hdrs[1]),"}],"source_content_type":"text/x-python","patch_set":23,"id":"84d1b039_d6b0fe4f","line":2514,"updated":"2024-01-24 06:40:51.000000000","message":"in production cluster, if three replicas of sub shard are not synced yet, is that possible we will get some weird results in ``(200, sub_sr_objs[0], sub_shard_resp_hdrs[0])``? but that\u0027ll be a listing consistency issue, not related to testing in this patch.\n\nhowever, I am thinking what if all three replicas of a sub shard are down? so when proxy container controller is iterating over a sharded container to get object listings from all sub-shards, but one shard is all down, so there will be a gap in the listing. will container controller work okay, and do we have a test case to cover this case?","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":false,"context_lines":[{"line_number":2511,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":2512,"context_line":"            (200, sr_objs[0], shard_resp_hdrs[0]),"},{"line_number":2513,"context_line":"            (200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs),"},{"line_number":2514,"context_line":"            (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]),"},{"line_number":2515,"context_line":"            (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]),"},{"line_number":2516,"context_line":"            (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):],"},{"line_number":2517,"context_line":"             shard_resp_hdrs[1]),"}],"source_content_type":"text/x-python","patch_set":23,"id":"02b8ed90_ae524c8d","line":2514,"in_reply_to":"1d48eab3_4ce32568","updated":"2024-01-25 00:32:03.000000000","message":"Acknowledged","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cbd604601a3097690472d496c3c5cc8f7a856bcf","unresolved":true,"context_lines":[{"line_number":2511,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":2512,"context_line":"            (200, sr_objs[0], shard_resp_hdrs[0]),"},{"line_number":2513,"context_line":"            (200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs),"},{"line_number":2514,"context_line":"            (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]),"},{"line_number":2515,"context_line":"            (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]),"},{"line_number":2516,"context_line":"            (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):],"},{"line_number":2517,"context_line":"             shard_resp_hdrs[1]),"}],"source_content_type":"text/x-python","patch_set":23,"id":"1d48eab3_4ce32568","line":2514,"in_reply_to":"84d1b039_d6b0fe4f","updated":"2024-01-24 16:43:48.000000000","message":"Great question!\n\nIn ContainerController._get_from_shards, if a shard GET does not succeed then the listing aborts and the client gets a 503, rather than a gappy listing.\n\nSee https://review.opendev.org/c/openstack/swift/+/901335/23/swift/proxy/controllers/container.py#566.\n\nThere are several tests that exercise this path, each test uses _do_test_GET_sharded_container_with_deleted_shards","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"681cf48b1ea4be61dda005194ea434dbb0c8326a","unresolved":false,"context_lines":[{"line_number":2892,"context_line":"        self.assertEqual(exp_backend_hdrs, fake_conn.requests[0][\u0027headers\u0027])"},{"line_number":2893,"context_line":"        self.assertNotIn(\u0027state\u003d\u0027, fake_conn.requests[0][\u0027qs\u0027])"},{"line_number":2894,"context_line":"        # NB: no namespaces cached"},{"line_number":2895,"context_line":"        self.assertEqual([mock.call.set(\u0027container/a/c\u0027, mock.ANY, time\u003d60)],"},{"line_number":2896,"context_line":"                         memcache.calls)"},{"line_number":2897,"context_line":"        self.assertEqual(sr_dicts, json.loads(resp.body))"},{"line_number":2898,"context_line":""}],"source_content_type":"text/x-python","patch_set":23,"id":"ccf9fefb_03e226b4","line":2895,"updated":"2024-01-24 06:40:51.000000000","message":"there is only container info set in memcache.","commit_id":"57c4e5cd01b8d358714de66069bfa6e62b89aab5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":true,"context_lines":[{"line_number":2473,"context_line":"            shard_bounds)"},{"line_number":2474,"context_line":"        # headers returned with obj listing from shard containers..."},{"line_number":2475,"context_line":"        shard_obj_resp_hdrs \u003d self._make_shard_resp_hdrs(sr_objs)"},{"line_number":2476,"context_line":"        # modify second shard\u0027s obj listing resp - this one is sharding..."},{"line_number":2477,"context_line":"        shard_obj_resp_hdrs[1][\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharded\u0027"},{"line_number":2478,"context_line":"        # ...and will return shards in \u0027response\u0027 to auto record-type..."},{"line_number":2479,"context_line":"        shard_1_shard_resp_hdrs \u003d dict(shard_obj_resp_hdrs[1])"}],"source_content_type":"text/x-python","patch_set":24,"id":"11123eb9_545d5594","line":2476,"updated":"2024-01-25 00:32:03.000000000","message":"nit: s/sharding/sharded","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"34caab6883c358d1ed4e85fff848dd36a4eeaecd","unresolved":false,"context_lines":[{"line_number":2473,"context_line":"            shard_bounds)"},{"line_number":2474,"context_line":"        # headers returned with obj listing from shard containers..."},{"line_number":2475,"context_line":"        shard_obj_resp_hdrs \u003d self._make_shard_resp_hdrs(sr_objs)"},{"line_number":2476,"context_line":"        # modify second shard\u0027s obj listing resp - this one is sharding..."},{"line_number":2477,"context_line":"        shard_obj_resp_hdrs[1][\u0027X-Backend-Sharding-State\u0027] \u003d \u0027sharded\u0027"},{"line_number":2478,"context_line":"        # ...and will return shards in \u0027response\u0027 to auto record-type..."},{"line_number":2479,"context_line":"        shard_1_shard_resp_hdrs \u003d dict(shard_obj_resp_hdrs[1])"}],"source_content_type":"text/x-python","patch_set":24,"id":"5ee62347_f178ce3d","line":2476,"in_reply_to":"11123eb9_545d5594","updated":"2024-01-25 17:26:59.000000000","message":"argh! I still got this wrong! I think that in my attempt to prove that the test would fail, I left the wrong state here.\n\nThe state of the second shard range should be sharding. The setup should be the same as the previous test ``test_GET_sharded_container_sharding_shard_no_memcache``.","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":true,"context_lines":[{"line_number":3628,"context_line":"                         self.logger.statsd_client.get_increment_counts())"},{"line_number":3629,"context_line":"        return resp"},{"line_number":3630,"context_line":""},{"line_number":3631,"context_line":"    def test_GET_shard_ranges_write_to_cache(self):"},{"line_number":3632,"context_line":"        exp_resp_hdrs \u003d {\u0027X-Backend-Recheck-Container-Existence\u0027: \u002760\u0027,"},{"line_number":3633,"context_line":"                         \u0027X-Backend-Override-Shard-Name-Filter\u0027: \u0027true\u0027,"},{"line_number":3634,"context_line":"                         \u0027X-Backend-Sharding-State\u0027: \u0027sharded\u0027}"}],"source_content_type":"text/x-python","patch_set":24,"id":"8b93a270_7a62ae5e","line":3631,"updated":"2024-01-25 00:32:03.000000000","message":"nit: s/test_GET_shard_ranges_write_to_cache/test_GET_namespaces_write_to_cache","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"34caab6883c358d1ed4e85fff848dd36a4eeaecd","unresolved":false,"context_lines":[{"line_number":3628,"context_line":"                         self.logger.statsd_client.get_increment_counts())"},{"line_number":3629,"context_line":"        return resp"},{"line_number":3630,"context_line":""},{"line_number":3631,"context_line":"    def test_GET_shard_ranges_write_to_cache(self):"},{"line_number":3632,"context_line":"        exp_resp_hdrs \u003d {\u0027X-Backend-Recheck-Container-Existence\u0027: \u002760\u0027,"},{"line_number":3633,"context_line":"                         \u0027X-Backend-Override-Shard-Name-Filter\u0027: \u0027true\u0027,"},{"line_number":3634,"context_line":"                         \u0027X-Backend-Sharding-State\u0027: \u0027sharded\u0027}"}],"source_content_type":"text/x-python","patch_set":24,"id":"19167924_2f7f4604","line":3631,"in_reply_to":"8b93a270_7a62ae5e","updated":"2024-01-25 17:26:59.000000000","message":"Done","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"bae6a4a5a70cf84981f6eddd3cfdad87537c0efd","unresolved":true,"context_lines":[{"line_number":4067,"context_line":"                         self.memcache.calls[0][1][1][\u0027sharding_state\u0027])"},{"line_number":4068,"context_line":""},{"line_number":4069,"context_line":"    def test_GET_shard_ranges_no_cache_when_requesting_listing_shards(self):"},{"line_number":4070,"context_line":"        # verify that a GET for shard record type in listting states does not"},{"line_number":4071,"context_line":"        # lookup or store in cache"},{"line_number":4072,"context_line":"        self._do_test_GET_shard_ranges_no_cache_listing(\u0027unsharded\u0027)"},{"line_number":4073,"context_line":"        self._do_test_GET_shard_ranges_no_cache_listing(\u0027sharding\u0027)"}],"source_content_type":"text/x-python","patch_set":24,"id":"209d4710_166daeda","line":4070,"updated":"2024-01-25 00:32:03.000000000","message":"s/listting/listing","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"34caab6883c358d1ed4e85fff848dd36a4eeaecd","unresolved":false,"context_lines":[{"line_number":4067,"context_line":"                         self.memcache.calls[0][1][1][\u0027sharding_state\u0027])"},{"line_number":4068,"context_line":""},{"line_number":4069,"context_line":"    def test_GET_shard_ranges_no_cache_when_requesting_listing_shards(self):"},{"line_number":4070,"context_line":"        # verify that a GET for shard record type in listting states does not"},{"line_number":4071,"context_line":"        # lookup or store in cache"},{"line_number":4072,"context_line":"        self._do_test_GET_shard_ranges_no_cache_listing(\u0027unsharded\u0027)"},{"line_number":4073,"context_line":"        self._do_test_GET_shard_ranges_no_cache_listing(\u0027sharding\u0027)"}],"source_content_type":"text/x-python","patch_set":24,"id":"b7dc47a1_6333a51b","line":4070,"in_reply_to":"209d4710_166daeda","updated":"2024-01-25 17:26:59.000000000","message":"Done","commit_id":"e1b43037a365eb62c10bc37841cbd120bdf2dee4"}]}