)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":20,"context_line":""},{"line_number":21,"context_line":"  To enable this, the container server and ContainerBroker are"},{"line_number":22,"context_line":"  modified to support requests for namespaces with query params"},{"line_number":23,"context_line":"  includes, marker, end_marker and reverse. These are significant the"},{"line_number":24,"context_line":"  proxy when there is no memcache. (When there is memcache, these"},{"line_number":25,"context_line":"  params are still sent to the backend but overridden if the container"},{"line_number":26,"context_line":"  is sharded.)"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"884156df_2d02fe5d","line":23,"range":{"start_line":23,"start_character":65,"end_line":23,"end_character":66},"updated":"2023-11-14 04:05:42.000000000","message":"Missing \u0027to\u0027 maybe?","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":20,"context_line":""},{"line_number":21,"context_line":"  To enable this, the container server and ContainerBroker are"},{"line_number":22,"context_line":"  modified to support requests for namespaces with query params"},{"line_number":23,"context_line":"  includes, marker, end_marker and reverse. These are significant the"},{"line_number":24,"context_line":"  proxy when there is no memcache. (When there is memcache, these"},{"line_number":25,"context_line":"  params are still sent to the backend but overridden if the container"},{"line_number":26,"context_line":"  is sharded.)"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"84a3919b_0629c3c0","line":23,"range":{"start_line":23,"start_character":65,"end_line":23,"end_character":66},"in_reply_to":"884156df_2d02fe5d","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":"A bug is fixed in _get_from_shards: it now always sets the"},{"line_number":47,"context_line":"X-Backend-Record-Type header to \u0027object\u0027, rather than sometimes"},{"line_number":48,"context_line":"leaving the value as \u0027shard\u0027."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"Some unit tests are also refactored to be more realistic when mocking"},{"line_number":51,"context_line":"_get_from_shards."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":17,"id":"0aa9766f_034c8ad5","line":48,"updated":"2023-11-29 21:56:41.000000000","message":"I think this is like the 3rd thing this change is doing.\n\n * be consistent in making Namespace objects from resp/cache\n * don\u0027t request ShardRanges when Namespaces would work\n * clean out nonsensical backend req header form client responses\n \nI guess \"refactor _get_from_shards\" could be a 4th, but seems related to 1 \u0026 2","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":"A bug is fixed in _get_from_shards: it now always sets the"},{"line_number":47,"context_line":"X-Backend-Record-Type header to \u0027object\u0027, rather than sometimes"},{"line_number":48,"context_line":"leaving the value as \u0027shard\u0027."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"Some unit tests are also refactored to be more realistic when mocking"},{"line_number":51,"context_line":"_get_from_shards."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":17,"id":"76630de2_fb282692","line":48,"in_reply_to":"0aa9766f_034c8ad5","updated":"2023-11-30 15:43:06.000000000","message":"Done","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"ab18e863_1998b0ba","updated":"2023-11-10 19:41:06.000000000","message":"I think this is spking out STRONG in a direction that we definately need to pusue -but I don\u0027t think I got to the part that makes all the container-server namespace behaviors make sense.  I think there might be a couple of things going on in this patch at once - but I\u0027m pretty sure I love them all!  ... but didn\u0027t get them all loaded up in my head this afternoon fast enough to learn what I was hoping to undrestand about state of the pre-reqs for the release next week.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"965cae9a1dc2508b5055ffeabdf7b778c550b49d","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":3,"id":"6e59704f_0c482ada","in_reply_to":"ab18e863_1998b0ba","updated":"2023-11-13 05:45:10.000000000","message":"For the release next week, we are still good to carry all three patches(container namespace 890470, proxy change 895602, and new metrics) in the chain which we already are carrying. Following the discussions we had offline, I feel that\u0027ll be great if we can:\n  1. Squash the container new params support (marker, include, deleted) in this patch to the container namespace patch 890470.\n  2. Squash the proxy namespace related changes (which convert all ShardRange usages to Namespaces) to the proxy change patch 895602.\n  3. Leave others changes (which refactors GET to NOT read/write to memcache when handling a container GET) here as a standalone patch.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b2b7e69ff27ef1c916ba35696f8e967ce02c7ccd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"a6d44f91_7f7f2339","updated":"2023-11-13 16:14:58.000000000","message":"Thanks for the comments @Clay, you\u0027ve spurred me on to push harder on the bits I was already unhappy with! Working on next patchset...","commit_id":"d4f5c420f369b9a293db3bca22497e3895605f97"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"606a268f_e26226ea","updated":"2023-11-14 04:05:42.000000000","message":"Nice work on this Al. \n\nGoing to have a play with, but first going to rework the chain below it a little based on comments.","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e53338e6fb0496659af9b9402f19ce1b484fea20","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"5fed989e_1581fadd","updated":"2023-11-13 19:21:27.000000000","message":"Pushing what I have at end of my day.\n\nI feel better about this version - eliminated the re-serializing of namespaces read from cache and re-de-serializing them in _get_from_shards. It has left _GET_auto a longer method, but lots of that is comment and breaking it up felt like I\u0027d be returning 3-tuples of (resp, namespaces, cache_state).","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6793cf982b6494a24a3a0afe2c13a3ad5c2c6e49","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"14430f5f_18c7aeeb","updated":"2023-11-14 04:25:22.000000000","message":"just a rebase","commit_id":"d9a9780436b9b45cec1689551acb9248e1aa42a9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"ca87379e_caad0d16","updated":"2023-11-16 22:41:19.000000000","message":"i spent most of the day looking at this and did not get through it - please consider breaking it up\n\na lot of this would look perfectly reasonable even before the proxy cuts over to sending record_format\u003dnamespace, but that may not be practical unless we\u0027re willing to carry wip refactors.  OTOH, some of the new tests might just be hardening of existing correctness vs behavior fixes introduced as part of this refactor.\n\nI love everything you\u0027re trying to do here - if I had more hours I might be able to get it all loaded in my head and say it\u0027s good to go and we could ship it.  But it\u0027s tricky and I\u0027m dense.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"03f8a375_eafeb8e0","updated":"2023-11-19 00:57:26.000000000","message":"LGTM. No +1 yet, since I am not sure what does it mean for the below line change:\nhttps://review.opendev.org/c/openstack/swift/+/900350/11/test/unit/proxy/controllers/test_container.py#1240","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eac2c0a866f10723334ad06c0c42aa6296d1b18a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"188761be_78bfe40e","updated":"2023-11-20 14:47:05.000000000","message":"copied some of Clays comments to https://review.opendev.org/c/openstack/swift/+/901335","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fa1a6041b5505459e4f67a6473bb020239823432","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"626bc1cd_04b2c017","updated":"2023-11-17 19:20:30.000000000","message":"thanks for reviews!\n\nI have broken the patch up to hopefully make it easier to digest, so now we have:\n\n- 2 pre-refactor patches:\n\n  - move _get_shard_ranges (move it, and the tests, separately from changing it in this patch)\n\n  - introduce some cache helpers for obj and container controllers to share\n\n- this patch to clarify namespace vs shard ranges\n\n- a final patch to simplify the container controller GET path\n\nI have not had time today to work through comments, but I\u0027ll get there 😊","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"c3620ce3_c1b26012","updated":"2023-11-29 21:56:41.000000000","message":"I didn\u0027t get through this whole change today, but I did have some meetings distracting me.  It is actually fairly managable given everything it\u0027s trying to accomplish.\n\nI would suggest moving the x-backend-record-type fix into the probe test change; other than that I think this is pretty well contained.\n\nI still need to grok some of the changes in test.unit.proxy.controllers.test_container - in particular there may be some pre-existing inconsistency with the stubbed responses and how a modern container server would response vis a vis the \u0027x-backend-ignore-name-filter: sharded\u0027 header.","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a8b539bbad3653e175a5cc4606c55a5ab5919bc6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"26b747f5_6fd87c59","updated":"2023-11-28 05:54:41.000000000","message":"LGTM","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":18,"id":"ef5deeff_93a5b5f0","updated":"2023-11-30 15:43:06.000000000","message":"I appreciate the reviews - there\u0027s a lot to wade through!\n\nI have pulled the X-Backend_record-Type fix out to the parent probe test patch.\n\nI have climbed down and proposed adding the account/container properties to Namespace https://review.opendev.org/c/openstack/swift/+/902308 (which can be squashed if OK).\n\nI\u0027d like to explore the issue @Jianjian found in the probe tests when reordering the proxy vs backend changes.","commit_id":"8bd68b0cfc7d63d26f8abe4c041f9673202033ee"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"17624fcaccb59d92b89c67c5b3f6a2af4236b312","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":18,"id":"2e076f3a_293bb186","updated":"2023-11-29 23:41:00.000000000","message":"oops, i rushed and messed up - I didn\u0027t mean to make this dependent on my sq to the probe test - I don\u0027t even know if we\u0027ll want to sq that.\n\nHopefully I didn\u0027t accidently merge in any dirty wip from when I was playing with this; I can revert the stack tomorrow after chatting with Al about how we want to proceed.","commit_id":"8bd68b0cfc7d63d26f8abe4c041f9673202033ee"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d3f745af5ed21230edd84eba02d53f1a88a42cef","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":21,"id":"735ef285_6bcd4927","updated":"2023-12-05 12:31:37.000000000","message":"moved the Namespace.name change to a pre-patch https://review.opendev.org/c/openstack/swift/+/902646","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"03a6cb99f27e548ae62980b7b32fb6d8c7aa9311","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":21,"id":"7e003446_b0207ad2","updated":"2023-12-04 16:39:40.000000000","message":"sorry I had on my list to look at the \"namespaces have a/c names\" patch but I guess I didn\u0027t get to it before you wanted to squash.\n\nIf the plan is still to squash this into \"proxy-uses-namespaces\" parent and everyone is happy with \"namespaces have a/c names\" why not put that bit at the front of the chain like we\u0027d done with the other proxy pre-factors?\n\nIIRC the next step for me on this chagne is to look at the proxy.controller.test_container changes.","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"69255e1e707ef5ee3d617e20900aeef4f0daafe7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"13c5e6ec_d54f145e","updated":"2023-12-14 05:17:38.000000000","message":"This is basically a badly needed rename of functions from shard_range to namespaces and finally removing the \"we can remove this once get_namespaces support\". So nice one!\n\nAlso I know the container controller get\u0027s reworked a bunch in the next patchset. So let\u0027s move on to that one 😊","commit_id":"9360a785c5936ea4af627cc3426cb3f79ca0577d"}],"swift/common/utils/__init__.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"03a6cb99f27e548ae62980b7b32fb6d8c7aa9311","unresolved":true,"context_lines":[{"line_number":5280,"context_line":"            raise ValueError("},{"line_number":5281,"context_line":"                \"Name must be of the form \u0027\u003caccount\u003e/\u003ccontainer\u003e\u0027, got %r\" %"},{"line_number":5282,"context_line":"                name)"},{"line_number":5283,"context_line":"        self._account, self._container \u003d name.split(\u0027/\u0027)"},{"line_number":5284,"context_line":""},{"line_number":5285,"context_line":"    @property"},{"line_number":5286,"context_line":"    def timestamp(self):"}],"source_content_type":"text/x-python","patch_set":21,"id":"1ad7f54b_f5eb6295","side":"PARENT","line":5283,"updated":"2023-12-04 16:39:40.000000000","message":"ok so previously ShardRange stored the values split up and now it will store them together.  The interface appears the same regarldess - is one way or the other obviously better?  Consistency seems reasonably desierable, unless there\u0027s a reason for them to be different.","commit_id":"b1c2dc1d6da2c0ae2ba6f128c7cf3b21c9f09fb5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"18d2f1fff47fe4b4711272ecad469d0bc96a23b8","unresolved":true,"context_lines":[{"line_number":5280,"context_line":"            raise ValueError("},{"line_number":5281,"context_line":"                \"Name must be of the form \u0027\u003caccount\u003e/\u003ccontainer\u003e\u0027, got %r\" %"},{"line_number":5282,"context_line":"                name)"},{"line_number":5283,"context_line":"        self._account, self._container \u003d name.split(\u0027/\u0027)"},{"line_number":5284,"context_line":""},{"line_number":5285,"context_line":"    @property"},{"line_number":5286,"context_line":"    def timestamp(self):"}],"source_content_type":"text/x-python","patch_set":21,"id":"0f392a34_6e48764c","side":"PARENT","line":5283,"in_reply_to":"1ad7f54b_f5eb6295","updated":"2023-12-04 18:51:22.000000000","message":"great question! TBH I had been driven by \u0027smallest number of slots\u0027 since Jianjian had previously reported that made some difference.\n\nThe relative cost of the split depends on how often we instantiate the class vs how often we access account and container: I benchmarked splitting to two attributes in the name setter vs splitting from one attr in the account \u0026 container getter: if we get account and container for every instance we create, splitting in the setter is best. But I think we construct more instances than we access, in which case splitting in the getters is better.\n\nhttps://paste.openstack.org/show/b87LwdHo1xyeoHyoKO7I/","commit_id":"b1c2dc1d6da2c0ae2ba6f128c7cf3b21c9f09fb5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"03a6cb99f27e548ae62980b7b32fb6d8c7aa9311","unresolved":true,"context_lines":[{"line_number":4697,"context_line":""},{"line_number":4698,"context_line":"    @container.setter"},{"line_number":4699,"context_line":"    def container(self, value):"},{"line_number":4700,"context_line":"        self.name \u003d \u0027%s/%s\u0027 % (self.account, value)"},{"line_number":4701,"context_line":""},{"line_number":4702,"context_line":"    @property"},{"line_number":4703,"context_line":"    def name(self):"}],"source_content_type":"text/x-python","patch_set":21,"id":"7805457c_494d5e86","line":4700,"updated":"2023-12-04 16:39:40.000000000","message":"these would be easier to add later than to remove; do we want these properties to be modifiable for some reason?","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"18d2f1fff47fe4b4711272ecad469d0bc96a23b8","unresolved":true,"context_lines":[{"line_number":4697,"context_line":""},{"line_number":4698,"context_line":"    @container.setter"},{"line_number":4699,"context_line":"    def container(self, value):"},{"line_number":4700,"context_line":"        self.name \u003d \u0027%s/%s\u0027 % (self.account, value)"},{"line_number":4701,"context_line":""},{"line_number":4702,"context_line":"    @property"},{"line_number":4703,"context_line":"    def name(self):"}],"source_content_type":"text/x-python","patch_set":21,"id":"e4788c90_28f36bd5","line":4700,"in_reply_to":"7805457c_494d5e86","updated":"2023-12-04 18:51:22.000000000","message":"I don\u0027t think we need them to be settable (except in test maybe) but they have always been \"publicly\" settable - it was only recently that account and container became properties https://review.opendev.org/c/openstack/swift/+/899993/4/swift/common/utils/__init__.py - prior to that they were public attributes.","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"03a6cb99f27e548ae62980b7b32fb6d8c7aa9311","unresolved":true,"context_lines":[{"line_number":4706,"context_line":"    @name.setter"},{"line_number":4707,"context_line":"    def name(self, name):"},{"line_number":4708,"context_line":"        name \u003d self._encode(name)"},{"line_number":4709,"context_line":"        if not name or len(name.split(\u0027/\u0027)) !\u003d 2 or not all(name.split(\u0027/\u0027)):"},{"line_number":4710,"context_line":"            raise ValueError("},{"line_number":4711,"context_line":"                \"Name must be of the form \u0027\u003caccount\u003e/\u003ccontainer\u003e\u0027, got %r\" %"},{"line_number":4712,"context_line":"                name)"}],"source_content_type":"text/x-python","patch_set":21,"id":"dfb64565_1562b419","line":4709,"updated":"2023-12-04 16:39:40.000000000","message":"I think I\u0027d be ok skipping this verification if proxy handling of namespaces (for caching0 has been performance sensitive code-path that Jian has been carefully optimizing.","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"03a6cb99f27e548ae62980b7b32fb6d8c7aa9311","unresolved":true,"context_lines":[{"line_number":5038,"context_line":"    persisted."},{"line_number":5039,"context_line":""},{"line_number":5040,"context_line":"    :param name: the name of the shard range; this should take the form of a"},{"line_number":5041,"context_line":"        path to a container i.e. \u003caccount_name\u003e/\u003ccontainer_name\u003e."},{"line_number":5042,"context_line":"    :param timestamp: a timestamp that represents the time at which the"},{"line_number":5043,"context_line":"        shard range\u0027s ``lower``, ``upper`` or ``deleted`` attributes were"},{"line_number":5044,"context_line":"        last modified."}],"source_content_type":"text/x-python","patch_set":21,"id":"0128c2b4_d157d252","line":5041,"updated":"2023-12-04 16:39:40.000000000","message":"seems consistent with the doco update on the Namespace","commit_id":"8212898c56041425d373364a36108cdd2adda343"}],"swift/container/backend.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":1716,"context_line":"        :param fill_gaps: if True, insert a modified copy of own shard range to"},{"line_number":1717,"context_line":"            fill any gap between the end of any found shard ranges and the"},{"line_number":1718,"context_line":"            upper bound of own shard range. Gaps enclosed within the found"},{"line_number":1719,"context_line":"            shard ranges are not filled."},{"line_number":1720,"context_line":"        :return: a list of Namespace objects."},{"line_number":1721,"context_line":"        \"\"\""},{"line_number":1722,"context_line":"        if includes is None and (marker \u003d\u003d Namespace.MAX"}],"source_content_type":"text/x-python","patch_set":5,"id":"bd61c339_5add8819","line":1719,"updated":"2023-11-14 04:05:42.000000000","message":"Obviously include params for marker, end_marker, includes and reverse.","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":1716,"context_line":"        :param fill_gaps: if True, insert a modified copy of own shard range to"},{"line_number":1717,"context_line":"            fill any gap between the end of any found shard ranges and the"},{"line_number":1718,"context_line":"            upper bound of own shard range. Gaps enclosed within the found"},{"line_number":1719,"context_line":"            shard ranges are not filled."},{"line_number":1720,"context_line":"        :return: a list of Namespace objects."},{"line_number":1721,"context_line":"        \"\"\""},{"line_number":1722,"context_line":"        if includes is None and (marker \u003d\u003d Namespace.MAX"}],"source_content_type":"text/x-python","patch_set":5,"id":"19baf92e_7c9743ee","line":1719,"in_reply_to":"bd61c339_5add8819","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"}],"swift/container/server.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":792,"context_line":"        # This will allow proxy server who is going to retrieve Namespace"},{"line_number":793,"context_line":"        # to talk to older version of container servers who don\u0027t support"},{"line_number":794,"context_line":"        # Namespace yet during upgrade."},{"line_number":795,"context_line":"        # Note: there is no support of \u0027includes\u0027, \u0027marker\u0027, \u0027end_marker\u0027"},{"line_number":796,"context_line":"        # or \u0027reverse\u0027 for \u0027namespace\u0027 GET."},{"line_number":797,"context_line":"        if shard_format \u003d\u003d \u0027namespace\u0027:"},{"line_number":798,"context_line":"            override_deleted \u003d False"},{"line_number":799,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":5,"id":"989607e8_04fe3301","line":796,"range":{"start_line":795,"start_character":8,"end_line":796,"end_character":43},"updated":"2023-11-14 04:05:42.000000000","message":"We don\u0027t need this comment anymore","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":792,"context_line":"        # This will allow proxy server who is going to retrieve Namespace"},{"line_number":793,"context_line":"        # to talk to older version of container servers who don\u0027t support"},{"line_number":794,"context_line":"        # Namespace yet during upgrade."},{"line_number":795,"context_line":"        # Note: there is no support of \u0027includes\u0027, \u0027marker\u0027, \u0027end_marker\u0027"},{"line_number":796,"context_line":"        # or \u0027reverse\u0027 for \u0027namespace\u0027 GET."},{"line_number":797,"context_line":"        if shard_format \u003d\u003d \u0027namespace\u0027:"},{"line_number":798,"context_line":"            override_deleted \u003d False"},{"line_number":799,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":5,"id":"56c29d67_f3069d44","line":796,"range":{"start_line":795,"start_character":8,"end_line":796,"end_character":43},"in_reply_to":"989607e8_04fe3301","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":2407,"context_line":"            # ShardRanges, but that\u0027s ok: namespaces just need \u0027name\u0027, \u0027lower\u0027"},{"line_number":2408,"context_line":"            # and \u0027upper\u0027 keys."},{"line_number":2409,"context_line":"            return [Namespace(data[\u0027name\u0027], data[\u0027lower\u0027], data[\u0027upper\u0027])"},{"line_number":2410,"context_line":"                    for data in listing]"},{"line_number":2411,"context_line":"        except (ValueError, TypeError, KeyError) as err:"},{"line_number":2412,"context_line":"            self.logger.error("},{"line_number":2413,"context_line":"                \"Failed to get namespaces from %s: invalid data: %r\","}],"source_content_type":"text/x-python","patch_set":3,"id":"9271423f_c454f142","line":2410,"updated":"2023-11-09 18:12:54.000000000","message":"this should be the one place where we deal with legacy container servers sending us shard range dicts","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":2407,"context_line":"            # ShardRanges, but that\u0027s ok: namespaces just need \u0027name\u0027, \u0027lower\u0027"},{"line_number":2408,"context_line":"            # and \u0027upper\u0027 keys."},{"line_number":2409,"context_line":"            return [Namespace(data[\u0027name\u0027], data[\u0027lower\u0027], data[\u0027upper\u0027])"},{"line_number":2410,"context_line":"                    for data in listing]"},{"line_number":2411,"context_line":"        except (ValueError, TypeError, KeyError) as err:"},{"line_number":2412,"context_line":"            self.logger.error("},{"line_number":2413,"context_line":"                \"Failed to get namespaces from %s: invalid data: %r\","}],"source_content_type":"text/x-python","patch_set":3,"id":"fcf48e3a_695c6742","line":2410,"in_reply_to":"9271423f_c454f142","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":2414,"context_line":"                req.path_qs, err)"},{"line_number":2415,"context_line":"            return None"},{"line_number":2416,"context_line":""},{"line_number":2417,"context_line":"    def _get_namespaces("},{"line_number":2418,"context_line":"            self, req, account, container, includes\u003dNone, states\u003dNone):"},{"line_number":2419,"context_line":"        \"\"\""},{"line_number":2420,"context_line":"        Fetch shard ranges from given `account/container`. If `includes` is"}],"source_content_type":"text/x-python","patch_set":3,"id":"b2cd08c3_f27a2231","line":2417,"updated":"2023-11-09 18:12:54.000000000","message":"I think this is only used by obj.py to get updating states namespaces - maybe we should move it/rename it. I keep getting confused by seeing it here.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":2414,"context_line":"                req.path_qs, err)"},{"line_number":2415,"context_line":"            return None"},{"line_number":2416,"context_line":""},{"line_number":2417,"context_line":"    def _get_namespaces("},{"line_number":2418,"context_line":"            self, req, account, container, includes\u003dNone, states\u003dNone):"},{"line_number":2419,"context_line":"        \"\"\""},{"line_number":2420,"context_line":"        Fetch shard ranges from given `account/container`. If `includes` is"}],"source_content_type":"text/x-python","patch_set":3,"id":"d7c76357_f670c530","line":2417,"in_reply_to":"59e913ae_e084ff7d","updated":"2023-11-16 22:41:19.000000000","message":"right, so this is just _get_updating_namespaces_from_backend()\n\nthere\u0027s no use-case for this method to support record-type auto, if the root turns out to be unsharded then update_shard is None and _get_update_target just sends the object update to the root.\n\n... and checking/setting cache is handled in _get_update_shard directly","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":2414,"context_line":"                req.path_qs, err)"},{"line_number":2415,"context_line":"            return None"},{"line_number":2416,"context_line":""},{"line_number":2417,"context_line":"    def _get_namespaces("},{"line_number":2418,"context_line":"            self, req, account, container, includes\u003dNone, states\u003dNone):"},{"line_number":2419,"context_line":"        \"\"\""},{"line_number":2420,"context_line":"        Fetch shard ranges from given `account/container`. If `includes` is"}],"source_content_type":"text/x-python","patch_set":3,"id":"59e913ae_e084ff7d","line":2417,"in_reply_to":"b2cd08c3_f27a2231","updated":"2023-11-14 04:05:42.000000000","message":"+1 I do the same thing.. and I always seem to be double checking that it\u0027s only used by just the obj path.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":2414,"context_line":"                req.path_qs, err)"},{"line_number":2415,"context_line":"            return None"},{"line_number":2416,"context_line":""},{"line_number":2417,"context_line":"    def _get_namespaces("},{"line_number":2418,"context_line":"            self, req, account, container, includes\u003dNone, states\u003dNone):"},{"line_number":2419,"context_line":"        \"\"\""},{"line_number":2420,"context_line":"        Fetch shard ranges from given `account/container`. If `includes` is"}],"source_content_type":"text/x-python","patch_set":3,"id":"29aa501a_5a74599e","line":2417,"in_reply_to":"d7c76357_f670c530","updated":"2023-11-20 14:38:32.000000000","message":"relocated _get_shard_ranges to obj.py in https://review.opendev.org/c/openstack/swift/+/901334/2?usp\u003drelated-change","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":953,"context_line":"            cache_state \u003d \u0027set\u0027"},{"line_number":954,"context_line":"        except MemcacheConnectionError:"},{"line_number":955,"context_line":"            cache_state \u003d \u0027set_error\u0027"},{"line_number":956,"context_line":"    return cache_state"},{"line_number":957,"context_line":""},{"line_number":958,"context_line":""},{"line_number":959,"context_line":"def _prepare_pre_auth_info_request(env, path, swift_source):"}],"source_content_type":"text/x-python","patch_set":9,"id":"8058a279_a87b7321","line":956,"updated":"2023-11-16 22:41:19.000000000","message":"this is starting to unify the memcache access for listing/updating namespaces between obj and container controllers.\n\nI\u0027m not sure how much it could be separated from the container controller GET path refactor, or if that would be sufficiently valuable in scoping this diff.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":953,"context_line":"            cache_state \u003d \u0027set\u0027"},{"line_number":954,"context_line":"        except MemcacheConnectionError:"},{"line_number":955,"context_line":"            cache_state \u003d \u0027set_error\u0027"},{"line_number":956,"context_line":"    return cache_state"},{"line_number":957,"context_line":""},{"line_number":958,"context_line":""},{"line_number":959,"context_line":"def _prepare_pre_auth_info_request(env, path, swift_source):"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f965be5_733517e9","line":956,"in_reply_to":"8058a279_a87b7321","updated":"2023-11-20 14:38:32.000000000","message":"moved to https://review.opendev.org/c/openstack/swift/+/901333/1?usp\u003drelated-change","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":2456,"context_line":"            \u0027Get listing from %s %s\u0027 % (subreq.path_qs, headers))"},{"line_number":2457,"context_line":"        response \u003d self.app.handle_request(subreq)"},{"line_number":2458,"context_line":"        data \u003d self._parse_listing_response(req, response)"},{"line_number":2459,"context_line":"        return data, response"},{"line_number":2460,"context_line":""},{"line_number":2461,"context_line":"    def _parse_namespaces(self, req, listing, response):"},{"line_number":2462,"context_line":"        if listing is None:"}],"source_content_type":"text/x-python","patch_set":9,"id":"af077b31_a08fc2de","line":2459,"updated":"2023-11-16 22:41:19.000000000","message":"this is used in this module by get_namespaces and also in ContainerController._get_from_shards\n\nIn _get_from_shards we target a shard account/container, typically with record_type\u003dauto, but sometimes record_type\u003dobject to avoid possible loops - we NEVER expect a cached response\n\nI think get_namespaces is the true \"public\" interface, needed by ObjectController for object PUT/POST/DELETE, it\u0027s a different state and also never from cache.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":2456,"context_line":"            \u0027Get listing from %s %s\u0027 % (subreq.path_qs, headers))"},{"line_number":2457,"context_line":"        response \u003d self.app.handle_request(subreq)"},{"line_number":2458,"context_line":"        data \u003d self._parse_listing_response(req, response)"},{"line_number":2459,"context_line":"        return data, response"},{"line_number":2460,"context_line":""},{"line_number":2461,"context_line":"    def _parse_namespaces(self, req, listing, response):"},{"line_number":2462,"context_line":"        if listing is None:"}],"source_content_type":"text/x-python","patch_set":9,"id":"babaa0b6_5d8337ab","line":2459,"in_reply_to":"af077b31_a08fc2de","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":44,"context_line":"from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \\"},{"line_number":45,"context_line":"    public, split_path, list_from_csv, GreenthreadSafeIterator, \\"},{"line_number":46,"context_line":"    GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \\"},{"line_number":47,"context_line":"    document_iters_to_http_response_body, ShardRange, cache_from_env, \\"},{"line_number":48,"context_line":"    CooperativeIterator, NamespaceBoundList"},{"line_number":49,"context_line":"from swift.common.bufferedhttp import http_connect"},{"line_number":50,"context_line":"from swift.common import constraints"}],"source_content_type":"text/x-python","patch_set":17,"id":"602fa2a2_98aefd4a","side":"PARENT","line":47,"updated":"2023-11-29 21:56:41.000000000","message":"no more ShardRange!  seems consisitent with what the patch is going for.","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":44,"context_line":"from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \\"},{"line_number":45,"context_line":"    public, split_path, list_from_csv, GreenthreadSafeIterator, \\"},{"line_number":46,"context_line":"    GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \\"},{"line_number":47,"context_line":"    document_iters_to_http_response_body, ShardRange, cache_from_env, \\"},{"line_number":48,"context_line":"    CooperativeIterator, NamespaceBoundList"},{"line_number":49,"context_line":"from swift.common.bufferedhttp import http_connect"},{"line_number":50,"context_line":"from swift.common import constraints"}],"source_content_type":"text/x-python","patch_set":17,"id":"fc7e185a_9999a0f8","side":"PARENT","line":47,"in_reply_to":"602fa2a2_98aefd4a","updated":"2023-11-30 15:43:06.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":2473,"context_line":""},{"line_number":2474,"context_line":"        try:"},{"line_number":2475,"context_line":"            # Note: a legacy container-server could return a list of"},{"line_number":2476,"context_line":"            # ShardRanges, but that\u0027s ok: namespaces just need \u0027name\u0027, \u0027lower\u0027"},{"line_number":2477,"context_line":"            # and \u0027upper\u0027 keys. If we ever need to know we can look for a"},{"line_number":2478,"context_line":"            # \u0027x-backend-record-shard-format\u0027 header from newer container"},{"line_number":2479,"context_line":"            # servers."}],"source_content_type":"text/x-python","patch_set":17,"id":"209182d0_6542bbfb","line":2476,"updated":"2023-11-29 21:56:41.000000000","message":"\"ShardRanges\" shows up in greps, but it\u0027s just a comment saying shard-format: full response is ok here","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":2473,"context_line":""},{"line_number":2474,"context_line":"        try:"},{"line_number":2475,"context_line":"            # Note: a legacy container-server could return a list of"},{"line_number":2476,"context_line":"            # ShardRanges, but that\u0027s ok: namespaces just need \u0027name\u0027, \u0027lower\u0027"},{"line_number":2477,"context_line":"            # and \u0027upper\u0027 keys. If we ever need to know we can look for a"},{"line_number":2478,"context_line":"            # \u0027x-backend-record-shard-format\u0027 header from newer container"},{"line_number":2479,"context_line":"            # servers."}],"source_content_type":"text/x-python","patch_set":17,"id":"61fa1f3e_cb6bf718","line":2476,"in_reply_to":"209182d0_6542bbfb","updated":"2023-11-30 15:43:06.000000000","message":"Acknowledged","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":230,"context_line":"            return None"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        ns_bound_list \u003d NamespaceBoundList.parse(backend_shard_ranges)"},{"line_number":233,"context_line":"        if resp.headers.get(\u0027x-backend-sharding-state\u0027) \u003d\u003d \u0027sharded\u0027:"},{"line_number":234,"context_line":"            # cache in infocache even if no shard ranges returned; this"},{"line_number":235,"context_line":"            # is unexpected but use that result for this request"},{"line_number":236,"context_line":"            infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"}],"source_content_type":"text/x-python","patch_set":3,"id":"8ee7d508_521f2b9b","side":"PARENT","line":233,"updated":"2023-11-10 19:41:06.000000000","message":"some of the s/shard-range/namespace/ changes clutter up this refactor a little - there\u0027s a (small? intentional?) behavior change here beyond renaming some variables I think.","commit_id":"9040ca6b1bae0fcc0752b7e88c5ffb81be0311ca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":230,"context_line":"            return None"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        ns_bound_list \u003d NamespaceBoundList.parse(backend_shard_ranges)"},{"line_number":233,"context_line":"        if resp.headers.get(\u0027x-backend-sharding-state\u0027) \u003d\u003d \u0027sharded\u0027:"},{"line_number":234,"context_line":"            # cache in infocache even if no shard ranges returned; this"},{"line_number":235,"context_line":"            # is unexpected but use that result for this request"},{"line_number":236,"context_line":"            infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"}],"source_content_type":"text/x-python","patch_set":3,"id":"ca552971_245d8706","side":"PARENT","line":233,"in_reply_to":"8ee7d508_521f2b9b","updated":"2023-11-16 22:41:19.000000000","message":"Ack","commit_id":"9040ca6b1bae0fcc0752b7e88c5ffb81be0311ca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":258,"context_line":"        If the response headers indicate that the response body contains a"},{"line_number":259,"context_line":"        complete, unfiltered, list of namespaces for a sharded container then"},{"line_number":260,"context_line":"        the response body will be read, transformed to a"},{"line_number":261,"context_line":"        ``NamespaceBoundsList`` and cached. A new response body will be"},{"line_number":262,"context_line":"        synthesised containing a list of namespace dicts, filtered according to"},{"line_number":263,"context_line":"        the original request\u0027s ``marker``, ``end_marker``, ``includes`` and"},{"line_number":264,"context_line":"        ``reverse`` params."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        :param req: an instance of ``swob.Request``."},{"line_number":267,"context_line":"        :return: an instance of ``swob.Response``."}],"source_content_type":"text/x-python","patch_set":3,"id":"001caaf2_82658f49","line":264,"range":{"start_line":261,"start_character":44,"end_line":264,"end_character":27},"updated":"2023-11-10 19:41:06.000000000","message":"WAT?  The first part sounded great.  I was hoping that would just return and be like \"I know you didn\u0027t know what you would get: here\u0027s your namespaces - you\u0027re welcome\"\n\nwho wants filtered namespaces?","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b2b7e69ff27ef1c916ba35696f8e967ce02c7ccd","unresolved":true,"context_lines":[{"line_number":258,"context_line":"        If the response headers indicate that the response body contains a"},{"line_number":259,"context_line":"        complete, unfiltered, list of namespaces for a sharded container then"},{"line_number":260,"context_line":"        the response body will be read, transformed to a"},{"line_number":261,"context_line":"        ``NamespaceBoundsList`` and cached. A new response body will be"},{"line_number":262,"context_line":"        synthesised containing a list of namespace dicts, filtered according to"},{"line_number":263,"context_line":"        the original request\u0027s ``marker``, ``end_marker``, ``includes`` and"},{"line_number":264,"context_line":"        ``reverse`` params."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        :param req: an instance of ``swob.Request``."},{"line_number":267,"context_line":"        :return: an instance of ``swob.Response``."}],"source_content_type":"text/x-python","patch_set":3,"id":"c96f4e58_24473168","line":264,"range":{"start_line":261,"start_character":44,"end_line":264,"end_character":27},"in_reply_to":"001caaf2_82658f49","updated":"2023-11-13 16:14:58.000000000","message":"_get_from_shards expects a filtered list of shards\n\nI\u0027m more bothered about the fact that we re-serialize to a resp body only for that to be deserialized in _get_from_shards. I\u0027d like to deserialize just once, and normalize to Namespaces just once - there\u0027s still a case when _get_from_shards is passed a list of ShardRanges (XXX in tests) :/","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":258,"context_line":"        If the response headers indicate that the response body contains a"},{"line_number":259,"context_line":"        complete, unfiltered, list of namespaces for a sharded container then"},{"line_number":260,"context_line":"        the response body will be read, transformed to a"},{"line_number":261,"context_line":"        ``NamespaceBoundsList`` and cached. A new response body will be"},{"line_number":262,"context_line":"        synthesised containing a list of namespace dicts, filtered according to"},{"line_number":263,"context_line":"        the original request\u0027s ``marker``, ``end_marker``, ``includes`` and"},{"line_number":264,"context_line":"        ``reverse`` params."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        :param req: an instance of ``swob.Request``."},{"line_number":267,"context_line":"        :return: an instance of ``swob.Response``."}],"source_content_type":"text/x-python","patch_set":3,"id":"6e96205a_986e0a5c","line":264,"range":{"start_line":261,"start_character":44,"end_line":264,"end_character":27},"in_reply_to":"c96f4e58_24473168","updated":"2023-11-16 22:41:19.000000000","message":"ok, _get_from_shards wants filtered namesapces - and it clearly gets them in _GET_auto whether they come from the made up cache response or a backend-auto response","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":343,"context_line":"                cache_state, resp)"},{"line_number":344,"context_line":""},{"line_number":345,"context_line":"    def _GET_using_cache(self, req, info):"},{"line_number":346,"context_line":"        # returns response with body of either objects or NamespaceBoundsList"},{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        # If the container state is \u0027sharded\u0027 then look for cached namespaces."},{"line_number":349,"context_line":"        # However, if X-Newest is true then we always fetch from the backend"}],"source_content_type":"text/x-python","patch_set":3,"id":"14ef2f06_fa278548","line":346,"range":{"start_line":346,"start_character":58,"end_line":346,"end_character":77},"updated":"2023-11-09 18:12:54.000000000","message":"wrong - should be list of Namespace dicts","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebe11d824b643b5c91546364850080fc7a59ab5b","unresolved":false,"context_lines":[{"line_number":343,"context_line":"                cache_state, resp)"},{"line_number":344,"context_line":""},{"line_number":345,"context_line":"    def _GET_using_cache(self, req, info):"},{"line_number":346,"context_line":"        # returns response with body of either objects or NamespaceBoundsList"},{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        # If the container state is \u0027sharded\u0027 then look for cached namespaces."},{"line_number":349,"context_line":"        # However, if X-Newest is true then we always fetch from the backend"}],"source_content_type":"text/x-python","patch_set":3,"id":"0d240226_aa253e76","line":346,"range":{"start_line":346,"start_character":58,"end_line":346,"end_character":77},"in_reply_to":"14ef2f06_fa278548","updated":"2023-11-22 12:39:23.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        # If the container state is \u0027sharded\u0027 then look for cached namespaces."},{"line_number":349,"context_line":"        # However, if X-Newest is true then we always fetch from the backend"},{"line_number":350,"context_line":"        # servers."},{"line_number":351,"context_line":"        headers \u003d headers_from_container_info(info)"},{"line_number":352,"context_line":"        if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":353,"context_line":"            cache_state \u003d \u0027force_skip\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"d01146e2_eb7af3ad","line":350,"updated":"2023-11-10 19:41:06.000000000","message":"GET_using_cache can optionaly NOT get-using-cache based on a paramter?  Can we make the param be `why_are_you_calling_me_then\u003d\u0027lawls\u0027`","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        # If the container state is \u0027sharded\u0027 then look for cached namespaces."},{"line_number":349,"context_line":"        # However, if X-Newest is true then we always fetch from the backend"},{"line_number":350,"context_line":"        # servers."},{"line_number":351,"context_line":"        headers \u003d headers_from_container_info(info)"},{"line_number":352,"context_line":"        if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":353,"context_line":"            cache_state \u003d \u0027force_skip\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"23c4bdef_4b2599c1","line":350,"in_reply_to":"715512fb_41327e8b","updated":"2023-11-16 22:41:19.000000000","message":"Ack","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b2b7e69ff27ef1c916ba35696f8e967ce02c7ccd","unresolved":true,"context_lines":[{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        # If the container state is \u0027sharded\u0027 then look for cached namespaces."},{"line_number":349,"context_line":"        # However, if X-Newest is true then we always fetch from the backend"},{"line_number":350,"context_line":"        # servers."},{"line_number":351,"context_line":"        headers \u003d headers_from_container_info(info)"},{"line_number":352,"context_line":"        if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":353,"context_line":"            cache_state \u003d \u0027force_skip\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"715512fb_41327e8b","line":350,"in_reply_to":"d01146e2_eb7af3ad","updated":"2023-11-13 16:14:58.000000000","message":"\u0027using\u0027 covers reading AND writing to cache\n\n...but, I want to rework this anyway","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":367,"context_line":"            # state in metadata indicates the container was unsharded."},{"line_number":368,"context_line":"            cache_state \u003d \u0027bypass\u0027"},{"line_number":369,"context_line":"        # The request was not fulfilled from cache so send to backend server."},{"line_number":370,"context_line":"        return self._get_namespaces_or_objects_from_backend(req), cache_state"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"    def _GET_auto(self, req):"},{"line_number":373,"context_line":"        # This is an object listing but the backend may be sharded. Instruct"}],"source_content_type":"text/x-python","patch_set":3,"id":"56f10d03_93c27ae5","line":370,"updated":"2023-11-10 19:41:06.000000000","message":"I think I expect:\n\n    resp \u003d _GET_using_cache()\n    if not resp:\n        resp \u003d _GET_using_backend()\n\nmaybe put that in a function _GET_using_cache_or_backend\n\nbut maybe I don\u0027t actually want a single function that tries to normalize something from memcache and something from backend into the same type?  Maybe a function like:  get_namespaces_from_cache_or_backend could ALWAYS return a NamespaceList, but not a swob.Response?","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":367,"context_line":"            # state in metadata indicates the container was unsharded."},{"line_number":368,"context_line":"            cache_state \u003d \u0027bypass\u0027"},{"line_number":369,"context_line":"        # The request was not fulfilled from cache so send to backend server."},{"line_number":370,"context_line":"        return self._get_namespaces_or_objects_from_backend(req), cache_state"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"    def _GET_auto(self, req):"},{"line_number":373,"context_line":"        # This is an object listing but the backend may be sharded. Instruct"}],"source_content_type":"text/x-python","patch_set":3,"id":"98dd84b9_1c6b1bb7","line":370,"in_reply_to":"56f10d03_93c27ae5","updated":"2023-11-16 22:41:19.000000000","message":"I think aside from BaseController._get_namesapces we don\u0027t have a use-case for method than normalizes a cache response into a backend response - and it returns the backend response mainly for cache-skip metrics.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebe11d824b643b5c91546364850080fc7a59ab5b","unresolved":false,"context_lines":[{"line_number":367,"context_line":"            # state in metadata indicates the container was unsharded."},{"line_number":368,"context_line":"            cache_state \u003d \u0027bypass\u0027"},{"line_number":369,"context_line":"        # The request was not fulfilled from cache so send to backend server."},{"line_number":370,"context_line":"        return self._get_namespaces_or_objects_from_backend(req), cache_state"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"    def _GET_auto(self, req):"},{"line_number":373,"context_line":"        # This is an object listing but the backend may be sharded. Instruct"}],"source_content_type":"text/x-python","patch_set":3,"id":"ac57fc2b_581eb8ea","line":370,"in_reply_to":"98dd84b9_1c6b1bb7","updated":"2023-11-22 12:39:23.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":369,"context_line":"        # The request was not fulfilled from cache so send to backend server."},{"line_number":370,"context_line":"        return self._get_namespaces_or_objects_from_backend(req), cache_state"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"    def _GET_auto(self, req):"},{"line_number":373,"context_line":"        # This is an object listing but the backend may be sharded. Instruct"},{"line_number":374,"context_line":"        # the backend server to \u0027automatically\u0027 return namespaces of shards in"},{"line_number":375,"context_line":"        # a \u0027listing\u0027 state if the container is sharded, and that the more"}],"source_content_type":"text/x-python","patch_set":3,"id":"d34e380f_99f5384b","line":372,"updated":"2023-11-10 19:41:06.000000000","message":"yes!  this is th method I was hoping for!\n\n    objects \u003d _GET_listing_auto(req)\n\n    def _GET_listing_auto(req):\n        namespaces \u003d from_cache()\n        if not namespaces:\n            resp \u003d who_knows_what_the_container_has()\n            if resp \u003d\u003d namespaces:\n               cache_them()\n            else:\n               return resp\n        return _get_objects(req, namespace\u003dnamespaces)  # N.B. namespace is cached","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":369,"context_line":"        # The request was not fulfilled from cache so send to backend server."},{"line_number":370,"context_line":"        return self._get_namespaces_or_objects_from_backend(req), cache_state"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"    def _GET_auto(self, req):"},{"line_number":373,"context_line":"        # This is an object listing but the backend may be sharded. Instruct"},{"line_number":374,"context_line":"        # the backend server to \u0027automatically\u0027 return namespaces of shards in"},{"line_number":375,"context_line":"        # a \u0027listing\u0027 state if the container is sharded, and that the more"}],"source_content_type":"text/x-python","patch_set":3,"id":"768109ba_db463a2b","line":372,"in_reply_to":"d34e380f_99f5384b","updated":"2023-11-16 22:41:19.000000000","message":"Ack","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":379,"context_line":"        req.headers[\u0027X-Backend-Record-Shard-Format\u0027] \u003d \u0027namespace\u0027"},{"line_number":380,"context_line":"        # \u0027x-backend-include-deleted\u0027 is not expected in \u0027auto\u0027 requests to the"},{"line_number":381,"context_line":"        # proxy (it\u0027s not supported for objects and is used by the sharder when"},{"line_number":382,"context_line":"        # explicitly fetching \u0027shard\u0027 record type), but we explicitly set it to"},{"line_number":383,"context_line":"        # false here just in case. A newer container server would ignore it"},{"line_number":384,"context_line":"        # when returning namespaces, but an older container server would"},{"line_number":385,"context_line":"        # include unwanted deleted shard range."},{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"98db5958_fc95737e","line":383,"range":{"start_line":382,"start_character":52,"end_line":383,"end_character":34},"updated":"2023-11-10 19:41:06.000000000","message":"*for clarity* \u003c3","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":379,"context_line":"        req.headers[\u0027X-Backend-Record-Shard-Format\u0027] \u003d \u0027namespace\u0027"},{"line_number":380,"context_line":"        # \u0027x-backend-include-deleted\u0027 is not expected in \u0027auto\u0027 requests to the"},{"line_number":381,"context_line":"        # proxy (it\u0027s not supported for objects and is used by the sharder when"},{"line_number":382,"context_line":"        # explicitly fetching \u0027shard\u0027 record type), but we explicitly set it to"},{"line_number":383,"context_line":"        # false here just in case. A newer container server would ignore it"},{"line_number":384,"context_line":"        # when returning namespaces, but an older container server would"},{"line_number":385,"context_line":"        # include unwanted deleted shard range."},{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"1de9910b_7bcc1800","line":383,"range":{"start_line":382,"start_character":52,"end_line":383,"end_character":34},"in_reply_to":"98db5958_fc95737e","updated":"2023-11-16 22:41:19.000000000","message":"Ack","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":382,"context_line":"        # explicitly fetching \u0027shard\u0027 record type), but we explicitly set it to"},{"line_number":383,"context_line":"        # false here just in case. A newer container server would ignore it"},{"line_number":384,"context_line":"        # when returning namespaces, but an older container server would"},{"line_number":385,"context_line":"        # include unwanted deleted shard range."},{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"9130a3a3_e11ad67f","line":385,"updated":"2023-11-10 19:41:06.000000000","message":"how much older?  is master caching deleted shard ranges for listings \u0026 updates!?","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":false,"context_lines":[{"line_number":382,"context_line":"        # explicitly fetching \u0027shard\u0027 record type), but we explicitly set it to"},{"line_number":383,"context_line":"        # false here just in case. A newer container server would ignore it"},{"line_number":384,"context_line":"        # when returning namespaces, but an older container server would"},{"line_number":385,"context_line":"        # include unwanted deleted shard range."},{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"8cdd8264_36900d6d","line":385,"in_reply_to":"5847eaab_118910e3","updated":"2023-11-29 21:56:41.000000000","message":"Acknowledged","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b2b7e69ff27ef1c916ba35696f8e967ce02c7ccd","unresolved":true,"context_lines":[{"line_number":382,"context_line":"        # explicitly fetching \u0027shard\u0027 record type), but we explicitly set it to"},{"line_number":383,"context_line":"        # false here just in case. A newer container server would ignore it"},{"line_number":384,"context_line":"        # when returning namespaces, but an older container server would"},{"line_number":385,"context_line":"        # include unwanted deleted shard range."},{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"a9ef331d_67133fbd","line":385,"in_reply_to":"9130a3a3_e11ad67f","updated":"2023-11-13 16:14:58.000000000","message":"master does not cache deleted shard ranges. On master, we don\u0027t use the caching path if X-Backend-Include-Deleted \u003d \u0027true\u0027. I\u0027m flipping the logic so that if we *are* using the caching path then we ensure X-Backend-Include-Deleted \u003d \u0027false\u0027 so that we\u0027re still safe talking to older container servers.\n\nIt\u0027s been on my mind that there are conditions that we might consider to be BadRequest if received from a client, such as X-Backend-Include-Deleted being sent when record_type !\u003d \u0027shard\u0027. But when it\u0027s a backend header I\u0027m not sure if we have precedence for raising BadRequest or just assuming our code is correct. I guess I\u0027m compromising on being defensive, and as a bonus the code is very explicit: cached listing do not have deleted shards.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":382,"context_line":"        # explicitly fetching \u0027shard\u0027 record type), but we explicitly set it to"},{"line_number":383,"context_line":"        # false here just in case. A newer container server would ignore it"},{"line_number":384,"context_line":"        # when returning namespaces, but an older container server would"},{"line_number":385,"context_line":"        # include unwanted deleted shard range."},{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"f6182dd6_e8c0d688","line":385,"in_reply_to":"a9ef331d_67133fbd","updated":"2023-11-16 22:41:19.000000000","message":"yeah I think a backend error on requests we don\u0027t *expect* is more trouble than it\u0027s worth\n\nI like being explicit about the params we send for the responses we want to cache.\n\nI still don\u0027t understand the concern with \"older container servers\" - i would assume `x-backend-include-deleted: true` is new and not the default behavior.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebe11d824b643b5c91546364850080fc7a59ab5b","unresolved":true,"context_lines":[{"line_number":382,"context_line":"        # explicitly fetching \u0027shard\u0027 record type), but we explicitly set it to"},{"line_number":383,"context_line":"        # false here just in case. A newer container server would ignore it"},{"line_number":384,"context_line":"        # when returning namespaces, but an older container server would"},{"line_number":385,"context_line":"        # include unwanted deleted shard range."},{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"5847eaab_118910e3","line":385,"in_reply_to":"f6182dd6_e8c0d688","updated":"2023-11-22 12:39:23.000000000","message":"An up-to-date container server would respond with 400 if we ask for format\u003dnamespace and send include-deleted\u003dtrue\n\nAn older container-server would return *shard ranges* with deleted included, which is bad. So I am being defensive and explicitly ensuring that include-deleted is false.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":389,"context_line":"        req.params \u003d params"},{"line_number":390,"context_line":""},{"line_number":391,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":392,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":3,"id":"85e10210_89530c92","line":389,"updated":"2023-11-09 18:12:54.000000000","message":"we get opinionated here - this path is building an object listing, so listing states is all that makes sense.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebe11d824b643b5c91546364850080fc7a59ab5b","unresolved":false,"context_lines":[{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":389,"context_line":"        req.params \u003d params"},{"line_number":390,"context_line":""},{"line_number":391,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":392,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":3,"id":"0d5a5478_6d8ee30c","line":389,"in_reply_to":"0f8e1d0b_9845d5d5","updated":"2023-11-22 12:39:23.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b2b7e69ff27ef1c916ba35696f8e967ce02c7ccd","unresolved":true,"context_lines":[{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":389,"context_line":"        req.params \u003d params"},{"line_number":390,"context_line":""},{"line_number":391,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":392,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":3,"id":"9327b773_c70a7595","line":389,"in_reply_to":"311f123a_80a9ac59","updated":"2023-11-13 16:14:58.000000000","message":"I don\u0027t mind renaming the method, but all container GETs are for listings of some sort?","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":389,"context_line":"        req.params \u003d params"},{"line_number":390,"context_line":""},{"line_number":391,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":392,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":3,"id":"311f123a_80a9ac59","line":389,"in_reply_to":"85e10210_89530c92","updated":"2023-11-10 19:41:06.000000000","message":"so... _GET_listing_auto()???","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":386,"context_line":"        req.headers[\u0027X-Backend-Include-Deleted\u0027] \u003d \u0027false\u0027"},{"line_number":387,"context_line":"        params \u003d req.params"},{"line_number":388,"context_line":"        params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":389,"context_line":"        req.params \u003d params"},{"line_number":390,"context_line":""},{"line_number":391,"context_line":"        # Only lookup container info from cache and skip the backend HEAD,"},{"line_number":392,"context_line":"        # since we are going to GET the backend container anyway."}],"source_content_type":"text/x-python","patch_set":3,"id":"0f8e1d0b_9845d5d5","line":389,"in_reply_to":"9327b773_c70a7595","updated":"2023-11-16 22:41:19.000000000","message":"I\u0027m definately starting to think that the public interface for ContainerController.GET has three distinct \"types of listings\"\n\nlisting of shard ranges (typically namespaces, sometimes from cache)\nbackend listing of objects (really only from shards to avoid loops)\n\"container listings\" (created automatically by combining requests of the above types, but often making backend requests of type \"auto\")","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":415,"context_line":"        # else: we got back objects"},{"line_number":416,"context_line":"        return resp"},{"line_number":417,"context_line":""},{"line_number":418,"context_line":"    def _get_or_head_pre_check(self, req):"},{"line_number":419,"context_line":"        ai \u003d self.account_info(self.account_name, req)"},{"line_number":420,"context_line":"        auto_account \u003d self.account_name.startswith("},{"line_number":421,"context_line":"            self.app.auto_create_account_prefix)"}],"source_content_type":"text/x-python","patch_set":3,"id":"950a5358_e6840ec7","line":418,"updated":"2023-11-09 18:12:54.000000000","message":"I\u0027m flipping and flopping between preferring these helpers that are called at the start and end of separate GET and HEAD methods, or retaining the GETorHEAD method that starts and ends with these code chunks.\n\nMaybe someone else will have a strong preference to swing the vote.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":415,"context_line":"        # else: we got back objects"},{"line_number":416,"context_line":"        return resp"},{"line_number":417,"context_line":""},{"line_number":418,"context_line":"    def _get_or_head_pre_check(self, req):"},{"line_number":419,"context_line":"        ai \u003d self.account_info(self.account_name, req)"},{"line_number":420,"context_line":"        auto_account \u003d self.account_name.startswith("},{"line_number":421,"context_line":"            self.app.auto_create_account_prefix)"}],"source_content_type":"text/x-python","patch_set":3,"id":"b4e9868b_dd196e9a","line":418,"in_reply_to":"950a5358_e6840ec7","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":457,"context_line":"            resp.headers[\u0027X-Container-Sharding\u0027] \u003d config_true_value("},{"line_number":458,"context_line":"                resp.headers.get(get_sys_meta_prefix(\u0027container\u0027) + \u0027Sharding\u0027,"},{"line_number":459,"context_line":"                                 \u0027False\u0027))"},{"line_number":460,"context_line":"        return resp"},{"line_number":461,"context_line":""},{"line_number":462,"context_line":"    @public"},{"line_number":463,"context_line":"    @delay_denial"}],"source_content_type":"text/x-python","patch_set":3,"id":"028da49c_34e90a49","line":460,"updated":"2023-11-10 19:41:06.000000000","message":"these pre or post check helpers are probably KEY!  kudos pulling out the trimmings into re-usable helpers!","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":457,"context_line":"            resp.headers[\u0027X-Container-Sharding\u0027] \u003d config_true_value("},{"line_number":458,"context_line":"                resp.headers.get(get_sys_meta_prefix(\u0027container\u0027) + \u0027Sharding\u0027,"},{"line_number":459,"context_line":"                                 \u0027False\u0027))"},{"line_number":460,"context_line":"        return resp"},{"line_number":461,"context_line":""},{"line_number":462,"context_line":"    @public"},{"line_number":463,"context_line":"    @delay_denial"}],"source_content_type":"text/x-python","patch_set":3,"id":"2c036bd0_5133bd54","line":460,"in_reply_to":"028da49c_34e90a49","updated":"2023-11-14 18:42:53.000000000","message":"Ack","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":491,"context_line":"            # ranges, and by the proxy itself when explicitly requesting"},{"line_number":492,"context_line":"            # objects while recursively building a listing from shards."},{"line_number":493,"context_line":"            # Note: shard record type could be namespace or full format"},{"line_number":494,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            # Requests that do not explicitly specify a record type, or specify"},{"line_number":497,"context_line":"            # \u0027auto\u0027, default to returning an object listing. The listing may"}],"source_content_type":"text/x-python","patch_set":3,"id":"7ff6e6c6_7519e133","line":494,"updated":"2023-11-10 19:41:06.000000000","message":"oh yes, VERY nice!  if the caller knows what they want we can go get it for them from the backend end.  super explicit - KUDOS!\n\nBut I assume even when we\u0027re returning record_type \u003d object we may still leverage cached namespaces for the listing and if they\u0027re not in cache when we get a response from the root we may cache them and make another request recursively into this method with type... auto?","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b2b7e69ff27ef1c916ba35696f8e967ce02c7ccd","unresolved":true,"context_lines":[{"line_number":491,"context_line":"            # ranges, and by the proxy itself when explicitly requesting"},{"line_number":492,"context_line":"            # objects while recursively building a listing from shards."},{"line_number":493,"context_line":"            # Note: shard record type could be namespace or full format"},{"line_number":494,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            # Requests that do not explicitly specify a record type, or specify"},{"line_number":497,"context_line":"            # \u0027auto\u0027, default to returning an object listing. The listing may"}],"source_content_type":"text/x-python","patch_set":3,"id":"de42af3f_123a9e78","line":494,"in_reply_to":"7ff6e6c6_7519e133","updated":"2023-11-13 16:14:58.000000000","message":"record_type \u003d\u003d objects in a request means no recursion, I want objects from *this* container (root, shard, regardless of its state), do not go to shards. We use it if a loop is detected in the shard recursion (e.g a gap-filling shard which loops back to the root) - \"this container gave me back shards, I listed the shards, but now I\u0027m back talking to *this same container* again so this time just *give me objects*\"\n\n\u0027auto\u0027 is where we want objects but if we get back shards then recurse into the shards until we have objects.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":491,"context_line":"            # ranges, and by the proxy itself when explicitly requesting"},{"line_number":492,"context_line":"            # objects while recursively building a listing from shards."},{"line_number":493,"context_line":"            # Note: shard record type could be namespace or full format"},{"line_number":494,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            # Requests that do not explicitly specify a record type, or specify"},{"line_number":497,"context_line":"            # \u0027auto\u0027, default to returning an object listing. The listing may"}],"source_content_type":"text/x-python","patch_set":3,"id":"dd1129f0_4b09756f","line":494,"in_reply_to":"de42af3f_123a9e78","updated":"2023-11-16 22:41:19.000000000","message":"ah yes, KEY!  \u0027objects\u0027 really really really means I want the list of objects from this database.  When coming into this method \"auto\" isn\u0027t merely a \"default\" it\u0027s an entirely different behavior.  the proxy-app can get \"shard\" for the sharder, it can get \"auto\" for clients (which means \"sharded-object-listing-using-recursion\") and it can alo get \"object\" which will NEVER recurse; obviously, it goes stright to _GETorHEAD_from_backend!","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":493,"context_line":"            # Note: shard record type could be namespace or full format"},{"line_number":494,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            # Requests that do not explicitly specify a record type, or specify"},{"line_number":497,"context_line":"            # \u0027auto\u0027, default to returning an object listing. The listing may"},{"line_number":498,"context_line":"            # be built from shards and may involve reading/writing namespaces"},{"line_number":499,"context_line":"            # in cache. This path is used for client requests and by the proxy"},{"line_number":500,"context_line":"            # itself while recursively building a listing from shards."}],"source_content_type":"text/x-python","patch_set":3,"id":"9251eedb_6c175250","line":497,"range":{"start_line":496,"start_character":14,"end_line":497,"end_character":60},"updated":"2023-11-10 19:41:06.000000000","message":"right, record_type is assumed to be object listing - we use auto to tell the container-server we want objects unless it\u0027s sharded\n\nDoes the proxy-server itself also have a record_type\u003dauto or do clients always get objects via the proxy and sharder maybe explicitly record_type\u003d\u0027shard\u0027 via internalclient?","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":493,"context_line":"            # Note: shard record type could be namespace or full format"},{"line_number":494,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            # Requests that do not explicitly specify a record type, or specify"},{"line_number":497,"context_line":"            # \u0027auto\u0027, default to returning an object listing. The listing may"},{"line_number":498,"context_line":"            # be built from shards and may involve reading/writing namespaces"},{"line_number":499,"context_line":"            # in cache. This path is used for client requests and by the proxy"},{"line_number":500,"context_line":"            # itself while recursively building a listing from shards."}],"source_content_type":"text/x-python","patch_set":3,"id":"a966c1c6_ee101a6c","line":497,"range":{"start_line":496,"start_character":14,"end_line":497,"end_character":60},"in_reply_to":"9067db8b_2e62b967","updated":"2023-11-16 22:41:19.000000000","message":"\u003e record_type is assumed to be object listing\n\n^ this is a confusing thing to say, i was confused\n\nthis method supports explict record_type object/shard to go stright to the backend and explict record_type auto (the default) which means \"use cached namespaces and recurision to build a list of objects from sharded databases\"\n\nObviously swift public API clients only ever expected or care about the latter; but internalclient users (and the proxy app itself!) will explictly call into this method with record_type in (\u0027object\u0027, \u0027shard\u0027) when wanting to get routed directly to _GETorHEAD_from_backend.  But also often times recurse in with type \"auto\" since a shard itself may be recently sharding and cached namespaces out-of-date.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b2b7e69ff27ef1c916ba35696f8e967ce02c7ccd","unresolved":true,"context_lines":[{"line_number":493,"context_line":"            # Note: shard record type could be namespace or full format"},{"line_number":494,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            # Requests that do not explicitly specify a record type, or specify"},{"line_number":497,"context_line":"            # \u0027auto\u0027, default to returning an object listing. The listing may"},{"line_number":498,"context_line":"            # be built from shards and may involve reading/writing namespaces"},{"line_number":499,"context_line":"            # in cache. This path is used for client requests and by the proxy"},{"line_number":500,"context_line":"            # itself while recursively building a listing from shards."}],"source_content_type":"text/x-python","patch_set":3,"id":"c9e6e12a_360bfcc6","line":497,"range":{"start_line":496,"start_character":14,"end_line":497,"end_character":60},"in_reply_to":"9251eedb_6c175250","updated":"2023-11-13 16:14:58.000000000","message":"On master I\u0027m not sure \u0027auto\u0027 is ever passed *in* to ContainerController.GET, just added to the backend requests.\n\nIn this patch I am adding a record_type\u003d\u003d\u0027auto\u0027 when _get_from_shards recurses into ContainerController.GET (line 599) just to be explicit - it isn\u0027t necessary because it\u0027s the default anyway, but it at least helped guide my brain to the _GET_auto path.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":493,"context_line":"            # Note: shard record type could be namespace or full format"},{"line_number":494,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            # Requests that do not explicitly specify a record type, or specify"},{"line_number":497,"context_line":"            # \u0027auto\u0027, default to returning an object listing. The listing may"},{"line_number":498,"context_line":"            # be built from shards and may involve reading/writing namespaces"},{"line_number":499,"context_line":"            # in cache. This path is used for client requests and by the proxy"},{"line_number":500,"context_line":"            # itself while recursively building a listing from shards."}],"source_content_type":"text/x-python","patch_set":3,"id":"9067db8b_2e62b967","line":497,"range":{"start_line":496,"start_character":14,"end_line":497,"end_character":60},"in_reply_to":"c9e6e12a_360bfcc6","updated":"2023-11-14 04:05:42.000000000","message":"ahem, line 555 ;)","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":532,"context_line":"        # therefore cannot use ShardRange.from_dict(), and the ShardRange"},{"line_number":533,"context_line":"        # instances constructed here will only have \u0027name\u0027, \u0027lower\u0027 and \u0027upper\u0027"},{"line_number":534,"context_line":"        # attributes set."},{"line_number":535,"context_line":"        # Ideally we would construct Namespace objects here, but later we use"},{"line_number":536,"context_line":"        # the ShardRange account and container properties to access parsed"},{"line_number":537,"context_line":"        # parts of the name."},{"line_number":538,"context_line":"        namespaces \u003d self._parse_namespaces("},{"line_number":539,"context_line":"            req, self._parse_listing_response(req, resp), resp)"},{"line_number":540,"context_line":"        if not namespaces:"}],"source_content_type":"text/x-python","patch_set":3,"id":"dbfb787f_31f10be2","line":537,"range":{"start_line":535,"start_character":10,"end_line":537,"end_character":28},"updated":"2023-11-09 18:12:54.000000000","message":"stale comment","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":532,"context_line":"        # therefore cannot use ShardRange.from_dict(), and the ShardRange"},{"line_number":533,"context_line":"        # instances constructed here will only have \u0027name\u0027, \u0027lower\u0027 and \u0027upper\u0027"},{"line_number":534,"context_line":"        # attributes set."},{"line_number":535,"context_line":"        # Ideally we would construct Namespace objects here, but later we use"},{"line_number":536,"context_line":"        # the ShardRange account and container properties to access parsed"},{"line_number":537,"context_line":"        # parts of the name."},{"line_number":538,"context_line":"        namespaces \u003d self._parse_namespaces("},{"line_number":539,"context_line":"            req, self._parse_listing_response(req, resp), resp)"},{"line_number":540,"context_line":"        if not namespaces:"}],"source_content_type":"text/x-python","patch_set":3,"id":"892884d3_68266dcf","line":537,"range":{"start_line":535,"start_character":10,"end_line":537,"end_character":28},"in_reply_to":"dbfb787f_31f10be2","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":536,"context_line":"        # the ShardRange account and container properties to access parsed"},{"line_number":537,"context_line":"        # parts of the name."},{"line_number":538,"context_line":"        namespaces \u003d self._parse_namespaces("},{"line_number":539,"context_line":"            req, self._parse_listing_response(req, resp), resp)"},{"line_number":540,"context_line":"        if not namespaces:"},{"line_number":541,"context_line":"            # can\u0027t find ranges or there was a problem getting the ranges. So"},{"line_number":542,"context_line":"            # return what we have."}],"source_content_type":"text/x-python","patch_set":3,"id":"b01f0102_890ecf0d","line":539,"updated":"2023-11-10 19:41:06.000000000","message":"maybe move this outside of this function and change the signature to:\n\n    objects \u003d get_from_shards(req, namespaces)\n    \nhttps://review.opendev.org/c/openstack/swift/+/900690 passes WAY more tests than I would expect.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":536,"context_line":"        # the ShardRange account and container properties to access parsed"},{"line_number":537,"context_line":"        # parts of the name."},{"line_number":538,"context_line":"        namespaces \u003d self._parse_namespaces("},{"line_number":539,"context_line":"            req, self._parse_listing_response(req, resp), resp)"},{"line_number":540,"context_line":"        if not namespaces:"},{"line_number":541,"context_line":"            # can\u0027t find ranges or there was a problem getting the ranges. So"},{"line_number":542,"context_line":"            # return what we have."}],"source_content_type":"text/x-python","patch_set":3,"id":"481d82d7_63659f59","line":539,"in_reply_to":"b01f0102_890ecf0d","updated":"2023-11-16 22:41:19.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":666,"context_line":"                break"},{"line_number":667,"context_line":""},{"line_number":668,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":669,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":670,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":671,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"},{"line_number":672,"context_line":"        if not constrained and len(objects) \u003c req_limit:"}],"source_content_type":"text/x-python","patch_set":3,"id":"3df52f2f_2e5b5653","line":669,"updated":"2023-11-10 19:41:06.000000000","message":"it\u0027s not exactly obvious to me why we\u0027re re-writing the passed in response rather than crafting a new one.  Something about the root container metadata?","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":true,"context_lines":[{"line_number":666,"context_line":"                break"},{"line_number":667,"context_line":""},{"line_number":668,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":669,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":670,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":671,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"},{"line_number":672,"context_line":"        if not constrained and len(objects) \u003c req_limit:"}],"source_content_type":"text/x-python","patch_set":3,"id":"5abcff29_18a954d7","line":669,"in_reply_to":"3df52f2f_2e5b5653","updated":"2023-11-14 18:42:53.000000000","message":"that, we\u0027re keeping all the headers and giving it a new body","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":false,"context_lines":[{"line_number":666,"context_line":"                break"},{"line_number":667,"context_line":""},{"line_number":668,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":669,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":670,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":671,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"},{"line_number":672,"context_line":"        if not constrained and len(objects) \u003c req_limit:"}],"source_content_type":"text/x-python","patch_set":3,"id":"d2149003_4ba07537","line":669,"in_reply_to":"5abcff29_18a954d7","updated":"2023-11-16 22:41:19.000000000","message":"Ack","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"dbb86be9d2d22e9a4b8c3751daa71acdb9fa6f36","unresolved":true,"context_lines":[{"line_number":690,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":691,"context_line":"        if aresp:"},{"line_number":692,"context_line":"            return aresp"},{"line_number":693,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":694,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":695,"context_line":""},{"line_number":696,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfdfddaf_8f9b1ffa","line":693,"updated":"2023-11-10 19:41:06.000000000","message":"that\u0027s nice and explict - and also a little less scary now that I think about it!","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":690,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":691,"context_line":"        if aresp:"},{"line_number":692,"context_line":"            return aresp"},{"line_number":693,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":694,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":695,"context_line":""},{"line_number":696,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":3,"id":"7183f6bb_9483fac1","line":693,"in_reply_to":"1fe65387_4bbcc13b","updated":"2023-11-16 22:41:19.000000000","message":"I remmeber thinking it\u0027s really scary to me that the public api might start making up container responses from cache - the proxy-pipeline itself makes calls to container_info all the time (\u003e10k/s) but that\u0027s already cache accelerated and no client ever tries to do that; so I thought it was a weird optimization to make in general, but I didn\u0027t sound to down on it:\n\nhttps://review.opendev.org/c/openstack/swift/+/668064/5#message-225835e9a7f5685a19f0ff679e8f78a747578019","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":false,"context_lines":[{"line_number":690,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":691,"context_line":"        if aresp:"},{"line_number":692,"context_line":"            return aresp"},{"line_number":693,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":694,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":695,"context_line":""},{"line_number":696,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":3,"id":"d4c3bb21_0ba6fc11","line":693,"in_reply_to":"7183f6bb_9483fac1","updated":"2023-11-29 21:56:41.000000000","message":"Acknowledged","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":690,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":691,"context_line":"        if aresp:"},{"line_number":692,"context_line":"            return aresp"},{"line_number":693,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":694,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":695,"context_line":""},{"line_number":696,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":3,"id":"f3f75dc3_9610a48b","line":693,"in_reply_to":"bfdfddaf_8f9b1ffa","updated":"2023-11-14 04:05:42.000000000","message":"So HEADs wont go via the cache? We do cache container infos which are bacially container HEADs. This makes HEADs rather expensive under load doesn\u0027t it. I wonder if we should try and take advantage of container info for cached head requests, and maybe an X-Newest could bypass :hmm:","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":true,"context_lines":[{"line_number":690,"context_line":"        aresp \u003d self._get_or_head_pre_check(req)"},{"line_number":691,"context_line":"        if aresp:"},{"line_number":692,"context_line":"            return aresp"},{"line_number":693,"context_line":"        resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":694,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":695,"context_line":""},{"line_number":696,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":3,"id":"1fe65387_4bbcc13b","line":693,"in_reply_to":"f3f75dc3_9610a48b","updated":"2023-11-14 18:42:53.000000000","message":"apparently this has been proposed before 😊\n\nhttps://review.opendev.org/c/openstack/swift/+/668064\nhttps://review.opendev.org/c/openstack/swift/+/883638","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":362,"context_line":"        return self._get_shard_ranges_from_backend(req), cache_state"},{"line_number":363,"context_line":""},{"line_number":364,"context_line":"    def GETorHEAD(self, req):"},{"line_number":365,"context_line":"        \"\"\"Handler for HTTP GET/HEAD requests.\"\"\""},{"line_number":366,"context_line":"        ai \u003d self.account_info(self.account_name, req)"},{"line_number":367,"context_line":"        auto_account \u003d self.account_name.startswith("},{"line_number":368,"context_line":"            self.app.auto_create_account_prefix)"}],"source_content_type":"text/x-python","patch_set":9,"id":"4e801346_4df4009d","side":"PARENT","line":365,"updated":"2023-11-16 22:41:19.000000000","message":"This method went away entirely and good riddence, the new pre/post check methods are much better used explicitly in GET and HEAD\u0027s individual methods.","commit_id":"b67c88714a4da72235f61ffa0bd2cf024bc0a76f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":362,"context_line":"        return self._get_shard_ranges_from_backend(req), cache_state"},{"line_number":363,"context_line":""},{"line_number":364,"context_line":"    def GETorHEAD(self, req):"},{"line_number":365,"context_line":"        \"\"\"Handler for HTTP GET/HEAD requests.\"\"\""},{"line_number":366,"context_line":"        ai \u003d self.account_info(self.account_name, req)"},{"line_number":367,"context_line":"        auto_account \u003d self.account_name.startswith("},{"line_number":368,"context_line":"            self.app.auto_create_account_prefix)"}],"source_content_type":"text/x-python","patch_set":9,"id":"e294fdaa_6c48b899","side":"PARENT","line":365,"in_reply_to":"4e801346_4df4009d","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"b67c88714a4da72235f61ffa0bd2cf024bc0a76f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":434,"context_line":""},{"line_number":435,"context_line":"        if all((req.method \u003d\u003d \"GET\", record_type \u003d\u003d \u0027auto\u0027,"},{"line_number":436,"context_line":"               resp_record_type.lower() \u003d\u003d \u0027shard\u0027)):"},{"line_number":437,"context_line":"            resp \u003d self._get_from_shards(req, resp)"},{"line_number":438,"context_line":""},{"line_number":439,"context_line":"        if not config_true_value("},{"line_number":440,"context_line":"                resp.headers.get(\u0027X-Backend-Cached-Results\u0027)):"}],"source_content_type":"text/x-python","patch_set":9,"id":"90ae3962_1965f48d","side":"PARENT","line":437,"updated":"2023-11-16 22:41:19.000000000","message":"so here we see:\n\n    if shards_cacheable:\n        resp \u003d get_using_cache()\n    else:\n        resp \u003d get_from_backend()\n    if got_shards:\n        resp \u003d get_from_shards()","commit_id":"b67c88714a4da72235f61ffa0bd2cf024bc0a76f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":434,"context_line":""},{"line_number":435,"context_line":"        if all((req.method \u003d\u003d \"GET\", record_type \u003d\u003d \u0027auto\u0027,"},{"line_number":436,"context_line":"               resp_record_type.lower() \u003d\u003d \u0027shard\u0027)):"},{"line_number":437,"context_line":"            resp \u003d self._get_from_shards(req, resp)"},{"line_number":438,"context_line":""},{"line_number":439,"context_line":"        if not config_true_value("},{"line_number":440,"context_line":"                resp.headers.get(\u0027X-Backend-Cached-Results\u0027)):"}],"source_content_type":"text/x-python","patch_set":9,"id":"c7e9a139_b889088e","side":"PARENT","line":437,"in_reply_to":"90ae3962_1965f48d","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"b67c88714a4da72235f61ffa0bd2cf024bc0a76f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":106,"context_line":"        resp \u003d self.GETorHEAD_base("},{"line_number":107,"context_line":"            req, \u0027Container\u0027, node_iter, part,"},{"line_number":108,"context_line":"            req.swift_entity_path, concurrency)"},{"line_number":109,"context_line":"        return resp"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    def _filter_complete_listing(self, req, namespaces):"},{"line_number":112,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"a674b5ac_3f2a5b68","line":109,"updated":"2023-11-16 22:41:19.000000000","message":"this cute little _GETorHEAD_from_backend method didn\u0027t have to change at all\n\nit takes a request, it makes backend requests with it, it returns it.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":106,"context_line":"        resp \u003d self.GETorHEAD_base("},{"line_number":107,"context_line":"            req, \u0027Container\u0027, node_iter, part,"},{"line_number":108,"context_line":"            req.swift_entity_path, concurrency)"},{"line_number":109,"context_line":"        return resp"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    def _filter_complete_listing(self, req, namespaces):"},{"line_number":112,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"c506f785_ff23095e","line":109,"in_reply_to":"a674b5ac_3f2a5b68","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":185,"context_line":"                \u0027Caching listing namespaces for %s (%d namespaces)\u0027,"},{"line_number":186,"context_line":"                cache_key, len(ns_bound_list.bounds))"},{"line_number":187,"context_line":"        # return the de-gapped namespaces"},{"line_number":188,"context_line":"        return ns_bound_list.get_namespaces()"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"    def _record_shard_listing_cache_metrics("},{"line_number":191,"context_line":"            self, cache_state, resp, resp_record_type, info):"}],"source_content_type":"text/x-python","patch_set":9,"id":"d7a4bd29_67988658","line":188,"updated":"2023-11-16 22:41:19.000000000","message":"all these methods/interfaces seem perfectly reasonable and easy to grok on the surface.\n\nI didn\u0027t notice at first they\u0027re also trying to add some consistency with the object controller\u0027s handling of cached *updating* namespaces by using new common helpers.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebe11d824b643b5c91546364850080fc7a59ab5b","unresolved":false,"context_lines":[{"line_number":185,"context_line":"                \u0027Caching listing namespaces for %s (%d namespaces)\u0027,"},{"line_number":186,"context_line":"                cache_key, len(ns_bound_list.bounds))"},{"line_number":187,"context_line":"        # return the de-gapped namespaces"},{"line_number":188,"context_line":"        return ns_bound_list.get_namespaces()"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"    def _record_shard_listing_cache_metrics("},{"line_number":191,"context_line":"            self, cache_state, resp, resp_record_type, info):"}],"source_content_type":"text/x-python","patch_set":9,"id":"a1995ad7_f1c27907","line":188,"in_reply_to":"1bba44a2_7c9d17fc","updated":"2023-11-22 12:39:23.000000000","message":"Done","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":true,"context_lines":[{"line_number":185,"context_line":"                \u0027Caching listing namespaces for %s (%d namespaces)\u0027,"},{"line_number":186,"context_line":"                cache_key, len(ns_bound_list.bounds))"},{"line_number":187,"context_line":"        # return the de-gapped namespaces"},{"line_number":188,"context_line":"        return ns_bound_list.get_namespaces()"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"    def _record_shard_listing_cache_metrics("},{"line_number":191,"context_line":"            self, cache_state, resp, resp_record_type, info):"}],"source_content_type":"text/x-python","patch_set":9,"id":"1bba44a2_7c9d17fc","line":188,"in_reply_to":"d7a4bd29_67988658","updated":"2023-11-20 14:38:32.000000000","message":"I broke out a separate patch for the cache helpers, hopefully clearer https://review.opendev.org/c/openstack/swift/+/901333/1?usp\u003drelated-change","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":259,"context_line":"                # don\u0027t know if the container was sharded, or the case that the"},{"line_number":260,"context_line":"                # sharding state in metadata indicates the container was"},{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"99d9c5f0_9d0a2b47","line":262,"updated":"2023-11-16 22:41:19.000000000","message":"some additional differentiation might be useful in this case\n\n... but it looks like this part of diff is only (mostly?) carrying over existing behavior from _GET_using_cache but inlined/indented.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":true,"context_lines":[{"line_number":259,"context_line":"                # don\u0027t know if the container was sharded, or the case that the"},{"line_number":260,"context_line":"                # sharding state in metadata indicates the container was"},{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"ec880518_cea9abf3","line":262,"in_reply_to":"99d9c5f0_9d0a2b47","updated":"2023-11-20 14:38:32.000000000","message":"yes this is existing behavior, and it is a bit of a catch-all for \"we\u0027re not bothering with cache\"\n\nwe could add more granularity in a follow up?","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eac2c0a866f10723334ad06c0c42aa6296d1b18a","unresolved":false,"context_lines":[{"line_number":259,"context_line":"                # don\u0027t know if the container was sharded, or the case that the"},{"line_number":260,"context_line":"                # sharding state in metadata indicates the container was"},{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"484f5b52_3d7dad60","line":262,"in_reply_to":"ec880518_cea9abf3","updated":"2023-11-20 14:47:05.000000000","message":"copied comment to https://review.opendev.org/c/openstack/swift/+/901335","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        if namespaces:"},{"line_number":267,"context_line":"            # Namespaces found in cache so there is no need to go to backend,"}],"source_content_type":"text/x-python","patch_set":9,"id":"ede55871_8196496b","line":264,"updated":"2023-11-16 22:41:19.000000000","message":"I don\u0027t understand this TODO; and/or the include-deleted use-case?\n\nIf the sharder wanted deleted shards for an audit it wouldn\u0027t go into auto, the explicit record-type\u003dshards always gets routed to the backend in GET","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":true,"context_lines":[{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        if namespaces:"},{"line_number":267,"context_line":"            # Namespaces found in cache so there is no need to go to backend,"}],"source_content_type":"text/x-python","patch_set":9,"id":"f78394a5_ea3a03ec","line":264,"in_reply_to":"ede55871_8196496b","updated":"2023-11-20 14:38:32.000000000","message":"I think I was making a mental note that (on master) \u0027disabled\u0027 is returned when in fact it was an x-backend-include-deleted\u003dTrue header that caused us to not use cache.\n\nI should remove the comment, it is no longer relevant: x-backend-include-deleted isn\u0027t supported in this path, and is explicitly set to False if we end up going to the backend","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eac2c0a866f10723334ad06c0c42aa6296d1b18a","unresolved":false,"context_lines":[{"line_number":261,"context_line":"                # unsharded."},{"line_number":262,"context_line":"                cache_state \u003d \u0027bypass\u0027"},{"line_number":263,"context_line":"        else:"},{"line_number":264,"context_line":"            cache_state \u003d \u0027disabled\u0027  # TODO: could be include-deleted case?"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"        if namespaces:"},{"line_number":267,"context_line":"            # Namespaces found in cache so there is no need to go to backend,"}],"source_content_type":"text/x-python","patch_set":9,"id":"96cfb87e_6da03bb6","line":264,"in_reply_to":"f78394a5_ea3a03ec","updated":"2023-11-20 14:47:05.000000000","message":"linked comment to https://review.opendev.org/c/openstack/swift/+/901335","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":272,"context_line":"            headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":273,"context_line":"                            \u0027x-backend-record-shard-format\u0027: \u0027namespace\u0027,"},{"line_number":274,"context_line":"                            \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":275,"context_line":"            resp \u003d Response(request\u003dreq)"},{"line_number":276,"context_line":"            update_headers(resp, headers)"},{"line_number":277,"context_line":"            resp.last_modified \u003d Timestamp("},{"line_number":278,"context_line":"                headers[\u0027x-put-timestamp\u0027]).ceil()"}],"source_content_type":"text/x-python","patch_set":9,"id":"6b3012a0_f260d52f","line":275,"updated":"2023-11-16 22:41:19.000000000","message":"i like how there\u0027s no \"get_from_cache\" method that sometimes returns from the backend.  *if* we have cached namespces we craft a \"resp_from_cache\" - otherwise we go get them from the backend!\n\nwe NEVER return this made up cache response - it\u0027s just a parameter to _record_shard_listing_cache_metrics and _get_from_shards as placeholder for when DO have to fetch namespaces from the backend.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":272,"context_line":"            headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":273,"context_line":"                            \u0027x-backend-record-shard-format\u0027: \u0027namespace\u0027,"},{"line_number":274,"context_line":"                            \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":275,"context_line":"            resp \u003d Response(request\u003dreq)"},{"line_number":276,"context_line":"            update_headers(resp, headers)"},{"line_number":277,"context_line":"            resp.last_modified \u003d Timestamp("},{"line_number":278,"context_line":"                headers[\u0027x-put-timestamp\u0027]).ceil()"}],"source_content_type":"text/x-python","patch_set":9,"id":"56ba9a41_aeb311df","line":275,"in_reply_to":"6b3012a0_f260d52f","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":311,"context_line":"                # shards and caching them is therefore more likely to result in"},{"line_number":312,"context_line":"                # stale or incomplete listings on subsequent container GETs."},{"line_number":313,"context_line":"                req.headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027sharded\u0027"},{"line_number":314,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":315,"context_line":"            resp_record_type \u003d resp.headers.get("},{"line_number":316,"context_line":"                \u0027x-backend-record-type\u0027, \u0027\u0027).lower()"},{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("}],"source_content_type":"text/x-python","patch_set":9,"id":"3561fb02_5334908e","line":314,"updated":"2023-11-16 22:41:19.000000000","message":"this is the \"magic\" get-auto-from-backend request, very well commented and well contained in context","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":311,"context_line":"                # shards and caching them is therefore more likely to result in"},{"line_number":312,"context_line":"                # stale or incomplete listings on subsequent container GETs."},{"line_number":313,"context_line":"                req.headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027sharded\u0027"},{"line_number":314,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":315,"context_line":"            resp_record_type \u003d resp.headers.get("},{"line_number":316,"context_line":"                \u0027x-backend-record-type\u0027, \u0027\u0027).lower()"},{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("}],"source_content_type":"text/x-python","patch_set":9,"id":"f5c23030_c1c8f4ed","line":314,"in_reply_to":"3561fb02_5334908e","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("},{"line_number":318,"context_line":"                \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":319,"context_line":"            complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":320,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":321,"context_line":"            if resp_record_type \u003d\u003d \u0027shard\u0027:"},{"line_number":322,"context_line":"                data \u003d self._parse_listing_response(req, resp)"},{"line_number":323,"context_line":"                namespaces \u003d self._parse_namespaces(req, data, resp)"}],"source_content_type":"text/x-python","patch_set":9,"id":"f271d1e0_31d58b17","line":320,"updated":"2023-11-16 22:41:19.000000000","message":"this seems like a weird way to decide this?  test\u0027s stub responses may not be consistent with the backend implementation.\n\nI\u0027m looking at test_GET_sharded_container_sharding_shard - we enter under _get_from_shards twice, but never _set_listing_namespace_in_cache\n\nWould it be better/suffciient to parse namespaces and make sure ns[0].lower \u003d\u003d ns[-1].upper \u003d\u003d \u0027\u0027?","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":true,"context_lines":[{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("},{"line_number":318,"context_line":"                \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":319,"context_line":"            complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":320,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":321,"context_line":"            if resp_record_type \u003d\u003d \u0027shard\u0027:"},{"line_number":322,"context_line":"                data \u003d self._parse_listing_response(req, resp)"},{"line_number":323,"context_line":"                namespaces \u003d self._parse_namespaces(req, data, resp)"}],"source_content_type":"text/x-python","patch_set":9,"id":"fd0b0a68_8b5df6ff","line":320,"in_reply_to":"f271d1e0_31d58b17","updated":"2023-11-20 14:38:32.000000000","message":"now covered in https://review.opendev.org/c/openstack/swift/+/901335/2?usp\u003drelated-change","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eac2c0a866f10723334ad06c0c42aa6296d1b18a","unresolved":false,"context_lines":[{"line_number":317,"context_line":"            sharding_state \u003d resp.headers.get("},{"line_number":318,"context_line":"                \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":319,"context_line":"            complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":320,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":321,"context_line":"            if resp_record_type \u003d\u003d \u0027shard\u0027:"},{"line_number":322,"context_line":"                data \u003d self._parse_listing_response(req, resp)"},{"line_number":323,"context_line":"                namespaces \u003d self._parse_namespaces(req, data, resp)"}],"source_content_type":"text/x-python","patch_set":9,"id":"98d46a35_41a34fe0","line":320,"in_reply_to":"fd0b0a68_8b5df6ff","updated":"2023-11-20 14:47:05.000000000","message":"see later patch","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":333,"context_line":"                        req, namespaces)"},{"line_number":334,"context_line":"                    namespaces \u003d self._filter_complete_listing(req, namespaces)"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"        if cache_state:"},{"line_number":337,"context_line":"            resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":338,"context_line":"            self._record_shard_listing_cache_metrics("},{"line_number":339,"context_line":"                cache_state, resp, resp_record_type, info)"}],"source_content_type":"text/x-python","patch_set":9,"id":"16360e85_20cb659a","line":336,"updated":"2023-11-16 22:41:19.000000000","message":"I don\u0027t see when cache_state is falsy\n\n    diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py\n    index bea9a2312..84a21a1fb 100644\n    --- a/swift/proxy/controllers/container.py\n    +++ b/swift/proxy/controllers/container.py\n    @@ -337,6 +337,8 @@ class ContainerController(Controller):\n                 resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)\n                 self._record_shard_listing_cache_metrics(\n                     cache_state, resp, resp_record_type, info)\n    +        else:\n    +            asdf\n     \n             if namespaces is not None:\n                 # we got namespaces, so the container must be sharded; now build\n\n\n\n... seems to pass at least test.unit.proxy.controller.test_container","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":false,"context_lines":[{"line_number":333,"context_line":"                        req, namespaces)"},{"line_number":334,"context_line":"                    namespaces \u003d self._filter_complete_listing(req, namespaces)"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"        if cache_state:"},{"line_number":337,"context_line":"            resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":338,"context_line":"            self._record_shard_listing_cache_metrics("},{"line_number":339,"context_line":"                cache_state, resp, resp_record_type, info)"}],"source_content_type":"text/x-python","patch_set":9,"id":"9d751741_6389ca2c","line":336,"in_reply_to":"01a9829b_d19acd45","updated":"2023-11-29 21:56:41.000000000","message":"I\u0027m not sure why I was confused; in this patch it\u0027s set to None on L375","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eac2c0a866f10723334ad06c0c42aa6296d1b18a","unresolved":true,"context_lines":[{"line_number":333,"context_line":"                        req, namespaces)"},{"line_number":334,"context_line":"                    namespaces \u003d self._filter_complete_listing(req, namespaces)"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"        if cache_state:"},{"line_number":337,"context_line":"            resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":338,"context_line":"            self._record_shard_listing_cache_metrics("},{"line_number":339,"context_line":"                cache_state, resp, resp_record_type, info)"}],"source_content_type":"text/x-python","patch_set":9,"id":"01a9829b_d19acd45","line":336,"in_reply_to":"16360e85_20cb659a","updated":"2023-11-20 14:47:05.000000000","message":"see later patch","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":343,"context_line":"            # the listing from shards"},{"line_number":344,"context_line":"            # NB: the filtered namespaces list may be empty but we still need"},{"line_number":345,"context_line":"            # to build a response body with an empty list of shards"},{"line_number":346,"context_line":"            resp \u003d self._get_from_shards(req, resp, namespaces)"},{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        return resp"},{"line_number":349,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"4b415e6d_5f6640d2","line":346,"updated":"2023-11-16 22:41:19.000000000","message":"yes!  perfectly unifies the optimized shards-from-cache lookup with the surprise shards from object-listing backend-repsonse.\n\nI love this method now.\n\n    def get_object_listing_from_shards_if_needed_automatically():\n        is_sharded \u003d check_cache_only()\n        if is_sharded:\n            namespaces \u003d check_cache_only()\n        if not namespaces:\n            resp \u003d GET_from_backend(record_type\u003dauto)\n            if is_shards(resp):\n                namespaces \u003d parse_resp(resp)\n        if namespaces:\n            # out to BaseController and back through GET\n            resp \u003d get_from_shards(namespaces)\n        return resp","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":343,"context_line":"            # the listing from shards"},{"line_number":344,"context_line":"            # NB: the filtered namespaces list may be empty but we still need"},{"line_number":345,"context_line":"            # to build a response body with an empty list of shards"},{"line_number":346,"context_line":"            resp \u003d self._get_from_shards(req, resp, namespaces)"},{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        return resp"},{"line_number":349,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"87344be9_9431e159","line":346,"in_reply_to":"4b415e6d_5f6640d2","updated":"2023-11-20 14:38:32.000000000","message":"❤️ BTW much of the changes leading up to this call have moved to the next patch in the chain https://review.opendev.org/c/openstack/swift/+/901335/2?usp\u003drelated-change","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":430,"context_line":"            # be built from shards and may involve reading/writing namespaces"},{"line_number":431,"context_line":"            # in cache. This path is used for client requests and by the proxy"},{"line_number":432,"context_line":"            # itself while recursively building a listing from shards."},{"line_number":433,"context_line":"            resp \u003d self._GET_auto(req)"},{"line_number":434,"context_line":""},{"line_number":435,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":436,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"4fe72b04_9e8d95c0","line":433,"updated":"2023-11-16 22:41:19.000000000","message":"now the main entrypoint for GET is:\n\n    if backend_record_type:\n        # this is super simple and obvious\n        resp \u003d get_from_backend()\n    else:\n        # hold onto your butts!\n        resp \u003d get_object_listing_from_shards_if_needed_automatically()","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":430,"context_line":"            # be built from shards and may involve reading/writing namespaces"},{"line_number":431,"context_line":"            # in cache. This path is used for client requests and by the proxy"},{"line_number":432,"context_line":"            # itself while recursively building a listing from shards."},{"line_number":433,"context_line":"            resp \u003d self._GET_auto(req)"},{"line_number":434,"context_line":""},{"line_number":435,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":436,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"4a6ab743_308af992","line":433,"in_reply_to":"4fe72b04_9e8d95c0","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":434,"context_line":""},{"line_number":435,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"    def _get_from_shards(self, req, resp, namespaces):"},{"line_number":438,"context_line":"        # Construct listing using shards described by the list of namespaces."},{"line_number":439,"context_line":"        # The history of containers that have returned namespaces is"},{"line_number":440,"context_line":"        # maintained in the request environ so that loops can be avoided by"}],"source_content_type":"text/x-python","patch_set":9,"id":"00e522bf_eaa5ea05","line":437,"updated":"2023-11-16 22:41:19.000000000","message":"doc string?\n\nI think th resp here is made up resp headers from container_info in the common case\n\n... but may be an actual root container resp if there was a cache bypass kind of thing going on?","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":434,"context_line":""},{"line_number":435,"context_line":"        return self._get_or_head_post_check(req, resp)"},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"    def _get_from_shards(self, req, resp, namespaces):"},{"line_number":438,"context_line":"        # Construct listing using shards described by the list of namespaces."},{"line_number":439,"context_line":"        # The history of containers that have returned namespaces is"},{"line_number":440,"context_line":"        # maintained in the request environ so that loops can be avoided by"}],"source_content_type":"text/x-python","patch_set":9,"id":"bd71f1fc_ca813b4d","line":437,"in_reply_to":"00e522bf_eaa5ea05","updated":"2023-11-20 14:38:32.000000000","message":"doc string done.\n\nYes, if we find namespaces in cache then we use cached container info to build response headers. Otherwise the headers are whatever we got from the backend.","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":397,"context_line":""},{"line_number":398,"context_line":"        if all((req.method \u003d\u003d \"GET\", record_type \u003d\u003d \u0027auto\u0027,"},{"line_number":399,"context_line":"               resp_record_type.lower() \u003d\u003d \u0027shard\u0027)):"},{"line_number":400,"context_line":"            data \u003d self._parse_listing_response(req, resp)"},{"line_number":401,"context_line":"            namespaces \u003d self._parse_namespaces(req, data, resp)"},{"line_number":402,"context_line":"            if namespaces is not None:"},{"line_number":403,"context_line":"                # we got namespaces, so the container must be sharded; now"}],"source_content_type":"text/x-python","patch_set":11,"id":"e404b401_f151471d","line":400,"updated":"2023-11-19 00:57:26.000000000","message":"that\u0027s great we are reusing those internal functions, but not clear to me why you moved calling  ``_parse_listing_response`` and ``_parse_namespaces`` outside of ``_get_from_shards``","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8cca4f513237737d7af9d3e867c518458b53bae2","unresolved":false,"context_lines":[{"line_number":397,"context_line":""},{"line_number":398,"context_line":"        if all((req.method \u003d\u003d \"GET\", record_type \u003d\u003d \u0027auto\u0027,"},{"line_number":399,"context_line":"               resp_record_type.lower() \u003d\u003d \u0027shard\u0027)):"},{"line_number":400,"context_line":"            data \u003d self._parse_listing_response(req, resp)"},{"line_number":401,"context_line":"            namespaces \u003d self._parse_namespaces(req, data, resp)"},{"line_number":402,"context_line":"            if namespaces is not None:"},{"line_number":403,"context_line":"                # we got namespaces, so the container must be sharded; now"}],"source_content_type":"text/x-python","patch_set":11,"id":"e0c67400_d5f5b502","line":400,"in_reply_to":"3286e9d1_8fcaac19","updated":"2023-11-22 07:05:19.000000000","message":"Makes sense to me, ``self._get_from_shards(req, resp, namespaces)`` does make it clearer that the list is built using a list of namespaces.","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":true,"context_lines":[{"line_number":397,"context_line":""},{"line_number":398,"context_line":"        if all((req.method \u003d\u003d \"GET\", record_type \u003d\u003d \u0027auto\u0027,"},{"line_number":399,"context_line":"               resp_record_type.lower() \u003d\u003d \u0027shard\u0027)):"},{"line_number":400,"context_line":"            data \u003d self._parse_listing_response(req, resp)"},{"line_number":401,"context_line":"            namespaces \u003d self._parse_namespaces(req, data, resp)"},{"line_number":402,"context_line":"            if namespaces is not None:"},{"line_number":403,"context_line":"                # we got namespaces, so the container must be sharded; now"}],"source_content_type":"text/x-python","patch_set":11,"id":"3286e9d1_8fcaac19","line":400,"in_reply_to":"e404b401_f151471d","updated":"2023-11-20 14:38:32.000000000","message":"ok, I may have got a little ahead of myself with this interface change: it will be clear in the next patch https://review.opendev.org/c/openstack/swift/+/901335 when the resp body is no longer the source of the namespace list (the next patch stops reading namespaces from cache and serializing them into the response body)...so we *have* to pass in the list of namespaces.\n\nFor now, an admittedly weak justification is that it makes for better \"separation of concerns\" i.e. the caller deal with whatever is in the response, then _get_from_shards builds a listing, and perhaps makes it a little clearer that the list is built using a list of *namespaces*.","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":507,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(namespace.end_marker)"},{"line_number":508,"context_line":""},{"line_number":509,"context_line":"            headers \u003d {}"},{"line_number":510,"context_line":"            shard_account, shard_container \u003d namespace.name.split(\u0027/\u0027)"},{"line_number":511,"context_line":"            if (shard_account, shard_container) in shard_listing_history:"},{"line_number":512,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":513,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":11,"id":"fe737bd9_f4c54d08","line":510,"updated":"2023-11-19 00:57:26.000000000","message":"even though we decided not to store account/container in Namespace class, but we still add a helper function to parse the name and get account/container.","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8cca4f513237737d7af9d3e867c518458b53bae2","unresolved":false,"context_lines":[{"line_number":507,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(namespace.end_marker)"},{"line_number":508,"context_line":""},{"line_number":509,"context_line":"            headers \u003d {}"},{"line_number":510,"context_line":"            shard_account, shard_container \u003d namespace.name.split(\u0027/\u0027)"},{"line_number":511,"context_line":"            if (shard_account, shard_container) in shard_listing_history:"},{"line_number":512,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":513,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":11,"id":"1c7f45c5_c5cfec96","line":510,"in_reply_to":"c7b585a6_f8d3adde","updated":"2023-11-22 07:05:19.000000000","message":"Acknowledged","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":true,"context_lines":[{"line_number":507,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(namespace.end_marker)"},{"line_number":508,"context_line":""},{"line_number":509,"context_line":"            headers \u003d {}"},{"line_number":510,"context_line":"            shard_account, shard_container \u003d namespace.name.split(\u0027/\u0027)"},{"line_number":511,"context_line":"            if (shard_account, shard_container) in shard_listing_history:"},{"line_number":512,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":513,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":11,"id":"c7b585a6_f8d3adde","line":510,"in_reply_to":"fe737bd9_f4c54d08","updated":"2023-11-20 14:38:32.000000000","message":"not sure I understand the comment: I decided against adding any helper because the split is pretty trivial. I don\u0027t want Namespace to be aware of \u0027a/c\u0027 path format, and ShardRange.split_name(name) class method seems overkill.","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":579,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":580,"context_line":"                break"},{"line_number":581,"context_line":""},{"line_number":582,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":583,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":584,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":585,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":11,"id":"e63c87d6_8bd7199a","line":582,"updated":"2023-11-19 00:57:26.000000000","message":"this is a bug fix?","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":true,"context_lines":[{"line_number":579,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":580,"context_line":"                break"},{"line_number":581,"context_line":""},{"line_number":582,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":583,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":584,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":585,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":11,"id":"e760ee9b_d4f2cbfa","line":582,"in_reply_to":"e63c87d6_8bd7199a","updated":"2023-11-20 14:38:32.000000000","message":"I don\u0027t believe anything relies on this, but we were leaking X-Backend-Record-Type \u003d \u0027shard\u0027.\n\nSee for example test.unit.proxy.controllers.test_container.TestGetShardedContainer.test_GET_sharded_container_no_memcache and revert this change.\n\nIt seems appropriate to be explicit about the record type in the response, rather than just deleting X-Backend-Record-Type \u003d \u0027shard\u0027.","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8cca4f513237737d7af9d3e867c518458b53bae2","unresolved":false,"context_lines":[{"line_number":579,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":580,"context_line":"                break"},{"line_number":581,"context_line":""},{"line_number":582,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":583,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":584,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":585,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":11,"id":"98078432_71f7096b","line":582,"in_reply_to":"e760ee9b_d4f2cbfa","updated":"2023-11-22 07:05:19.000000000","message":"Acknowledged","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":458,"context_line":"        # attributes set."},{"line_number":459,"context_line":"        # Ideally we would construct Namespace objects here, but later we use"},{"line_number":460,"context_line":"        # the ShardRange account and container properties to access parsed"},{"line_number":461,"context_line":"        # parts of the name."},{"line_number":462,"context_line":"        shard_ranges \u003d [ShardRange(**data) for data in json.loads(resp.body)]"},{"line_number":463,"context_line":"        self.logger.debug(\u0027GET listing from %s shards for: %s\u0027,"},{"line_number":464,"context_line":"                          len(shard_ranges), req.path_qs)"}],"source_content_type":"text/x-python","patch_set":17,"id":"d315ffd8_62a8ec33","side":"PARENT","line":461,"updated":"2023-11-29 21:56:41.000000000","message":"looks like we have a nice TODO cleanup going on here!","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":458,"context_line":"        # attributes set."},{"line_number":459,"context_line":"        # Ideally we would construct Namespace objects here, but later we use"},{"line_number":460,"context_line":"        # the ShardRange account and container properties to access parsed"},{"line_number":461,"context_line":"        # parts of the name."},{"line_number":462,"context_line":"        shard_ranges \u003d [ShardRange(**data) for data in json.loads(resp.body)]"},{"line_number":463,"context_line":"        self.logger.debug(\u0027GET listing from %s shards for: %s\u0027,"},{"line_number":464,"context_line":"                          len(shard_ranges), req.path_qs)"}],"source_content_type":"text/x-python","patch_set":17,"id":"1adfc224_358b098e","side":"PARENT","line":461,"in_reply_to":"d315ffd8_62a8ec33","updated":"2023-11-30 15:43:06.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":483,"context_line":""},{"line_number":484,"context_line":"        limit \u003d req_limit"},{"line_number":485,"context_line":"        all_resp_status \u003d []"},{"line_number":486,"context_line":"        for i, namespace in enumerate(namespaces):"},{"line_number":487,"context_line":"            params[\u0027limit\u0027] \u003d limit"},{"line_number":488,"context_line":"            # Always set marker to ensure that object names less than or equal"},{"line_number":489,"context_line":"            # to those already in the listing are not fetched; if the listing"}],"source_content_type":"text/x-python","patch_set":17,"id":"fb490721_1d31aada","line":486,"updated":"2023-11-29 21:56:41.000000000","message":"unrealted: `NamespaceBoundsList` supports iteration via get_namespaces()","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":483,"context_line":""},{"line_number":484,"context_line":"        limit \u003d req_limit"},{"line_number":485,"context_line":"        all_resp_status \u003d []"},{"line_number":486,"context_line":"        for i, namespace in enumerate(namespaces):"},{"line_number":487,"context_line":"            params[\u0027limit\u0027] \u003d limit"},{"line_number":488,"context_line":"            # Always set marker to ensure that object names less than or equal"},{"line_number":489,"context_line":"            # to those already in the listing are not fetched; if the listing"}],"source_content_type":"text/x-python","patch_set":17,"id":"60f1ec1a_d05fb77a","line":486,"in_reply_to":"fb490721_1d31aada","updated":"2023-11-30 15:43:06.000000000","message":"Acknowledged","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":518,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(namespace.end_marker)"},{"line_number":519,"context_line":""},{"line_number":520,"context_line":"            headers \u003d {}"},{"line_number":521,"context_line":"            shard_account, shard_container \u003d namespace.name.split(\u0027/\u0027)"},{"line_number":522,"context_line":"            if (shard_account, shard_container) in shard_listing_history:"},{"line_number":523,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":524,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":17,"id":"bebbe36d_8f07044c","line":521,"updated":"2023-11-29 21:56:41.000000000","message":"I don\u0027t understand the driving principle behind keeping the Namespace.name agnostic to having a `/` - it seems like a pretty firm requirement.\n\nMoreover if this split is only to populate the shard_listing_history we could just put the whole unsplit name into the history.\n\nBut it looks like a LOT of the interfaces in base.py expect account/container to be split for consistency with controller.account/controller.container; so it seems like it\u0027d be convienient for Namespace to support that as well.","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":518,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(namespace.end_marker)"},{"line_number":519,"context_line":""},{"line_number":520,"context_line":"            headers \u003d {}"},{"line_number":521,"context_line":"            shard_account, shard_container \u003d namespace.name.split(\u0027/\u0027)"},{"line_number":522,"context_line":"            if (shard_account, shard_container) in shard_listing_history:"},{"line_number":523,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":524,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":17,"id":"cef23fab_ae150127","line":521,"in_reply_to":"bebbe36d_8f07044c","updated":"2023-11-30 15:43:06.000000000","message":"The history is seeded with (self.account, self.container) which is why we can\u0027t just use Namespace.name in the history\n\nI\u0027ve been quite determined in trying to stop a primitive that encapsulates a pair of bounds with a name *having* to have a \u0027/\u0027 in the name. It just feels like an undesirable constraint on a simple primitive.\n\nBUT, having looked over https://review.opendev.org/c/openstack/swift/+/898244 again, I have to admit that the times when we use Namespace and do *not* need an \u0027a/c\u0027 name, we also do not need a name *at all*! So I need to let go of my obsession! I guess we could have\n\nNamespace(lower, upper)\n^\nNamedNamespace(name, lower, upper)\n^\nShardRange(....)\n\n...but that\u0027s probably overkill for the sake of avoiding Namespace(\u0027dont/care\u0027, lower, upper) .\n\nHowever, IIRC @Jianjian was concerned about Namespace having two more attributes (account and container). I\u0027ll put together a follow-on to see how it might work out.\n\n-\u003e https://review.opendev.org/c/openstack/swift/+/902308","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23a1aef2dbdb36a45b67572ffa1bd0a5f4f75c3f","unresolved":false,"context_lines":[{"line_number":518,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(namespace.end_marker)"},{"line_number":519,"context_line":""},{"line_number":520,"context_line":"            headers \u003d {}"},{"line_number":521,"context_line":"            shard_account, shard_container \u003d namespace.name.split(\u0027/\u0027)"},{"line_number":522,"context_line":"            if (shard_account, shard_container) in shard_listing_history:"},{"line_number":523,"context_line":"                # directed back to same container - force GET of objects"},{"line_number":524,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":17,"id":"8f7d83c0_46727cdf","line":521,"in_reply_to":"cef23fab_ae150127","updated":"2024-01-04 16:33:04.000000000","message":"Done","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":545,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"},{"line_number":546,"context_line":"                i, namespace, namespace.name, headers)"},{"line_number":547,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":548,"context_line":"                req, shard_account, shard_container,"},{"line_number":549,"context_line":"                headers\u003dheaders, params\u003dparams)"},{"line_number":550,"context_line":"            all_resp_status.append(shard_resp.status_int)"},{"line_number":551,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"05ee27fd_4b5fe018","line":548,"updated":"2023-11-29 21:56:41.000000000","message":"here we use the Namespace account/container again, but _get_container_listing is also called from proxy.controller.obj via _get_update_target with self.account_name, self.container_name","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":545,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"},{"line_number":546,"context_line":"                i, namespace, namespace.name, headers)"},{"line_number":547,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":548,"context_line":"                req, shard_account, shard_container,"},{"line_number":549,"context_line":"                headers\u003dheaders, params\u003dparams)"},{"line_number":550,"context_line":"            all_resp_status.append(shard_resp.status_int)"},{"line_number":551,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"78aa3a32_2cf5de07","line":548,"in_reply_to":"05ee27fd_4b5fe018","updated":"2023-11-30 15:43:06.000000000","message":"Acknowledged","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":590,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":591,"context_line":"                break"},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":594,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":595,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":596,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"facef0a1_6f7bb42b","line":593,"updated":"2023-11-29 21:56:41.000000000","message":"this is going to be the client response, which sans cached listing namespaces would have originally come from a root container response that when sharded returned record-type shard\n\nAFAIK this trivial fix is totally orthogonal to everything else going on in this change and could be reasonably moved into the change that introduces the probetest and discovers the issue.","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23a1aef2dbdb36a45b67572ffa1bd0a5f4f75c3f","unresolved":false,"context_lines":[{"line_number":590,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":591,"context_line":"                break"},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":594,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":595,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":596,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"36d03750_66a13cd7","line":593,"in_reply_to":"2d6e7328_37d2d7fe","updated":"2024-01-04 16:33:04.000000000","message":"Done","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":590,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":591,"context_line":"                break"},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":594,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":595,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":596,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"2d6e7328_37d2d7fe","line":593,"in_reply_to":"facef0a1_6f7bb42b","updated":"2023-11-30 15:43:06.000000000","message":"moved to the probe test patch","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":366,"context_line":""},{"line_number":367,"context_line":"    def _get_update_shard(self, req, account, container, obj):"},{"line_number":368,"context_line":"        \"\"\""},{"line_number":369,"context_line":"        Find the appropriate shard range for an object update."},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"        Note that this fetches and caches (in both the per-request infocache"},{"line_number":372,"context_line":"        and memcache, if available) all shard ranges for the given root"}],"source_content_type":"text/x-python","patch_set":5,"id":"bd6b51bb_e2b63cd0","line":369,"range":{"start_line":369,"start_character":29,"end_line":369,"end_character":40},"updated":"2023-11-14 04:05:42.000000000","message":"Have far do we go changing shards to namespaces. I mean we are trying to find the right shard, saying trying to find the right namespace might be a little too vague? (ie do we need to rename the finction too?). Update shard here in my opinion is the \"shard\" responsible, namespace is just smaller representation of the same shard.. but we\u0027re still looking for a shard.","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":366,"context_line":""},{"line_number":367,"context_line":"    def _get_update_shard(self, req, account, container, obj):"},{"line_number":368,"context_line":"        \"\"\""},{"line_number":369,"context_line":"        Find the appropriate shard range for an object update."},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"        Note that this fetches and caches (in both the per-request infocache"},{"line_number":372,"context_line":"        and memcache, if available) all shard ranges for the given root"}],"source_content_type":"text/x-python","patch_set":5,"id":"1f9297c6_1e2f4009","line":369,"range":{"start_line":369,"start_character":29,"end_line":369,"end_character":40},"in_reply_to":"3ac45502_b4b850bb","updated":"2023-11-20 14:38:32.000000000","message":"Done","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":true,"context_lines":[{"line_number":366,"context_line":""},{"line_number":367,"context_line":"    def _get_update_shard(self, req, account, container, obj):"},{"line_number":368,"context_line":"        \"\"\""},{"line_number":369,"context_line":"        Find the appropriate shard range for an object update."},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"        Note that this fetches and caches (in both the per-request infocache"},{"line_number":372,"context_line":"        and memcache, if available) all shard ranges for the given root"}],"source_content_type":"text/x-python","patch_set":5,"id":"3ac45502_b4b850bb","line":369,"range":{"start_line":369,"start_character":29,"end_line":369,"end_character":40},"in_reply_to":"bd6b51bb_e2b63cd0","updated":"2023-11-14 18:42:53.000000000","message":"I try to avoid making rules because I know I will then break them :)\n\nI\u0027m *trying* to be clear that when we\u0027re referring to a python object type then the naming is appropriate (Namespaces are namespace, ShardRanges are shard_ranges). But when we\u0027re referring to a shard container, then still use \u0027shard\u0027.\n\nSo here we are trying to find a shard, but it is represented by a Namespace. I\u0027m probably painting myself into a corner :/","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":351,"context_line":""},{"line_number":352,"context_line":"        self.logger.info("},{"line_number":353,"context_line":"            \u0027Caching updating shards for %s (%d shards)\u0027,"},{"line_number":354,"context_line":"            cache_key, len(namespaces.bounds))"},{"line_number":355,"context_line":"        try:"},{"line_number":356,"context_line":"            memcache.set("},{"line_number":357,"context_line":"                cache_key, namespaces.bounds,"}],"source_content_type":"text/x-python","patch_set":9,"id":"433d9cac_4e0c2937","side":"PARENT","line":354,"updated":"2023-11-16 22:41:19.000000000","message":"i was looking for this!","commit_id":"b67c88714a4da72235f61ffa0bd2cf024bc0a76f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":351,"context_line":""},{"line_number":352,"context_line":"        self.logger.info("},{"line_number":353,"context_line":"            \u0027Caching updating shards for %s (%d shards)\u0027,"},{"line_number":354,"context_line":"            cache_key, len(namespaces.bounds))"},{"line_number":355,"context_line":"        try:"},{"line_number":356,"context_line":"            memcache.set("},{"line_number":357,"context_line":"                cache_key, namespaces.bounds,"}],"source_content_type":"text/x-python","patch_set":9,"id":"1bf26ea6_c8b998fa","side":"PARENT","line":354,"in_reply_to":"433d9cac_4e0c2937","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"b67c88714a4da72235f61ffa0bd2cf024bc0a76f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":351,"context_line":"                if set_cache_state \u003d\u003d \u0027set\u0027:"},{"line_number":352,"context_line":"                    self.logger.info("},{"line_number":353,"context_line":"                        \u0027Caching updating shards for %s (%d shards)\u0027,"},{"line_number":354,"context_line":"                        cache_key, len(namespaces))"},{"line_number":355,"context_line":"            update_shard \u003d find_namespace(obj, namespaces or [])"},{"line_number":356,"context_line":""},{"line_number":357,"context_line":"        record_cache_op_metrics("}],"source_content_type":"text/x-python","patch_set":9,"id":"2230f118_28228cf9","line":354,"updated":"2023-11-16 22:41:19.000000000","message":"this logging isn\u0027t new; it just moved.\n\nIt matches the logging in ContainerController._set_listing_namespaces_in_cache","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":351,"context_line":"                if set_cache_state \u003d\u003d \u0027set\u0027:"},{"line_number":352,"context_line":"                    self.logger.info("},{"line_number":353,"context_line":"                        \u0027Caching updating shards for %s (%d shards)\u0027,"},{"line_number":354,"context_line":"                        cache_key, len(namespaces))"},{"line_number":355,"context_line":"            update_shard \u003d find_namespace(obj, namespaces or [])"},{"line_number":356,"context_line":""},{"line_number":357,"context_line":"        record_cache_op_metrics("}],"source_content_type":"text/x-python","patch_set":9,"id":"b5fa0fb0_bf1978d7","line":354,"in_reply_to":"2230f118_28228cf9","updated":"2023-11-20 14:38:32.000000000","message":"Ack, this part of change is now in https://review.opendev.org/c/openstack/swift/+/901333/1?usp\u003drelated-change","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":391,"context_line":"            update_shard_ns \u003d self._get_update_shard("},{"line_number":392,"context_line":"                req, self.account_name, self.container_name, self.object_name)"},{"line_number":393,"context_line":"            if update_shard_ns:"},{"line_number":394,"context_line":"                account, container \u003d update_shard_ns.name.split(\u0027/\u0027)"},{"line_number":395,"context_line":"                partition, nodes \u003d self.app.container_ring.get_nodes("},{"line_number":396,"context_line":"                    account, container)"},{"line_number":397,"context_line":"                return partition, nodes, update_shard_ns.name"}],"source_content_type":"text/x-python","patch_set":11,"id":"4e835b1b_4074bdcf","line":394,"updated":"2023-11-19 00:57:26.000000000","message":"helper function to Namespace class","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8cca4f513237737d7af9d3e867c518458b53bae2","unresolved":false,"context_lines":[{"line_number":391,"context_line":"            update_shard_ns \u003d self._get_update_shard("},{"line_number":392,"context_line":"                req, self.account_name, self.container_name, self.object_name)"},{"line_number":393,"context_line":"            if update_shard_ns:"},{"line_number":394,"context_line":"                account, container \u003d update_shard_ns.name.split(\u0027/\u0027)"},{"line_number":395,"context_line":"                partition, nodes \u003d self.app.container_ring.get_nodes("},{"line_number":396,"context_line":"                    account, container)"},{"line_number":397,"context_line":"                return partition, nodes, update_shard_ns.name"}],"source_content_type":"text/x-python","patch_set":11,"id":"b6e2395f_20c56e7a","line":394,"in_reply_to":"4e835b1b_4074bdcf","updated":"2023-11-22 07:05:19.000000000","message":"Done","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":294,"context_line":"        :param includes: (optional) restricts the list of fetched namespaces"},{"line_number":295,"context_line":"            to those which include the given name."},{"line_number":296,"context_line":"        :return: a list of instances of :class:`swift.common.utils.Namespace`,"},{"line_number":297,"context_line":"            or None if there was a problem fetching the shard ranges"},{"line_number":298,"context_line":"        \"\"\""},{"line_number":299,"context_line":"        params \u003d req.params.copy()"},{"line_number":300,"context_line":"        params.pop(\u0027limit\u0027, None)"}],"source_content_type":"text/x-python","patch_set":17,"id":"21e09a2b_ff34aa0d","line":297,"updated":"2023-11-29 21:56:41.000000000","message":"unrelated: I don\u0027t really understand the different utility for \"list of Namespace objects\" vs a `NamespaceBoundList`\n\nit seems like the `NamespaceBoundList` is just a more compact representation of the \"list of Namespaces\" - and it supports both \"interfaces\" we need:\n\n    get_namespace(object) \u003d\u003e for obj/updating\n    get_namespaces() \u003d\u003e for container/listing\n\nI don\u0027t see where we have a use-case for \"list of Namespaces\" that wouldn\u0027t be better served by the cache representative `NamespaceBoundList`","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":294,"context_line":"        :param includes: (optional) restricts the list of fetched namespaces"},{"line_number":295,"context_line":"            to those which include the given name."},{"line_number":296,"context_line":"        :return: a list of instances of :class:`swift.common.utils.Namespace`,"},{"line_number":297,"context_line":"            or None if there was a problem fetching the shard ranges"},{"line_number":298,"context_line":"        \"\"\""},{"line_number":299,"context_line":"        params \u003d req.params.copy()"},{"line_number":300,"context_line":"        params.pop(\u0027limit\u0027, None)"}],"source_content_type":"text/x-python","patch_set":17,"id":"b3414459_dd049bfe","line":297,"in_reply_to":"21e09a2b_ff34aa0d","updated":"2023-11-30 15:43:06.000000000","message":"NamespaceBoundList is only useful to represent a list of namespaces that cover the *entire namespace*. So, when we only get back one Namespace dict (with an includes header), we cannot convert that to a NamespaceBoundList.\n\nThat said, there may be some paths that could dispense with the conversion to list. Perhaps we can explore that more in the next patch?","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":342,"context_line":"        :param container: container from which shard ranges should be fetched."},{"line_number":343,"context_line":"        :param obj: object getting updated."},{"line_number":344,"context_line":"        :return: an instance of :class:`swift.common.utils.ShardRange`,"},{"line_number":345,"context_line":"            or None if the update should go back to the root"},{"line_number":346,"context_line":"        \"\"\""},{"line_number":347,"context_line":"        if not self.app.recheck_updating_shard_ranges:"},{"line_number":348,"context_line":"            # caching is disabled"}],"source_content_type":"text/x-python","patch_set":17,"id":"d56e1deb_e0f110ab","line":345,"updated":"2023-11-29 21:56:41.000000000","message":"seems like this is a Namesapce now\n\nor at least in unittests it\u0027s always the case that isinstance(update_shard_ns, Namespace) in _get_update_target (if it\u0027s not None)","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":342,"context_line":"        :param container: container from which shard ranges should be fetched."},{"line_number":343,"context_line":"        :param obj: object getting updated."},{"line_number":344,"context_line":"        :return: an instance of :class:`swift.common.utils.ShardRange`,"},{"line_number":345,"context_line":"            or None if the update should go back to the root"},{"line_number":346,"context_line":"        \"\"\""},{"line_number":347,"context_line":"        if not self.app.recheck_updating_shard_ranges:"},{"line_number":348,"context_line":"            # caching is disabled"}],"source_content_type":"text/x-python","patch_set":17,"id":"e2349711_959315ef","line":345,"in_reply_to":"d56e1deb_e0f110ab","updated":"2023-11-30 15:43:06.000000000","message":"Done","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":376,"context_line":"                    self.logger.info("},{"line_number":377,"context_line":"                        \u0027Caching updating shards for %s (%d shards)\u0027,"},{"line_number":378,"context_line":"                        cache_key, len(namespaces))"},{"line_number":379,"context_line":"            update_shard_ns \u003d find_namespace(obj, namespaces or [])"},{"line_number":380,"context_line":"        record_cache_op_metrics("},{"line_number":381,"context_line":"            self.logger, self.server_type.lower(), \u0027shard_updating\u0027,"},{"line_number":382,"context_line":"            get_cache_state, response)"}],"source_content_type":"text/x-python","patch_set":17,"id":"2a071e4c_279d2173","line":379,"updated":"2023-11-29 21:56:41.000000000","message":"seems unoptimized to use find_namespace here if we have a ns_bounds_list","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":376,"context_line":"                    self.logger.info("},{"line_number":377,"context_line":"                        \u0027Caching updating shards for %s (%d shards)\u0027,"},{"line_number":378,"context_line":"                        cache_key, len(namespaces))"},{"line_number":379,"context_line":"            update_shard_ns \u003d find_namespace(obj, namespaces or [])"},{"line_number":380,"context_line":"        record_cache_op_metrics("},{"line_number":381,"context_line":"            self.logger, self.server_type.lower(), \u0027shard_updating\u0027,"},{"line_number":382,"context_line":"            get_cache_state, response)"}],"source_content_type":"text/x-python","patch_set":17,"id":"68c65c8f_ec059c95","line":379,"in_reply_to":"2a071e4c_279d2173","updated":"2023-11-30 15:43:06.000000000","message":"genuinely curious how one is more optimized: both would need a bisect?","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"597dbf8fe6305bd4ad5f883f7a6a26234c32765d","unresolved":false,"context_lines":[{"line_number":390,"context_line":"                req, self.account_name, self.container_name, self.object_name)"},{"line_number":391,"context_line":"            if update_shard_ns:"},{"line_number":392,"context_line":"                partition, nodes \u003d self.app.container_ring.get_nodes("},{"line_number":393,"context_line":"                    update_shard_ns.account, update_shard_ns.container)"},{"line_number":394,"context_line":"                return partition, nodes, update_shard_ns.name"},{"line_number":395,"context_line":""},{"line_number":396,"context_line":"        return container_info[\u0027partition\u0027], container_info[\u0027nodes\u0027], None"}],"source_content_type":"text/x-python","patch_set":24,"id":"85775432_8dae8d7b","line":393,"updated":"2023-12-07 19:32:24.000000000","message":"For a container with 10K shards, ``_get_update_shard`` will iterate all of them and only return one Namespace. So switching from ``shard_range.account/container`` to ``update_shard_ns.account/container`` will only add two more string split calls. Not a concern to performance in this critical path.","commit_id":"a6e9bb549c6acf98d4d4092565a1ac545d4f02e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"69255e1e707ef5ee3d617e20900aeef4f0daafe7","unresolved":true,"context_lines":[{"line_number":306,"context_line":"            params[\u0027includes\u0027] \u003d str_to_wsgi(includes)"},{"line_number":307,"context_line":"            # this line can be removed as soon as the container server"},{"line_number":308,"context_line":"            # supports get_namespaces api with \u0027includes\u0027."},{"line_number":309,"context_line":"            headers[\u0027X-Backend-Record-Shard-Format\u0027] \u003d \u0027full\u0027"},{"line_number":310,"context_line":"        listing, response \u003d self._get_container_listing("},{"line_number":311,"context_line":"            req, account, container, headers\u003dheaders, params\u003dparams)"},{"line_number":312,"context_line":"        return self._parse_shard_ranges(req, listing, response), response"}],"source_content_type":"text/x-python","patch_set":27,"id":"fc88a2d0_e0161cee","side":"PARENT","line":309,"updated":"2023-12-14 05:17:38.000000000","message":"\\o/","commit_id":"c5ea203be8f91f270cb1b82404c43316d15360e8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23a1aef2dbdb36a45b67572ffa1bd0a5f4f75c3f","unresolved":false,"context_lines":[{"line_number":306,"context_line":"            params[\u0027includes\u0027] \u003d str_to_wsgi(includes)"},{"line_number":307,"context_line":"            # this line can be removed as soon as the container server"},{"line_number":308,"context_line":"            # supports get_namespaces api with \u0027includes\u0027."},{"line_number":309,"context_line":"            headers[\u0027X-Backend-Record-Shard-Format\u0027] \u003d \u0027full\u0027"},{"line_number":310,"context_line":"        listing, response \u003d self._get_container_listing("},{"line_number":311,"context_line":"            req, account, container, headers\u003dheaders, params\u003dparams)"},{"line_number":312,"context_line":"        return self._parse_shard_ranges(req, listing, response), response"}],"source_content_type":"text/x-python","patch_set":27,"id":"27ac9c91_624f9c2b","side":"PARENT","line":309,"in_reply_to":"fc88a2d0_e0161cee","updated":"2024-01-04 16:33:04.000000000","message":"Acknowledged","commit_id":"c5ea203be8f91f270cb1b82404c43316d15360e8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"69255e1e707ef5ee3d617e20900aeef4f0daafe7","unresolved":true,"context_lines":[{"line_number":286,"context_line":"        \"\"\""},{"line_number":287,"context_line":"        Fetch namespaces in \u0027updating\u0027 states from given `account/container`."},{"line_number":288,"context_line":"        If `includes` is given then the shard range for that object name is"},{"line_number":289,"context_line":"        requested, otherwise all shard ranges are requested."},{"line_number":290,"context_line":""},{"line_number":291,"context_line":"        :param req: original Request instance."},{"line_number":292,"context_line":"        :param account: account from which namespaces should be fetched."}],"source_content_type":"text/x-python","patch_set":27,"id":"a262c32a_1e74d290","line":289,"range":{"start_line":289,"start_character":33,"end_line":289,"end_character":45},"updated":"2023-12-14 05:17:38.000000000","message":"namespaces? but I guess we could use them interchangably","commit_id":"9360a785c5936ea4af627cc3426cb3f79ca0577d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"23a1aef2dbdb36a45b67572ffa1bd0a5f4f75c3f","unresolved":false,"context_lines":[{"line_number":286,"context_line":"        \"\"\""},{"line_number":287,"context_line":"        Fetch namespaces in \u0027updating\u0027 states from given `account/container`."},{"line_number":288,"context_line":"        If `includes` is given then the shard range for that object name is"},{"line_number":289,"context_line":"        requested, otherwise all shard ranges are requested."},{"line_number":290,"context_line":""},{"line_number":291,"context_line":"        :param req: original Request instance."},{"line_number":292,"context_line":"        :param account: account from which namespaces should be fetched."}],"source_content_type":"text/x-python","patch_set":27,"id":"faebcc27_baecfcc4","line":289,"range":{"start_line":289,"start_character":33,"end_line":289,"end_character":45},"in_reply_to":"a262c32a_1e74d290","updated":"2024-01-04 16:33:04.000000000","message":"Done","commit_id":"9360a785c5936ea4af627cc3426cb3f79ca0577d"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":2928,"context_line":"        self.assertEqual(0, int(metadata.get(\u0027x-account-bytes-used\u0027)))"},{"line_number":2929,"context_line":""},{"line_number":2930,"context_line":""},{"line_number":2931,"context_line":"class TestShardedAPI(BaseAutoContainerSharding):"},{"line_number":2932,"context_line":"    def test_GET(self):"},{"line_number":2933,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"},{"line_number":2934,"context_line":"        self.put_objects(all_obj_names)"}],"source_content_type":"text/x-python","patch_set":3,"id":"38a1a6f0_2904a637","line":2931,"updated":"2023-11-09 18:12:54.000000000","message":"IDK if this is really a \u0027probe\u0027 test, but we can\u0027t test against a sharded container with functional tests.","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":2928,"context_line":"        self.assertEqual(0, int(metadata.get(\u0027x-account-bytes-used\u0027)))"},{"line_number":2929,"context_line":""},{"line_number":2930,"context_line":""},{"line_number":2931,"context_line":"class TestShardedAPI(BaseAutoContainerSharding):"},{"line_number":2932,"context_line":"    def test_GET(self):"},{"line_number":2933,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"},{"line_number":2934,"context_line":"        self.put_objects(all_obj_names)"}],"source_content_type":"text/x-python","patch_set":3,"id":"cefac0aa_af6f7c7f","line":2931,"in_reply_to":"38a1a6f0_2904a637","updated":"2023-11-16 22:41:19.000000000","message":"sounds good to me!\n\nit seems like we mostly use internal client and sometimes x-backend headers; so this is a \"super set\" of public/proxy API (which is maybe simple/borning in comparison)","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":2928,"context_line":"        self.assertEqual(0, int(metadata.get(\u0027x-account-bytes-used\u0027)))"},{"line_number":2929,"context_line":""},{"line_number":2930,"context_line":""},{"line_number":2931,"context_line":"class TestShardedAPI(BaseAutoContainerSharding):"},{"line_number":2932,"context_line":"    def test_GET(self):"},{"line_number":2933,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"},{"line_number":2934,"context_line":"        self.put_objects(all_obj_names)"}],"source_content_type":"text/x-python","patch_set":3,"id":"790ffd39_12c27515","line":2931,"in_reply_to":"cefac0aa_af6f7c7f","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":3010,"context_line":""},{"line_number":3011,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3012,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3013,"context_line":"        self.assertEqual([], objs)"},{"line_number":3014,"context_line":""},{"line_number":3015,"context_line":""},{"line_number":3016,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":3,"id":"8906b464_3c166ae3","line":3013,"updated":"2023-11-09 18:12:54.000000000","message":"add comment: the root has no object rows","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":3010,"context_line":""},{"line_number":3011,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3012,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3013,"context_line":"        self.assertEqual([], objs)"},{"line_number":3014,"context_line":""},{"line_number":3015,"context_line":""},{"line_number":3016,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":3,"id":"5ab2996b_47aca546","line":3013,"in_reply_to":"8906b464_3c166ae3","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ba206970edf6b09a2789b15b0e820de547ff1f5d","unresolved":true,"context_lines":[{"line_number":2950,"context_line":"                [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":2951,"context_line":"                 for sr in that])"},{"line_number":2952,"context_line":""},{"line_number":2953,"context_line":"        orig_shard_ranges \u003d self.get_container_shard_ranges()"},{"line_number":2954,"context_line":"        self.assertEqual(2, len(orig_shard_ranges))"},{"line_number":2955,"context_line":"        namespaces \u003d self.get_container_namespaces()"},{"line_number":2956,"context_line":"        check_equivalence(orig_shard_ranges, namespaces)"}],"source_content_type":"text/x-python","patch_set":5,"id":"76290533_0a3e0ccc","line":2953,"range":{"start_line":2953,"start_character":27,"end_line":2953,"end_character":61},"updated":"2023-11-14 04:05:42.000000000","message":"Cool so this will use ShardRange.from_dict() to build the shardranges. This would fail if we didn\u0027t get full SR dicts back. So shows we haven\u0027t broken the internal client get full shard ranges path the sharder uses.","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":2950,"context_line":"                [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":2951,"context_line":"                 for sr in that])"},{"line_number":2952,"context_line":""},{"line_number":2953,"context_line":"        orig_shard_ranges \u003d self.get_container_shard_ranges()"},{"line_number":2954,"context_line":"        self.assertEqual(2, len(orig_shard_ranges))"},{"line_number":2955,"context_line":"        namespaces \u003d self.get_container_namespaces()"},{"line_number":2956,"context_line":"        check_equivalence(orig_shard_ranges, namespaces)"}],"source_content_type":"text/x-python","patch_set":5,"id":"8439a49f_c9c6b40b","line":2953,"range":{"start_line":2953,"start_character":27,"end_line":2953,"end_character":61},"in_reply_to":"76290533_0a3e0ccc","updated":"2023-11-14 18:42:53.000000000","message":"Ack","commit_id":"8b9925c5a8dfae8c56adb9fb952558b381ce4c0d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b7f9ebd72672668fde3117a12f28303b45a985c1","unresolved":true,"context_lines":[{"line_number":2930,"context_line":""},{"line_number":2931,"context_line":"class TestShardedAPI(BaseAutoContainerSharding):"},{"line_number":2932,"context_line":"    def test_GET(self):"},{"line_number":2933,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"},{"line_number":2934,"context_line":"        self.put_objects(all_obj_names)"},{"line_number":2935,"context_line":"        # Shard the container"},{"line_number":2936,"context_line":"        client.post_container(self.url, self.admin_token, self.container_name,"}],"source_content_type":"text/x-python","patch_set":9,"id":"fccb32d4_2c3c03e6","line":2933,"updated":"2023-11-16 22:41:19.000000000","message":"ah, and that\u0027s why we have exactly two shards","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":2930,"context_line":""},{"line_number":2931,"context_line":"class TestShardedAPI(BaseAutoContainerSharding):"},{"line_number":2932,"context_line":"    def test_GET(self):"},{"line_number":2933,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"},{"line_number":2934,"context_line":"        self.put_objects(all_obj_names)"},{"line_number":2935,"context_line":"        # Shard the container"},{"line_number":2936,"context_line":"        client.post_container(self.url, self.admin_token, self.container_name,"}],"source_content_type":"text/x-python","patch_set":9,"id":"d7b877af_9fb32c64","line":2933,"in_reply_to":"fccb32d4_2c3c03e6","updated":"2023-11-20 14:38:32.000000000","message":"Acknowledged","commit_id":"45d8510a56e9a4fd40cfab3949f78956dac1d80a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":199,"context_line":"        self.assertEqual(\u0027shard\u0027, resp.headers.get(\u0027X-Backend-Record-Type\u0027))"},{"line_number":200,"context_line":"        self.assertEqual(\u0027full\u0027,"},{"line_number":201,"context_line":"                         resp.headers.get(\u0027X-Backend-Record-Shard-Format\u0027))"},{"line_number":202,"context_line":"        return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)]"},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"    def get_container_namespaces(self, account\u003dNone, container\u003dNone,"},{"line_number":205,"context_line":"                                 headers\u003dNone, params\u003dNone):"}],"source_content_type":"text/x-python","patch_set":17,"id":"1b888a9f_f1f19182","line":202,"updated":"2023-11-29 21:56:41.000000000","message":"oic, ShardRange.from_dict(sr) blows up if the objects in the listing don\u0027t have a timestamp key\n\n        @classmethod\n        def from_dict(cls, params):\n            \"\"\"\n            Return an instance constructed using the given dict of params. This\n            method is deliberately less flexible than the class `__init__()` method\n            and requires all of the `__init__()` args to be given in the dict of\n            params.\n        \n            :param params: a dict of parameters\n            :return: an instance of this class\n            \"\"\"\n            return cls(\n    \u003e           params[\u0027name\u0027], params[\u0027timestamp\u0027], params[\u0027lower\u0027],\n                params[\u0027upper\u0027], params[\u0027object_count\u0027], params[\u0027bytes_used\u0027],\n                params[\u0027meta_timestamp\u0027], params[\u0027deleted\u0027], params[\u0027state\u0027],\n                params[\u0027state_timestamp\u0027], params[\u0027epoch\u0027],\n                params.get(\u0027reported\u0027, 0), params.get(\u0027tombstones\u0027, -1))\n    E       KeyError: \u0027timestamp\u0027","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":199,"context_line":"        self.assertEqual(\u0027shard\u0027, resp.headers.get(\u0027X-Backend-Record-Type\u0027))"},{"line_number":200,"context_line":"        self.assertEqual(\u0027full\u0027,"},{"line_number":201,"context_line":"                         resp.headers.get(\u0027X-Backend-Record-Shard-Format\u0027))"},{"line_number":202,"context_line":"        return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)]"},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"    def get_container_namespaces(self, account\u003dNone, container\u003dNone,"},{"line_number":205,"context_line":"                                 headers\u003dNone, params\u003dNone):"}],"source_content_type":"text/x-python","patch_set":17,"id":"31700ea8_7f890d1c","line":202,"in_reply_to":"1b888a9f_f1f19182","updated":"2023-11-30 15:43:06.000000000","message":"yes, I think this was touched on in the probe test patch, hence the XXX\u0027","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":2971,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2972,"context_line":"        # headers \u003d {\u0027X-Newest\u0027: \u0027true\u0027}"},{"line_number":2973,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":2974,"context_line":"        #                                                params\u003dparams)"},{"line_number":2975,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2976,"context_line":""},{"line_number":2977,"context_line":"        # this is what the sharder requests..."}],"source_content_type":"text/x-python","patch_set":17,"id":"c57f49c3_284b4878","line":2974,"updated":"2023-11-29 21:56:41.000000000","message":"For me these are still failing","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":2971,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2972,"context_line":"        # headers \u003d {\u0027X-Newest\u0027: \u0027true\u0027}"},{"line_number":2973,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":2974,"context_line":"        #                                                params\u003dparams)"},{"line_number":2975,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2976,"context_line":""},{"line_number":2977,"context_line":"        # this is what the sharder requests..."}],"source_content_type":"text/x-python","patch_set":17,"id":"b4724798_fc23397f","line":2974,"in_reply_to":"c57f49c3_284b4878","updated":"2023-11-30 15:43:06.000000000","message":"yes, these are addressed by the next patch (only use the cache for \u0027auto\u0027 listings)","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3898c99945588cf57e61f7d08679aa0d041e0e18","unresolved":false,"context_lines":[{"line_number":197,"context_line":"        resp \u003d self.get_container_listing(account, container, headers,"},{"line_number":198,"context_line":"                                          params\u003dparams)"},{"line_number":199,"context_line":"        self.assertEqual(\u0027shard\u0027, resp.headers.get(\u0027X-Backend-Record-Type\u0027))"},{"line_number":200,"context_line":"        self.assertEqual(\u0027full\u0027,"},{"line_number":201,"context_line":"                         resp.headers.get(\u0027X-Backend-Record-Shard-Format\u0027))"},{"line_number":202,"context_line":"        return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)]"},{"line_number":203,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"54f71309_b4cd650a","line":200,"updated":"2023-11-30 05:46:11.000000000","message":"Here is one place would cause \u003e10 probe tests to fail, if I run them with only proxy ONLY namespace patches on top of master branch. After I changed to ``\u0027full\u0027`` to ``None``, those probe tests would pass again.\nSince this only happens to probe test, nothing to worry about.","commit_id":"8bd68b0cfc7d63d26f8abe4c041f9673202033ee"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3898c99945588cf57e61f7d08679aa0d041e0e18","unresolved":false,"context_lines":[{"line_number":209,"context_line":"        resp \u003d self.get_container_listing(account, container, headers,"},{"line_number":210,"context_line":"                                          params\u003dparams)"},{"line_number":211,"context_line":"        self.assertEqual(\u0027shard\u0027, resp.headers.get(\u0027X-Backend-Record-Type\u0027))"},{"line_number":212,"context_line":"        self.assertEqual(\u0027namespace\u0027,"},{"line_number":213,"context_line":"                         resp.headers.get(\u0027X-Backend-Record-Shard-Format\u0027))"},{"line_number":214,"context_line":"        return [Namespace(**ns) for ns in json.loads(resp.body)]"},{"line_number":215,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"1f3878d3_f0df9baf","line":212,"updated":"2023-11-30 05:46:11.000000000","message":"This is another place that would cause a probe test to fail, if run it with proxy ONLY namespace patches on top of master branch. Again, nothing to worry about. After I changed it to ``None``, then test passes.","commit_id":"8bd68b0cfc7d63d26f8abe4c041f9673202033ee"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3898c99945588cf57e61f7d08679aa0d041e0e18","unresolved":true,"context_lines":[{"line_number":350,"context_line":"        headers \u003d client.head_container("},{"line_number":351,"context_line":"            self.url, self.token, self.container_name)"},{"line_number":352,"context_line":"        self.assertIn(\u0027x-container-object-count\u0027, headers)"},{"line_number":353,"context_line":"        self.assertEqual(str(expected_obj_count),"},{"line_number":354,"context_line":"                         headers[\u0027x-container-object-count\u0027])"},{"line_number":355,"context_line":""},{"line_number":356,"context_line":"    def assert_container_post_ok(self, meta_value):"}],"source_content_type":"text/x-python","patch_set":18,"id":"f41d189f_cb98af58","line":353,"updated":"2023-11-30 05:46:11.000000000","message":"@Alistair, if I run below test with proxy ONLY namespace patches on top of master branch, here will throw out below assertion error. Need to investigate more. \n\n____________________________________________ TestContainerShardingMoreUTF8.test_shrinking ____________________________________________\n\nswift/test/probe/test_sharder.py:1934:\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _\nswift/test/probe/test_sharder.py:1799: in do_shard_then_shrink\n    self.assert_container_object_count(exp_obj_count)\nswift/test/probe/test_sharder.py:353: in assert_container_object_count\n    self.assertEqual(str(expected_obj_count),\nE   AssertionError: \u002726\u0027 !\u003d \u002750\u0027\nE   - 26\nE   + 50","commit_id":"8bd68b0cfc7d63d26f8abe4c041f9673202033ee"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"354dada70fc329c6d221615872540859e5a584d5","unresolved":false,"context_lines":[{"line_number":350,"context_line":"        headers \u003d client.head_container("},{"line_number":351,"context_line":"            self.url, self.token, self.container_name)"},{"line_number":352,"context_line":"        self.assertIn(\u0027x-container-object-count\u0027, headers)"},{"line_number":353,"context_line":"        self.assertEqual(str(expected_obj_count),"},{"line_number":354,"context_line":"                         headers[\u0027x-container-object-count\u0027])"},{"line_number":355,"context_line":""},{"line_number":356,"context_line":"    def assert_container_post_ok(self, meta_value):"}],"source_content_type":"text/x-python","patch_set":18,"id":"0c8c7e93_760eeb03","line":353,"in_reply_to":"f41d189f_cb98af58","updated":"2023-11-30 17:49:37.000000000","message":"This is a flaky, not able to reproduce it with more than 10 test runs.","commit_id":"8bd68b0cfc7d63d26f8abe4c041f9673202033ee"}],"test/unit/common/middleware/s3api/test_obj.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":1,"context_line":""},{"line_number":2,"context_line":"# Copyright (c) 2014 OpenStack Foundation"},{"line_number":3,"context_line":"#"},{"line_number":4,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"}],"source_content_type":"text/x-python","patch_set":3,"id":"eace3654_d62bf1f9","line":1,"updated":"2023-11-09 18:12:54.000000000","message":"oops!","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":1,"context_line":""},{"line_number":2,"context_line":"# Copyright (c) 2014 OpenStack Foundation"},{"line_number":3,"context_line":"#"},{"line_number":4,"context_line":"# Licensed under the Apache License, Version 2.0 (the \"License\");"}],"source_content_type":"text/x-python","patch_set":3,"id":"d7d08e66_3ebeec73","line":1,"in_reply_to":"eace3654_d62bf1f9","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"}],"test/unit/common/test_utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"03a6cb99f27e548ae62980b7b32fb6d8c7aa9311","unresolved":true,"context_lines":[{"line_number":7592,"context_line":"        else:"},{"line_number":7593,"context_line":"            self.assertEqual(exp_name, ns.name)"},{"line_number":7594,"context_line":"            self.assertEqual(exp_account, ns.account)"},{"line_number":7595,"context_line":"            self.assertEqual(exp_container, ns.container)"},{"line_number":7596,"context_line":""},{"line_number":7597,"context_line":"    def test_name(self):"},{"line_number":7598,"context_line":"        # constructor"}],"source_content_type":"text/x-python","patch_set":21,"id":"0caef68e_76a9910a","line":7595,"updated":"2023-12-04 16:39:40.000000000","message":"i.e. \"py2/py3 consistently stores values as native string\"","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"03a6cb99f27e548ae62980b7b32fb6d8c7aa9311","unresolved":true,"context_lines":[{"line_number":7629,"context_line":"        ns.container \u003d \u0027c2\u0027"},{"line_number":7630,"context_line":"        self._check_name_account_container(ns, \u0027a/c2\u0027)"},{"line_number":7631,"context_line":"        ns.container \u003d u\u0027\\n{SNOWMAN}\u0027"},{"line_number":7632,"context_line":"        self._check_name_account_container(ns, u\u0027a/\\n{SNOWMAN}\u0027)"},{"line_number":7633,"context_line":""},{"line_number":7634,"context_line":"    def test_unicode_name(self):"},{"line_number":7635,"context_line":"        shard_bounds \u003d (\u0027\u0027, \u0027ham\u0027, \u0027pie\u0027, u\u0027\\N{SNOWMAN}\u0027, u\u0027\\U0001F334\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":21,"id":"3fed3742_5224a45a","line":7632,"updated":"2023-12-04 16:39:40.000000000","message":"still seems to be lacking explicit tests for utf8 encoded non-ascii byte-strings (on py3, on py2 native strings are byte-strings and AFAIK the behavior is already consistent)","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"18d2f1fff47fe4b4711272ecad469d0bc96a23b8","unresolved":true,"context_lines":[{"line_number":7629,"context_line":"        ns.container \u003d \u0027c2\u0027"},{"line_number":7630,"context_line":"        self._check_name_account_container(ns, \u0027a/c2\u0027)"},{"line_number":7631,"context_line":"        ns.container \u003d u\u0027\\n{SNOWMAN}\u0027"},{"line_number":7632,"context_line":"        self._check_name_account_container(ns, u\u0027a/\\n{SNOWMAN}\u0027)"},{"line_number":7633,"context_line":""},{"line_number":7634,"context_line":"    def test_unicode_name(self):"},{"line_number":7635,"context_line":"        shard_bounds \u003d (\u0027\u0027, \u0027ham\u0027, \u0027pie\u0027, u\u0027\\N{SNOWMAN}\u0027, u\u0027\\U0001F334\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":21,"id":"ae83df1a_0a7c3d7a","line":7632,"in_reply_to":"3fed3742_5224a45a","updated":"2023-12-04 18:51:22.000000000","message":"I\u0027ll break out a pre-patch and fix there, my tomorrow","commit_id":"8212898c56041425d373364a36108cdd2adda343"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d3f745af5ed21230edd84eba02d53f1a88a42cef","unresolved":true,"context_lines":[{"line_number":7629,"context_line":"        ns.container \u003d \u0027c2\u0027"},{"line_number":7630,"context_line":"        self._check_name_account_container(ns, \u0027a/c2\u0027)"},{"line_number":7631,"context_line":"        ns.container \u003d u\u0027\\n{SNOWMAN}\u0027"},{"line_number":7632,"context_line":"        self._check_name_account_container(ns, u\u0027a/\\n{SNOWMAN}\u0027)"},{"line_number":7633,"context_line":""},{"line_number":7634,"context_line":"    def test_unicode_name(self):"},{"line_number":7635,"context_line":"        shard_bounds \u003d (\u0027\u0027, \u0027ham\u0027, \u0027pie\u0027, u\u0027\\N{SNOWMAN}\u0027, u\u0027\\U0001F334\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":21,"id":"8a6f91c2_ad807a9e","line":7632,"in_reply_to":"ae83df1a_0a7c3d7a","updated":"2023-12-05 12:31:37.000000000","message":"https://review.opendev.org/c/openstack/swift/+/902646","commit_id":"8212898c56041425d373364a36108cdd2adda343"}],"test/unit/container/test_server.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":3763,"context_line":"            self.assertEqual(resp.content_type, \u0027application/json\u0027)"},{"line_number":3764,"context_line":"            expected \u003d ["},{"line_number":3765,"context_line":"                dict(sr, last_modified\u003dTimestamp(sr.timestamp).isoformat)"},{"line_number":3766,"context_line":"                for sr in expected_shard_ranges]"},{"line_number":3767,"context_line":"            self.assertEqual(expected, json.loads(resp.body))"},{"line_number":3768,"context_line":"            self.assertIn(\u0027X-Backend-Record-Type\u0027, resp.headers)"},{"line_number":3769,"context_line":"            self.assertEqual(\u0027shard\u0027, resp.headers[\u0027X-Backend-Record-Type\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"3c9535b2_4a557faa","side":"PARENT","line":3766,"updated":"2023-11-09 18:12:54.000000000","message":"IMO this weird edge case motivates completing the job and supporting marker, end_marker etc for get_namespaces","commit_id":"9040ca6b1bae0fcc0752b7e88c5ffb81be0311ca"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":3769,"context_line":"            self.assertEqual("},{"line_number":3770,"context_line":"                \u0027namespace\u0027, resp.headers[\u0027X-Backend-Record-Shard-Format\u0027])"},{"line_number":3771,"context_line":""},{"line_number":3772,"context_line":"        # TODOD: failing because filler namespace is returned?"},{"line_number":3773,"context_line":"        namespaces \u003d [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":3774,"context_line":"                      for sr in shard_ranges]"},{"line_number":3775,"context_line":"        check_shard_GET(namespaces[:3], \u0027a/c\u0027,"}],"source_content_type":"text/x-python","patch_set":3,"id":"ecec34eb_4887bf94","line":3772,"updated":"2023-11-09 18:12:54.000000000","message":"lol @ TODOD\n\nI fixed this so it\u0027s NOTODOD anyway ;-)","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":false,"context_lines":[{"line_number":3769,"context_line":"            self.assertEqual("},{"line_number":3770,"context_line":"                \u0027namespace\u0027, resp.headers[\u0027X-Backend-Record-Shard-Format\u0027])"},{"line_number":3771,"context_line":""},{"line_number":3772,"context_line":"        # TODOD: failing because filler namespace is returned?"},{"line_number":3773,"context_line":"        namespaces \u003d [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":3774,"context_line":"                      for sr in shard_ranges]"},{"line_number":3775,"context_line":"        check_shard_GET(namespaces[:3], \u0027a/c\u0027,"}],"source_content_type":"text/x-python","patch_set":3,"id":"7fb77d5c_3422e4ad","line":3772,"in_reply_to":"ecec34eb_4887bf94","updated":"2023-11-14 18:42:53.000000000","message":"Done","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"}],"test/unit/proxy/controllers/test_container.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"702e5d5836c1c270c92ea6a9b62d32fa8b2081d5","unresolved":true,"context_lines":[{"line_number":2527,"context_line":"        with mock.patch(\u0027swift.proxy.controllers.container.\u0027"},{"line_number":2528,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"},{"line_number":2529,"context_line":"                        mock_get_from_shards):"},{"line_number":2530,"context_line":"            return req.get_response(self.app)"},{"line_number":2531,"context_line":""},{"line_number":2532,"context_line":"    def _build_request(self, headers, params, infocache\u003dNone):"},{"line_number":2533,"context_line":"        # helper to make a GET request with caches set in environ"}],"source_content_type":"text/x-python","patch_set":3,"id":"ef2f4441_06891510","line":2530,"updated":"2023-11-09 18:12:54.000000000","message":"this is the existing pattern for these tests, just refactoring it","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e79a831f87e7d90b084a91db2ec05883be62c43","unresolved":true,"context_lines":[{"line_number":2527,"context_line":"        with mock.patch(\u0027swift.proxy.controllers.container.\u0027"},{"line_number":2528,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"},{"line_number":2529,"context_line":"                        mock_get_from_shards):"},{"line_number":2530,"context_line":"            return req.get_response(self.app)"},{"line_number":2531,"context_line":""},{"line_number":2532,"context_line":"    def _build_request(self, headers, params, infocache\u003dNone):"},{"line_number":2533,"context_line":"        # helper to make a GET request with caches set in environ"}],"source_content_type":"text/x-python","patch_set":3,"id":"f89f2841_ee4bf664","line":2530,"in_reply_to":"ef2f4441_06891510","updated":"2023-11-14 18:42:53.000000000","message":"I\u0027ve tried to improve this a little by actually making fake_get_from_shards return a body with an object listing!","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":2527,"context_line":"        with mock.patch(\u0027swift.proxy.controllers.container.\u0027"},{"line_number":2528,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"},{"line_number":2529,"context_line":"                        mock_get_from_shards):"},{"line_number":2530,"context_line":"            return req.get_response(self.app)"},{"line_number":2531,"context_line":""},{"line_number":2532,"context_line":"    def _build_request(self, headers, params, infocache\u003dNone):"},{"line_number":2533,"context_line":"        # helper to make a GET request with caches set in environ"}],"source_content_type":"text/x-python","patch_set":3,"id":"0df34342_f97afc02","line":2530,"in_reply_to":"f89f2841_ee4bf664","updated":"2023-11-30 15:43:06.000000000","message":"Acknowledged","commit_id":"52ae83fecf9f2a521a66a82d2b44701f8a2082bc"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":2940,"context_line":"        self.assertEqual("},{"line_number":2941,"context_line":"            captured_hdrs[\u0027X-Backend-Storage-Policy-Index\u0027], \u00270\u0027)"},{"line_number":2942,"context_line":""},{"line_number":2943,"context_line":"    def test_GET_shard_ranges(self):"},{"line_number":2944,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":2945,"context_line":"        # expect shard ranges cache time to be default value of 600"},{"line_number":2946,"context_line":"        self._do_test_caching(\u0027shard\u0027, 600)"}],"source_content_type":"text/x-python","patch_set":11,"id":"6162d826_5e810ae6","side":"PARENT","line":2943,"updated":"2023-11-19 00:57:26.000000000","message":"why you deleted this test case, after moving mocking into a helper function?","commit_id":"1bc35ca77f09840562cb27364a0cc3e9827ccc4a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":2940,"context_line":"        self.assertEqual("},{"line_number":2941,"context_line":"            captured_hdrs[\u0027X-Backend-Storage-Policy-Index\u0027], \u00270\u0027)"},{"line_number":2942,"context_line":""},{"line_number":2943,"context_line":"    def test_GET_shard_ranges(self):"},{"line_number":2944,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":2945,"context_line":"        # expect shard ranges cache time to be default value of 600"},{"line_number":2946,"context_line":"        self._do_test_caching(\u0027shard\u0027, 600)"}],"source_content_type":"text/x-python","patch_set":11,"id":"7882d1e7_ce95c571","side":"PARENT","line":2943,"in_reply_to":"6162d826_5e810ae6","updated":"2023-11-20 14:38:32.000000000","message":"that\u0027s a mistake from splitting the patch up! will fix","commit_id":"1bc35ca77f09840562cb27364a0cc3e9827ccc4a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":1237,"context_line":"        expected_objects \u003d all_objects[first_included:]"},{"line_number":1238,"context_line":"        mock_responses \u003d ["},{"line_number":1239,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1240,"context_line":"            # NB: root returns complete shard listing despite marker"},{"line_number":1241,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":1242,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1243,"context_line":"            (200, sr_objs[3][2:], shard_resp_hdrs[3]),"}],"source_content_type":"text/x-python","patch_set":11,"id":"34fd00af_1a427feb","line":1240,"updated":"2023-11-19 00:57:26.000000000","message":"so we have a bug in the container marker/end_marker patch?","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a8b539bbad3653e175a5cc4606c55a5ab5919bc6","unresolved":false,"context_lines":[{"line_number":1237,"context_line":"        expected_objects \u003d all_objects[first_included:]"},{"line_number":1238,"context_line":"        mock_responses \u003d ["},{"line_number":1239,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1240,"context_line":"            # NB: root returns complete shard listing despite marker"},{"line_number":1241,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":1242,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1243,"context_line":"            (200, sr_objs[3][2:], shard_resp_hdrs[3]),"}],"source_content_type":"text/x-python","patch_set":11,"id":"08ceb289_42f9eb5f","line":1240,"in_reply_to":"34c35289_074f52b6","updated":"2023-11-28 05:54:41.000000000","message":"okay, I finally got it after looking into the implementation of ``_check_GET_shard_listing``, this is mock only and it makes sense to return all the sr_dicts in this case. Thanks for explaining it!","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":1237,"context_line":"        expected_objects \u003d all_objects[first_included:]"},{"line_number":1238,"context_line":"        mock_responses \u003d ["},{"line_number":1239,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1240,"context_line":"            # NB: root returns complete shard listing despite marker"},{"line_number":1241,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":1242,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1243,"context_line":"            (200, sr_objs[3][2:], shard_resp_hdrs[3]),"}],"source_content_type":"text/x-python","patch_set":11,"id":"cefc4ae4_8506b90f","line":1240,"in_reply_to":"34fd00af_1a427feb","updated":"2023-11-20 14:38:32.000000000","message":"I don\u0027t think so: when memcache is available, the proxy sends ``X-Backend-Override-Shard-Name-Filter \u003d sharded`` and the backend returns the complete listing\n\nI\u0027ll add to the comment","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8cca4f513237737d7af9d3e867c518458b53bae2","unresolved":true,"context_lines":[{"line_number":1237,"context_line":"        expected_objects \u003d all_objects[first_included:]"},{"line_number":1238,"context_line":"        mock_responses \u003d ["},{"line_number":1239,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1240,"context_line":"            # NB: root returns complete shard listing despite marker"},{"line_number":1241,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":1242,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1243,"context_line":"            (200, sr_objs[3][2:], shard_resp_hdrs[3]),"}],"source_content_type":"text/x-python","patch_set":11,"id":"ed84f5f7_feee27f9","line":1240,"in_reply_to":"cefc4ae4_8506b90f","updated":"2023-11-22 07:05:19.000000000","message":"sorry I still haven\u0027t got it. but why previously this case returned ``sr_dicts[3:]`` for the shard range GET, it also had ``X-Backend-Override-Shard-Name-Filter \u003d sharded``?","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebe11d824b643b5c91546364850080fc7a59ab5b","unresolved":true,"context_lines":[{"line_number":1237,"context_line":"        expected_objects \u003d all_objects[first_included:]"},{"line_number":1238,"context_line":"        mock_responses \u003d ["},{"line_number":1239,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1240,"context_line":"            # NB: root returns complete shard listing despite marker"},{"line_number":1241,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":1242,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":1243,"context_line":"            (200, sr_objs[3][2:], shard_resp_hdrs[3]),"}],"source_content_type":"text/x-python","patch_set":11,"id":"34c35289_074f52b6","line":1240,"in_reply_to":"ed84f5f7_feee27f9","updated":"2023-11-22 12:39:23.000000000","message":"I think the ``sr_dicts[3:]`` in the test \u0027s mock response was not representative of how the real backend would respond. We assert that the backend received ``X-Backend-Override-Shard-Name-Filter \u003d sharded``, the backend is assumed to be sharded, so it should respond with all the sr_dicts.\n\nI\u0027m worried now that I am missing something? Please push back if you think this change is wrong :)","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8cca4f513237737d7af9d3e867c518458b53bae2","unresolved":true,"context_lines":[{"line_number":1483,"context_line":"        # verify that if request params result in the filtered namespaces list"},{"line_number":1484,"context_line":"        # being empty the response body still has an empty object list"},{"line_number":1485,"context_line":"        shard_bounds \u003d ((\u0027a\u0027, \u0027b\u0027), (\u0027b\u0027, \u0027c\u0027), (\u0027c\u0027, \u0027\u0027))"},{"line_number":1486,"context_line":"        shard_ranges, sr_dicts, _ \u003d self.create_server_response_data("},{"line_number":1487,"context_line":"            shard_bounds, remove_char\u003d\u0027/\u0027)"},{"line_number":1488,"context_line":"        root_resp_hdrs, root_shard_resp_hdrs \u003d self._make_root_resp_hdrs("},{"line_number":1489,"context_line":"            6, 12, extra_shard_hdrs\u003d{"}],"source_content_type":"text/x-python","patch_set":11,"id":"f696b2ab_2ad3e3f6","line":1486,"updated":"2023-11-22 07:05:19.000000000","message":"s/shard_ranges/namespaces, and s/sr_dicts/ns_dicts? Since this patch is converting proxy from using shard ranges to namespaces, probably we should convert all occurrences of ``create_server_response_data`` in this file to use namespaces too.","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebe11d824b643b5c91546364850080fc7a59ab5b","unresolved":false,"context_lines":[{"line_number":1483,"context_line":"        # verify that if request params result in the filtered namespaces list"},{"line_number":1484,"context_line":"        # being empty the response body still has an empty object list"},{"line_number":1485,"context_line":"        shard_bounds \u003d ((\u0027a\u0027, \u0027b\u0027), (\u0027b\u0027, \u0027c\u0027), (\u0027c\u0027, \u0027\u0027))"},{"line_number":1486,"context_line":"        shard_ranges, sr_dicts, _ \u003d self.create_server_response_data("},{"line_number":1487,"context_line":"            shard_bounds, remove_char\u003d\u0027/\u0027)"},{"line_number":1488,"context_line":"        root_resp_hdrs, root_shard_resp_hdrs \u003d self._make_root_resp_hdrs("},{"line_number":1489,"context_line":"            6, 12, extra_shard_hdrs\u003d{"}],"source_content_type":"text/x-python","patch_set":11,"id":"ff2c0723_e73d4eb5","line":1486,"in_reply_to":"f696b2ab_2ad3e3f6","updated":"2023-11-22 12:39:23.000000000","message":"Done","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":2914,"context_line":"            self.memcache.calls)"},{"line_number":2915,"context_line":""},{"line_number":2916,"context_line":"    def test_get_from_shards_add_root_spi(self):"},{"line_number":2917,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":2918,"context_line":"        shard_resp \u003d mock.MagicMock(status_int\u003d204, headers\u003d{})"},{"line_number":2919,"context_line":""},{"line_number":2920,"context_line":"        def mock_get_container_listing(self_, req, *args, **kargs):"}],"source_content_type":"text/x-python","patch_set":11,"id":"bbcf1e42_befa8536","line":2917,"updated":"2023-11-19 00:57:26.000000000","message":"no need to call it anymore, since it\u0027s called in ``setUp``?","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":2914,"context_line":"            self.memcache.calls)"},{"line_number":2915,"context_line":""},{"line_number":2916,"context_line":"    def test_get_from_shards_add_root_spi(self):"},{"line_number":2917,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":2918,"context_line":"        shard_resp \u003d mock.MagicMock(status_int\u003d204, headers\u003d{})"},{"line_number":2919,"context_line":""},{"line_number":2920,"context_line":"        def mock_get_container_listing(self_, req, *args, **kargs):"}],"source_content_type":"text/x-python","patch_set":11,"id":"473e896e_bfda0538","line":2917,"in_reply_to":"bbcf1e42_befa8536","updated":"2023-11-20 14:38:32.000000000","message":"thanks! Done","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":2998,"context_line":"    def test_GET_shard_ranges_404_response(self):"},{"line_number":2999,"context_line":"        # pre-warm cache with container info but not shard ranges so that the"},{"line_number":3000,"context_line":"        # backend request tries to get a cacheable listing, but backend 404\u0027s"},{"line_number":3001,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":3002,"context_line":"        self.memcache.delete_all()"},{"line_number":3003,"context_line":"        info \u003d headers_to_container_info(self.root_resp_hdrs)"},{"line_number":3004,"context_line":"        info[\u0027status\u0027] \u003d 200"}],"source_content_type":"text/x-python","patch_set":11,"id":"de48d3b5_84bd08e0","line":3001,"updated":"2023-11-19 00:57:26.000000000","message":"ditto, no need to call it anymore?","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":2998,"context_line":"    def test_GET_shard_ranges_404_response(self):"},{"line_number":2999,"context_line":"        # pre-warm cache with container info but not shard ranges so that the"},{"line_number":3000,"context_line":"        # backend request tries to get a cacheable listing, but backend 404\u0027s"},{"line_number":3001,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":3002,"context_line":"        self.memcache.delete_all()"},{"line_number":3003,"context_line":"        info \u003d headers_to_container_info(self.root_resp_hdrs)"},{"line_number":3004,"context_line":"        info[\u0027status\u0027] \u003d 200"}],"source_content_type":"text/x-python","patch_set":11,"id":"9c720719_b085a11d","line":3001,"in_reply_to":"de48d3b5_84bd08e0","updated":"2023-11-20 14:38:32.000000000","message":"Done","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b82552f6a0f3020213e64097cd28b5cdfeaf555","unresolved":true,"context_lines":[{"line_number":3247,"context_line":"        # when x-newest is sent, verify that there is no cache lookup to check"},{"line_number":3248,"context_line":"        # sharding state but then backend requests are made requesting complete"},{"line_number":3249,"context_line":"        # shard list which can be cached"},{"line_number":3250,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":3251,"context_line":"        self.memcache.delete_all()"},{"line_number":3252,"context_line":"        self.memcache.clear_calls()"},{"line_number":3253,"context_line":"        req_hdrs \u003d {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"}],"source_content_type":"text/x-python","patch_set":11,"id":"36184a6c_d3f2953c","line":3250,"updated":"2023-11-19 00:57:26.000000000","message":"ditto","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9407367ca3abf0d598aa255fdfb42bdc7f1523ec","unresolved":false,"context_lines":[{"line_number":3247,"context_line":"        # when x-newest is sent, verify that there is no cache lookup to check"},{"line_number":3248,"context_line":"        # sharding state but then backend requests are made requesting complete"},{"line_number":3249,"context_line":"        # shard list which can be cached"},{"line_number":3250,"context_line":"        self._setup_shard_range_stubs()"},{"line_number":3251,"context_line":"        self.memcache.delete_all()"},{"line_number":3252,"context_line":"        self.memcache.clear_calls()"},{"line_number":3253,"context_line":"        req_hdrs \u003d {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"}],"source_content_type":"text/x-python","patch_set":11,"id":"d15b67cc_61544a33","line":3250,"in_reply_to":"36184a6c_d3f2953c","updated":"2023-11-20 14:38:32.000000000","message":"Done","commit_id":"934fe827cb7120834769f2b623da9476d91138eb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":638,"context_line":"                         int(resp.headers[\u0027X-Container-Object-Count\u0027]))"},{"line_number":639,"context_line":"        self.assertEqual(exp_sharding_state,"},{"line_number":640,"context_line":"                         resp.headers[\u0027X-Backend-Sharding-State\u0027])"},{"line_number":641,"context_line":"        self.assertEqual(\u0027object\u0027, resp.headers[\u0027X-Backend-Record-Type\u0027])"},{"line_number":642,"context_line":"        for k, v in root_resp_hdrs.items():"},{"line_number":643,"context_line":"            if k.lower().startswith(\u0027x-container-meta\u0027):"},{"line_number":644,"context_line":"                self.assertEqual(v, resp.headers[k])"}],"source_content_type":"text/x-python","patch_set":17,"id":"92daf76f_c96a085d","line":641,"updated":"2023-11-29 21:56:41.000000000","message":"this is a common test helper, this assert fails in a number of tests w/o the root response record-type fixup, e.g.\n\n    swift/test/unit/proxy/controllers/test_container.py:641: in check_listing_response\n        self.assertEqual(\u0027object\u0027, resp.headers[\u0027X-Backend-Record-Type\u0027])\n    E   AssertionError: \u0027object\u0027 !\u003d \u0027shard\u0027\n    E   - object\n    E   + shard","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":638,"context_line":"                         int(resp.headers[\u0027X-Container-Object-Count\u0027]))"},{"line_number":639,"context_line":"        self.assertEqual(exp_sharding_state,"},{"line_number":640,"context_line":"                         resp.headers[\u0027X-Backend-Sharding-State\u0027])"},{"line_number":641,"context_line":"        self.assertEqual(\u0027object\u0027, resp.headers[\u0027X-Backend-Record-Type\u0027])"},{"line_number":642,"context_line":"        for k, v in root_resp_hdrs.items():"},{"line_number":643,"context_line":"            if k.lower().startswith(\u0027x-container-meta\u0027):"},{"line_number":644,"context_line":"                self.assertEqual(v, resp.headers[k])"}],"source_content_type":"text/x-python","patch_set":17,"id":"e36451a2_eb0e0f35","line":641,"in_reply_to":"92daf76f_c96a085d","updated":"2023-11-30 15:43:06.000000000","message":"I moved this change to the probe test patch, along with the fix","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d8f2a5ff50e7a915d6ba65161463ac68ed94d031","unresolved":false,"context_lines":[{"line_number":638,"context_line":"                         int(resp.headers[\u0027X-Container-Object-Count\u0027]))"},{"line_number":639,"context_line":"        self.assertEqual(exp_sharding_state,"},{"line_number":640,"context_line":"                         resp.headers[\u0027X-Backend-Sharding-State\u0027])"},{"line_number":641,"context_line":"        self.assertEqual(\u0027object\u0027, resp.headers[\u0027X-Backend-Record-Type\u0027])"},{"line_number":642,"context_line":"        for k, v in root_resp_hdrs.items():"},{"line_number":643,"context_line":"            if k.lower().startswith(\u0027x-container-meta\u0027):"},{"line_number":644,"context_line":"                self.assertEqual(v, resp.headers[k])"}],"source_content_type":"text/x-python","patch_set":17,"id":"a9f2d035_bbcc2cf3","line":641,"in_reply_to":"e36451a2_eb0e0f35","updated":"2023-12-19 19:06:25.000000000","message":"Done","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":650,"context_line":"        if not isinstance(bounds[0], (list, tuple)):"},{"line_number":651,"context_line":"            bounds \u003d [(l, u) for l, u in zip(bounds[:-1], bounds[1:])]"},{"line_number":652,"context_line":"        namespaces \u003d ["},{"line_number":653,"context_line":"            Namespace(\u0027.shards_a/c_%s\u0027 % upper.replace(remove_char, \u0027\u0027),"},{"line_number":654,"context_line":"                      lower, upper)"},{"line_number":655,"context_line":"            for lower, upper in bounds]"},{"line_number":656,"context_line":"        ns_dicts \u003d [dict(ns) for ns in namespaces]"}],"source_content_type":"text/x-python","patch_set":17,"id":"642ab817_f0aae17a","line":653,"updated":"2023-11-29 21:56:41.000000000","message":"what is this remove_char stuff all about?","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":650,"context_line":"        if not isinstance(bounds[0], (list, tuple)):"},{"line_number":651,"context_line":"            bounds \u003d [(l, u) for l, u in zip(bounds[:-1], bounds[1:])]"},{"line_number":652,"context_line":"        namespaces \u003d ["},{"line_number":653,"context_line":"            Namespace(\u0027.shards_a/c_%s\u0027 % upper.replace(remove_char, \u0027\u0027),"},{"line_number":654,"context_line":"                      lower, upper)"},{"line_number":655,"context_line":"            for lower, upper in bounds]"},{"line_number":656,"context_line":"        ns_dicts \u003d [dict(ns) for ns in namespaces]"}],"source_content_type":"text/x-python","patch_set":17,"id":"a9951072_ef4c23a4","line":653,"in_reply_to":"642ab817_f0aae17a","updated":"2023-11-30 15:43:06.000000000","message":"looks like it came with the parent patch (at least it became an arg to the helper that was introduced) - it\u0027s to deal with tests that have \u0027/\u0027 in the bounds. The \u0027/\u0027 can\u0027t be used in the made-up container name.\n\nI don\u0027t think it is necessary for it to be an optional arg, we can just always replace \u0027/\u0027 with some other char.","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d8f2a5ff50e7a915d6ba65161463ac68ed94d031","unresolved":false,"context_lines":[{"line_number":650,"context_line":"        if not isinstance(bounds[0], (list, tuple)):"},{"line_number":651,"context_line":"            bounds \u003d [(l, u) for l, u in zip(bounds[:-1], bounds[1:])]"},{"line_number":652,"context_line":"        namespaces \u003d ["},{"line_number":653,"context_line":"            Namespace(\u0027.shards_a/c_%s\u0027 % upper.replace(remove_char, \u0027\u0027),"},{"line_number":654,"context_line":"                      lower, upper)"},{"line_number":655,"context_line":"            for lower, upper in bounds]"},{"line_number":656,"context_line":"        ns_dicts \u003d [dict(ns) for ns in namespaces]"}],"source_content_type":"text/x-python","patch_set":17,"id":"e118d17c_90e0687f","line":653,"in_reply_to":"a9951072_ef4c23a4","updated":"2023-12-19 19:06:25.000000000","message":"Done","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":864,"context_line":"        expected_objects \u003d all_objects[first_included:]"},{"line_number":865,"context_line":"        mock_responses \u003d ["},{"line_number":866,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":867,"context_line":"            (200, ns_dicts[3:], root_shard_resp_hdrs),"},{"line_number":868,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":869,"context_line":"            (200, sr_objs[3][2:], shard_resp_hdrs[3]),"},{"line_number":870,"context_line":"            (200, sr_objs[4], shard_resp_hdrs[4]),"}],"source_content_type":"text/x-python","patch_set":17,"id":"55f96e98_f5cfecb4","line":867,"updated":"2023-11-29 21:56:41.000000000","message":"the changes to this file look like mostly a mechanical translation of existing tests, but I\u0027m not so sure the backend respones to the root would return a truncated list of namespaces to the proxy if the record-type: auto request discovered a sharded root.\n\nmaybe these tests pre-date the cache-accelerated-listing-shard-range behavior; the test fails with unexpected requests if I change this response to the whole ns_dicts list","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":864,"context_line":"        expected_objects \u003d all_objects[first_included:]"},{"line_number":865,"context_line":"        mock_responses \u003d ["},{"line_number":866,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":867,"context_line":"            (200, ns_dicts[3:], root_shard_resp_hdrs),"},{"line_number":868,"context_line":"            (404, \u0027\u0027, {}),"},{"line_number":869,"context_line":"            (200, sr_objs[3][2:], shard_resp_hdrs[3]),"},{"line_number":870,"context_line":"            (200, sr_objs[4], shard_resp_hdrs[4]),"}],"source_content_type":"text/x-python","patch_set":17,"id":"c7613ef0_d945e563","line":867,"in_reply_to":"55f96e98_f5cfecb4","updated":"2023-11-30 15:43:06.000000000","message":"this is a \u0027*_no_memcache\u0027 test so the proxy doesn\u0027t want a complete list, shouldn\u0027t send the X-Backend-Override-Shard-Name-Filter header, and the marker is applied to the backend response.\n\nThe next test uses memcache and we assert that X-Backend-Override-Shard-Name-Filter is sent\n\nsee line 614","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"a8b539bbad3653e175a5cc4606c55a5ab5919bc6","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"        limit \u003d 2"},{"line_number":1365,"context_line":"        expected_objects \u003d all_objects[first_included:first_excluded]"},{"line_number":1366,"context_line":"        mock_responses \u003d ["},{"line_number":1367,"context_line":"            (200, ns_dicts[3:4], root_shard_resp_hdrs),"},{"line_number":1368,"context_line":"            (200, sr_objs[3][2:6], shard_resp_hdrs[1])"},{"line_number":1369,"context_line":"        ]"},{"line_number":1370,"context_line":"        expected_requests \u003d ["}],"source_content_type":"text/x-python","patch_set":17,"id":"dd49db37_a845c39c","line":1367,"updated":"2023-11-28 05:54:41.000000000","message":"and here should be all ns_dicts as well? since ``\u0027X-Backend-Override-Shard-Name-Filter\u0027: `` is ``\u0027sharded\u0027``","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d8f2a5ff50e7a915d6ba65161463ac68ed94d031","unresolved":false,"context_lines":[{"line_number":1364,"context_line":"        limit \u003d 2"},{"line_number":1365,"context_line":"        expected_objects \u003d all_objects[first_included:first_excluded]"},{"line_number":1366,"context_line":"        mock_responses \u003d ["},{"line_number":1367,"context_line":"            (200, ns_dicts[3:4], root_shard_resp_hdrs),"},{"line_number":1368,"context_line":"            (200, sr_objs[3][2:6], shard_resp_hdrs[1])"},{"line_number":1369,"context_line":"        ]"},{"line_number":1370,"context_line":"        expected_requests \u003d ["}],"source_content_type":"text/x-python","patch_set":17,"id":"28467922_4558b172","line":1367,"in_reply_to":"8c252e8e_497c0709","updated":"2023-12-19 19:06:25.000000000","message":"Done","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":true,"context_lines":[{"line_number":1364,"context_line":"        limit \u003d 2"},{"line_number":1365,"context_line":"        expected_objects \u003d all_objects[first_included:first_excluded]"},{"line_number":1366,"context_line":"        mock_responses \u003d ["},{"line_number":1367,"context_line":"            (200, ns_dicts[3:4], root_shard_resp_hdrs),"},{"line_number":1368,"context_line":"            (200, sr_objs[3][2:6], shard_resp_hdrs[1])"},{"line_number":1369,"context_line":"        ]"},{"line_number":1370,"context_line":"        expected_requests \u003d ["}],"source_content_type":"text/x-python","patch_set":17,"id":"8c252e8e_497c0709","line":1367,"in_reply_to":"dd49db37_a845c39c","updated":"2023-11-30 15:43:06.000000000","message":"good catch!","commit_id":"6af59026dd9c2d0f126eaf07047a25a9d555d58b"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8d740c0dfc942f15fb4542c14c422bb463343c17","unresolved":true,"context_lines":[{"line_number":7195,"context_line":"            #   namespaces for requests with \u0027includes\u0027 param"},{"line_number":7196,"context_line":"            shards \u003d [sr for sr in shards if includes in sr]"},{"line_number":7197,"context_line":"            resp_headers \u003d {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":7198,"context_line":"                            \u0027X-Backend-Record-Shard-Format\u0027: \u0027full\u0027}"},{"line_number":7199,"context_line":"        body \u003d json.dumps([dict(sr) for sr in shards]).encode(\u0027ascii\u0027)"},{"line_number":7200,"context_line":"        return body, resp_headers"},{"line_number":7201,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"e62bbc58_6973198d","side":"PARENT","line":7198,"updated":"2023-11-29 21:56:41.000000000","message":"I think TestGetUpdatingNamespacesLegacy sufficiently demonstrates that the proxy is able to handle the shard-format full responses for updating-shard-range requests.","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4037bc2489c8af798352553a7d6e54cae9d873d","unresolved":false,"context_lines":[{"line_number":7195,"context_line":"            #   namespaces for requests with \u0027includes\u0027 param"},{"line_number":7196,"context_line":"            shards \u003d [sr for sr in shards if includes in sr]"},{"line_number":7197,"context_line":"            resp_headers \u003d {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":7198,"context_line":"                            \u0027X-Backend-Record-Shard-Format\u0027: \u0027full\u0027}"},{"line_number":7199,"context_line":"        body \u003d json.dumps([dict(sr) for sr in shards]).encode(\u0027ascii\u0027)"},{"line_number":7200,"context_line":"        return body, resp_headers"},{"line_number":7201,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"7cb66780_c83ceb11","side":"PARENT","line":7198,"in_reply_to":"e62bbc58_6973198d","updated":"2023-11-30 15:43:06.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"}]}