)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"ad548d10_e91f0434","updated":"2023-11-29 14:59:23.000000000","message":"I think having these explicit end-to-end tests on the proxy-\u003esharded-container interactions are REALLY helpful!  KUDOS!","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"af87fec043b1167f30c9b73a9c2297739f8314e3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"25000614_89291a9e","updated":"2023-11-29 21:58:05.000000000","message":"nit: I normally run my vsaio with auto_shard\u003dfalse since that\u0027s how we run swift in prod, I think this test could trivially call s-m-s-r and not require/skip unless auto_shard\u003dtrue\n\nhttps://review.opendev.org/c/openstack/swift/+/902205","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"63226960_c94c64b1","in_reply_to":"25000614_89291a9e","updated":"2023-11-30 12:47:38.000000000","message":"thanks!","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"dfc7ef1e_8e8a4b59","updated":"2023-12-05 15:26:17.000000000","message":"I still love these tests and appreciate your fix-up of the client object-listing response x-backend-record-type: shard header (which is obviously wrong)\n\nHow do you feel about changing what we overwrite/fix the x-backend-record-type: shard response header with something to explicitly indicate it\u0027s not *just* a x-backend-record-type: object response from the root?","commit_id":"b1c2dc1d6da2c0ae2ba6f128c7cf3b21c9f09fb5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a16c0b3d0c6fc70820a84f71a58fb299a299c5c0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"fdbe4e9f_4a2ba3af","updated":"2023-12-06 19:22:08.000000000","message":"Thanks Clay for pushing me to think more about the response header. I\u0027m not sure quite how to deal with the concern - see inline response.","commit_id":"2810a3755005fd46b912238085e294ef7ae2070a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c52e3698ee23329981b31dd6f79b38e8e77afdad","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"56d488fd_74d650f3","updated":"2023-12-07 17:38:48.000000000","message":"hahaha i\u0027ll test popping the header.","commit_id":"83f1a67a2652c5291a453e3a2e22cfaa5bb63fa0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ade21f3444a854583cded0338f60fb54f9d8b53","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":15,"id":"a73bf266_1f3fbbdd","updated":"2023-12-08 16:48:11.000000000","message":"i was just glancing at this between tests - i\u0027m sure it\u0027s great and if I spent more time with it I could answer my own confusions.","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8e76164cf0195e718651e4a1fdac4651f94bedd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"3342f5e5_858f9a9d","updated":"2023-12-11 18:59:52.000000000","message":"recheck","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"53bf92b5_c588479c","updated":"2023-12-13 00:24:48.000000000","message":"the implementation for the fix is great; no more x-backend-record-type on client-listing responses (esp nice for sharded containers when the value was super misleading).  And we add this consistency while continuing to support internal-client requests w/ x-backend-record-type: object|shard getting the resp header back!\n\nthe probe test is SO great! soooo useful!\n\n * the get_container_shard_ranges helper works exactly like you\u0027d assume.\n * the get_container_objects is opionated, don\u0027t try calling it with x-backend-record-type: shard\n * the assert_container_states helper seems nice\n * the sharders_once_non_auto seems to work, but I don\u0027t understand the motivation\n\nin unittests, \n\n * the invalid resp data test_sharder updates seem like a drive-by\n * the new assertions in proxy.controller.test_container are working\n * ... but the pre-existing mocking is mind bending\n\nI tried a few things to wrap my head around it and gave up:\n\nhttps://review.opendev.org/c/openstack/swift/+/903567\n\n^ there\u0027s some unrelated \"fixups\" in there that might be worth pulling out; but nothing different enough worth holding up this change.  Despite a few small speedbumps I really like it.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":true,"context_lines":[{"line_number":586,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":587,"context_line":"                break"},{"line_number":588,"context_line":""},{"line_number":589,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":590,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":591,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":592,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":6,"id":"6c57839b_ee671715","line":589,"updated":"2023-12-05 15:26:17.000000000","message":"I think x-backend-record-type\u003dobject is still a little misleading.  The InternalClient would get a different response if they sent back in the same request with x-backend-record-type\u003dobject (probably empty list of objects from root).\n\nSince we\u0027re overwriting the original root x-backend-record-type\u003dauto response headers maybe it make more sense to set to something to explicitly indicate the form of this listing response are client object-listing records.\n\nx-backend-record-type; auto\nx-backend-record-type: object-listing\nresp.headers.pop(\u0027x-backend-record-type\u0027)\n\n... but on measure object is better than shard; so this is probably an improvement.","commit_id":"b1c2dc1d6da2c0ae2ba6f128c7cf3b21c9f09fb5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a16c0b3d0c6fc70820a84f71a58fb299a299c5c0","unresolved":true,"context_lines":[{"line_number":586,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":587,"context_line":"                break"},{"line_number":588,"context_line":""},{"line_number":589,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":590,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":591,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":592,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":6,"id":"d90e1162_851c9480","line":589,"in_reply_to":"6c57839b_ee671715","updated":"2023-12-06 19:22:08.000000000","message":"OK, I think I understand your concern. \n\nI don\u0027t feel that record-type\u003dobject is misleading in the context of the response - it is an accurate statement of the *type* of what is listed. If I imagine needing to choose a parser for the body, x-backend-record-type would tell me which parser. \n\nBut, I agree, the *scope* of what the listing includes may vary between \u0027just the target\u0027 and \u0027all the shards\u0027.\n\n```\nreq record-type | container state | resp record-type | listing scope\nobject             unsharded          object             all\nobject             sharded            object             partial\n - or auto         unsharded          object             all\n - or auto         sharded            object             all\n```\n\nI\u0027m not quite sure how best to represent that, but I don\u0027t think we should conflate \u0027record-type\u0027 with \u0027listing-scope\u0027 in a single response header, because:\n\n- we only get to this line for a sharded container. So to be consistent we\u0027d have to also fix up an unsharded listing with object-type\u003dobject-full-listing (i.e. more work).\n\n- we could do that based on the x-backend-sharding-state, but the internal client can equally already infer that from the headers.\n\nI think, if anything, I\u0027d rather add another header like x-backend-listing-scope\u003d[full | partial]. (I\u0027m not sure I like those values). But I don\u0027t think we yet have a use case for an internal client that cares? \n\nI\u0027ll think on it some more.","commit_id":"b1c2dc1d6da2c0ae2ba6f128c7cf3b21c9f09fb5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":false,"context_lines":[{"line_number":586,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":587,"context_line":"                break"},{"line_number":588,"context_line":""},{"line_number":589,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":590,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":591,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":592,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":6,"id":"ebcea27d_0a9e6ec9","line":589,"in_reply_to":"892cd413_cac0ea66","updated":"2023-12-11 14:19:17.000000000","message":"Acknowledged","commit_id":"b1c2dc1d6da2c0ae2ba6f128c7cf3b21c9f09fb5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c52e3698ee23329981b31dd6f79b38e8e77afdad","unresolved":true,"context_lines":[{"line_number":586,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":587,"context_line":"                break"},{"line_number":588,"context_line":""},{"line_number":589,"context_line":"        resp.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":590,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":591,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":592,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":6,"id":"892cd413_cac0ea66","line":589,"in_reply_to":"d90e1162_851c9480","updated":"2023-12-07 17:38:48.000000000","message":"\u003e I don\u0027t think we should conflate \u0027record-type\u0027 with \u0027listing-scope\u0027 in a single response header\n\nI find this very convincing!\n\n\u003e If I imagine needing to choose a parser for the body, x-backend-record-type would tell me which parser\n\nI think the only place that\u0027s relevant is internal-client making backend root/shard requests; for client listings the format of the dicts is *always* objects.  We could just include x-backend-record-type - which matches what the client sends in.","commit_id":"b1c2dc1d6da2c0ae2ba6f128c7cf3b21c9f09fb5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"370af7e4f554461c6e38b12e225f4cc392eb16cc","unresolved":true,"context_lines":[{"line_number":397,"context_line":"            resp \u003d self._get_from_shards(req, resp)"},{"line_number":398,"context_line":""},{"line_number":399,"context_line":"        if orig_record_type not in (\u0027object\u0027, \u0027shard\u0027):"},{"line_number":400,"context_line":"            resp.headers.pop(\u0027X-Backend-Record-Type\u0027, None)"},{"line_number":401,"context_line":""},{"line_number":402,"context_line":"        if not config_true_value("},{"line_number":403,"context_line":"                resp.headers.get(\u0027X-Backend-Cached-Results\u0027)):"}],"source_content_type":"text/x-python","patch_set":14,"id":"3383a418_63efc3a5","line":400,"updated":"2023-12-08 16:01:32.000000000","message":"popping the header caused more test churn than I would have hoped!\n\nI make it conditional because there are places we check the response header and it is the only *explicit* way in which we detect that the backend is modern enough to know about record types. We could probably pop in always but that felt like more thinking than I had time for today!","commit_id":"cd6617bf168c59b44f263d1d3e4e5526a79b1a2b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":397,"context_line":"            resp \u003d self._get_from_shards(req, resp)"},{"line_number":398,"context_line":""},{"line_number":399,"context_line":"        if orig_record_type not in (\u0027object\u0027, \u0027shard\u0027):"},{"line_number":400,"context_line":"            resp.headers.pop(\u0027X-Backend-Record-Type\u0027, None)"},{"line_number":401,"context_line":""},{"line_number":402,"context_line":"        if not config_true_value("},{"line_number":403,"context_line":"                resp.headers.get(\u0027X-Backend-Cached-Results\u0027)):"}],"source_content_type":"text/x-python","patch_set":14,"id":"09c43b1a_40c63f57","line":400,"in_reply_to":"3383a418_63efc3a5","updated":"2023-12-13 00:24:48.000000000","message":"i think it\u0027s perfectly reasonable when returning the backend listing directly to include the record-type; the only place I felt it was \"bleeding\" into the InternalClient API was for client-listings-from-sharded-containers where the x-record-type\u003dshard was obviously wrong.\n\nI think the api for internal-client to fetch those backend listings directly is obviously to select an x-backend-record-type\u003dobject|shard so returning the value in only those cases where it\u0027s been explicitly requested seems reasonable.","commit_id":"cd6617bf168c59b44f263d1d3e4e5526a79b1a2b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ade21f3444a854583cded0338f60fb54f9d8b53","unresolved":true,"context_lines":[{"line_number":584,"context_line":"                break"},{"line_number":585,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":586,"context_line":"                break"},{"line_number":587,"context_line":""},{"line_number":588,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":589,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":590,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":15,"id":"c1764bce_3d1344b8","line":587,"updated":"2023-12-08 16:48:11.000000000","message":"in the original version I think we were overwriting the record-format: shard header to record-format: object down here; where we\u0027ve obviously constructed a client object-listing form shards (i.e. the auto request found shards, or we skipped the root request entirely using constructed headers \u0026 namespace-bounds fromm memcache).\n\n*This* is where I assumed we\u0027d pop the header and leave requests that explicitly request record-format \u003d object|shard to get the expected response header un-molested.\n\nThere must me more going on than i understand.\n\nI think I get that sometimes the proxy \"recursively\" calls into itself to get a listing from a shard w/ type \u0027auto\u0027 and the backend responds shards... but I guess I was hoping if it was previously ok with getting a transparent client-listing+record-type\u003dshard resp it\u0027d be ok with client-listing+record_type\u003dNone?\n\nI guess I didn\u0027t really have any problem with InternalClient making a client-listing requst and getting x-backend-record-type: object from an unsharded root.  But maybe I should?","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":true,"context_lines":[{"line_number":584,"context_line":"                break"},{"line_number":585,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":586,"context_line":"                break"},{"line_number":587,"context_line":""},{"line_number":588,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":589,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":590,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":15,"id":"a6265cb8_54fbe8da","line":587,"in_reply_to":"c1764bce_3d1344b8","updated":"2023-12-11 14:19:17.000000000","message":"\u003e ... leave requests that explicitly request record-format \u003d object|shard to get the expected response header un-molested.\n\nthat is what this patch does\n\n\u003e I think I get that sometimes the proxy \"recursively\" calls into itself to get a listing from a shard w/ type \u0027auto\u0027 and the backend responds shards...\n\nThe proxy API does not recognise \u0027auto\u0027. The proxy does not intentionally call back to its own API with a record-type header\u003dauto (it might bleed though but it\u0027s not significant). It only ever intentionally uses \u0027auto\u0027 when sending a request to the backend, because only the backend recognises \u0027auto\u0027.\n\n\u003e I guess I didn\u0027t really have any problem with InternalClient making a client-listing requst and getting x-backend-record-type: object from an unsharded root. But maybe I should?\n\nI think if we\u0027re going to bother popping the header at all then let\u0027s do it in a way that\u0027s easy to explain: \"you sent the header, you get it back\" vs \"you didn\u0027t send the header, but sometimes you get it back, depending on the sharding state, because...???\".","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":true,"context_lines":[{"line_number":584,"context_line":"                break"},{"line_number":585,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":586,"context_line":"                break"},{"line_number":587,"context_line":""},{"line_number":588,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":589,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":590,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":15,"id":"fdc5f252_706329c2","line":587,"in_reply_to":"c1764bce_3d1344b8","updated":"2023-12-11 14:19:17.000000000","message":"\u003e leave requests that explicitly request record-format \u003d object|shard to get the expected response header un-molested.\n\nthis patch does that.\n\n\u003e I think I get that sometimes the proxy \"recursively\" calls into itself to get a listing from a shard w/ type \u0027auto\u0027 and the backend responds shards..\n\nThe proxy API does not recognise record-type\u003d\u0027auto\u0027, and the proxy never calls back to its own API intentionally setting record-type\u003dauto. The proxy *does* call the backend servers with record-type\u003d\u0027auto\u0027 and the backend servers do recognise it. Whenever the proxy sends \u0027auto\u0027 to the backend it always returns objects in its API response.\n\n\n```\n         client                 proxy                    backend\n         ------                 -----                    -------\n  \n      record_type\u003dobject-\u003e      record_type\u003dobject-\u003e\n                              \u003c-record_type\u003dobject      \u003c-record_type\u003dobject\n\u003c-list of objects    \n\n\n      record_type\u003dshard-\u003e      record_type\u003dshard-\u003e\n                             \u003c-record_type\u003dshard      \u003c-record_type\u003dshard\n\u003c- list of shards    \n\n\n      X-\u003e                      record_type\u003dauto-\u003e\n                                                      \u003c-record_type\u003dshard\n                               record_type\u003dauto-\u003e\n                                                      \u003c-record_type\u003dobject\n\u003c- list of objects    \n\n\n      X-\u003e                      record_type\u003dauto-\u003e\n                                                      \u003c-record_type\u003dobject\n\u003c- list of objects    \n```\n\n\u003eI guess I didn\u0027t really have any problem with InternalClient making a client-listing requst and getting x-backend-record-type: object from an unsharded root. But maybe I should?\n\nresponse record-type\u003d\u0027object\u0027 would then mean \u0027the type of the dicts is obj and either the request had record-type\u003dobject, or the backend was not sharded, or both.\n\nI prefer the simpler meaning: \u0027client sent recognised record type, client gets it back to confirm it was supported; client didn\u0027t send record type, client doesn\u0027t get it back\u0027.","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":584,"context_line":"                break"},{"line_number":585,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":586,"context_line":"                break"},{"line_number":587,"context_line":""},{"line_number":588,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":589,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":590,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":15,"id":"b64e9268_e7dc621e","line":587,"in_reply_to":"fdc5f252_706329c2","updated":"2023-12-13 00:24:48.000000000","message":"\u003e I think if we\u0027re going to bother popping the header at all then let\u0027s do it in a way that\u0027s easy to explain: \"you sent the header, you get it back\" vs \"you didn\u0027t send the header, but sometimes you get it back, depending on the sharding state, because...???\".\n\n\u003e response record-type\u003d\u0027object\u0027 would then mean \u0027the type of the dicts is obj and either the request had record-type\u003dobject, or the backend was not sharded, or both.\n\nMakes sense, when I said:\n\n\u003e I guess I didn\u0027t really have any problem with InternalClient making a client-listing requst and getting x-backend-record-type: object from an unsharded root. But maybe I should?\n\n... those are the reasons it *should* have bothered me; thanks for writing them out.  I agree, \"merely\" popping the obviously incorrect record-type\u003dshard header for client-listing-from-sharded-contaienrs wouldn\u0027t have addressed sending back x-backend-record-type\u003dobject from unsharded containers when internal-client implicitly requested a client-listing by not sending the x-backend-record-type header.","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"}],"test/probe/test_sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":188,"context_line":"        resp \u003d self.get_container_listing(account, container, headers,"},{"line_number":189,"context_line":"                                          params\u003dparams)"},{"line_number":190,"context_line":"        # XXX proxy is currently leaking record-type\u003dshard"},{"line_number":191,"context_line":"        # self.assertEqual(\u0027object\u0027, resp.headers.get(\u0027X-Backend-Record-Type\u0027))"},{"line_number":192,"context_line":"        return json.loads(resp.body)"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone,"}],"source_content_type":"text/x-python","patch_set":4,"id":"bbbcea1d_7dd88e09","line":191,"updated":"2023-11-29 14:59:23.000000000","message":"i noticed this gets fixed in the next patch; I\u0027m not sure this is even the worst of the bad behaviors enumerated here.","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":false,"context_lines":[{"line_number":188,"context_line":"        resp \u003d self.get_container_listing(account, container, headers,"},{"line_number":189,"context_line":"                                          params\u003dparams)"},{"line_number":190,"context_line":"        # XXX proxy is currently leaking record-type\u003dshard"},{"line_number":191,"context_line":"        # self.assertEqual(\u0027object\u0027, resp.headers.get(\u0027X-Backend-Record-Type\u0027))"},{"line_number":192,"context_line":"        return json.loads(resp.body)"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone,"}],"source_content_type":"text/x-python","patch_set":4,"id":"2d2bf1ef_ef1e0e77","line":191,"in_reply_to":"4315d834_3cd90eeb","updated":"2023-12-05 15:26:17.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":true,"context_lines":[{"line_number":188,"context_line":"        resp \u003d self.get_container_listing(account, container, headers,"},{"line_number":189,"context_line":"                                          params\u003dparams)"},{"line_number":190,"context_line":"        # XXX proxy is currently leaking record-type\u003dshard"},{"line_number":191,"context_line":"        # self.assertEqual(\u0027object\u0027, resp.headers.get(\u0027X-Backend-Record-Type\u0027))"},{"line_number":192,"context_line":"        return json.loads(resp.body)"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone,"}],"source_content_type":"text/x-python","patch_set":4,"id":"4315d834_3cd90eeb","line":191,"in_reply_to":"bbbcea1d_7dd88e09","updated":"2023-11-30 12:47:38.000000000","message":"I\u0027m moving the fix to this patch","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":2966,"context_line":"        # values and then respond to the client with the cached *namespaces* !!"},{"line_number":2967,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2968,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(params\u003dparams)"},{"line_number":2969,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2970,"context_line":""},{"line_number":2971,"context_line":"        # XXX ditto..."},{"line_number":2972,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"}],"source_content_type":"text/x-python","patch_set":4,"id":"f89c9ba1_087c85c6","line":2969,"updated":"2023-11-29 14:59:23.000000000","message":"even tho proxy responses with cached namespaces they should still be \"equivalnt\" when the orig_shard_ranges are normalized to namespace objects?","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":false,"context_lines":[{"line_number":2966,"context_line":"        # values and then respond to the client with the cached *namespaces* !!"},{"line_number":2967,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2968,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(params\u003dparams)"},{"line_number":2969,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2970,"context_line":""},{"line_number":2971,"context_line":"        # XXX ditto..."},{"line_number":2972,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"}],"source_content_type":"text/x-python","patch_set":4,"id":"eac1bcae_32f065ad","line":2969,"in_reply_to":"976c8b46_6a4498fc","updated":"2023-12-05 15:26:17.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":true,"context_lines":[{"line_number":2966,"context_line":"        # values and then respond to the client with the cached *namespaces* !!"},{"line_number":2967,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2968,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(params\u003dparams)"},{"line_number":2969,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2970,"context_line":""},{"line_number":2971,"context_line":"        # XXX ditto..."},{"line_number":2972,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"}],"source_content_type":"text/x-python","patch_set":4,"id":"976c8b46_6a4498fc","line":2969,"in_reply_to":"f89c9ba1_087c85c6","updated":"2023-11-30 12:47:38.000000000","message":"yes, but the get_container_shard_ranges blows up when trying to construct ShardRanges because there is no Timestamp in the dicts\n\n```\nFAILED test/probe/test_sharder.py::TestShardedAPI::test_GET - KeyError: \u0027timestamp\u0027\n```","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":2970,"context_line":""},{"line_number":2971,"context_line":"        # XXX ditto..."},{"line_number":2972,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2973,"context_line":"        # headers \u003d {\u0027X-Newest\u0027: \u0027true\u0027}"},{"line_number":2974,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":2975,"context_line":"        #                                                params\u003dparams)"},{"line_number":2976,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"}],"source_content_type":"text/x-python","patch_set":4,"id":"f936cc0b_ac319b40","line":2973,"updated":"2023-11-29 14:59:23.000000000","message":"i would epxect thiw would cause existing code to bypass cache?","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":false,"context_lines":[{"line_number":2970,"context_line":""},{"line_number":2971,"context_line":"        # XXX ditto..."},{"line_number":2972,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2973,"context_line":"        # headers \u003d {\u0027X-Newest\u0027: \u0027true\u0027}"},{"line_number":2974,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":2975,"context_line":"        #                                                params\u003dparams)"},{"line_number":2976,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"}],"source_content_type":"text/x-python","patch_set":4,"id":"42120a2d_08431246","line":2973,"in_reply_to":"39311854_ca26c69b","updated":"2023-12-05 15:26:17.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":true,"context_lines":[{"line_number":2970,"context_line":""},{"line_number":2971,"context_line":"        # XXX ditto..."},{"line_number":2972,"context_line":"        # params \u003d {\u0027states\u0027: \u0027listing\u0027}"},{"line_number":2973,"context_line":"        # headers \u003d {\u0027X-Newest\u0027: \u0027true\u0027}"},{"line_number":2974,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":2975,"context_line":"        #                                                params\u003dparams)"},{"line_number":2976,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"}],"source_content_type":"text/x-python","patch_set":4,"id":"39311854_ca26c69b","line":2973,"in_reply_to":"f936cc0b_ac319b40","updated":"2023-11-30 12:47:38.000000000","message":"cache is bypassed in the read path, but the shard ranges returned from backend are compacted to namespaces before being written to cache, and the written value is then returned in the client response body !?!\n\n```\nFAILED test/probe/test_sharder.py::TestShardedAPI::test_GET - KeyError: \u0027timestamp\u0027\n```","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":2973,"context_line":"        # headers \u003d {\u0027X-Newest\u0027: \u0027true\u0027}"},{"line_number":2974,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":2975,"context_line":"        #                                                params\u003dparams)"},{"line_number":2976,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2977,"context_line":""},{"line_number":2978,"context_line":"        # this is what the sharder requests..."},{"line_number":2979,"context_line":"        params \u003d {\u0027states\u0027: \u0027auditing\u0027}"}],"source_content_type":"text/x-python","patch_set":4,"id":"bbef207c_f488f81d","line":2976,"updated":"2023-11-29 14:59:23.000000000","message":"ok these are sick; as if making up responses from cache wasn\u0027t bad enough we\u0027re not even given the client what they requested?","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":false,"context_lines":[{"line_number":2973,"context_line":"        # headers \u003d {\u0027X-Newest\u0027: \u0027true\u0027}"},{"line_number":2974,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":2975,"context_line":"        #                                                params\u003dparams)"},{"line_number":2976,"context_line":"        # check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2977,"context_line":""},{"line_number":2978,"context_line":"        # this is what the sharder requests..."},{"line_number":2979,"context_line":"        params \u003d {\u0027states\u0027: \u0027auditing\u0027}"}],"source_content_type":"text/x-python","patch_set":4,"id":"3e9df8bb_44bf315a","line":2976,"in_reply_to":"bbef207c_f488f81d","updated":"2023-11-30 12:47:38.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":3006,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3007,"context_line":""},{"line_number":3008,"context_line":"        params \u003d {\u0027end_marker\u0027: all_obj_names[1]}"},{"line_number":3009,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":3010,"context_line":"                                                       params\u003dparams)"},{"line_number":3011,"context_line":"        check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3012,"context_line":"        namespaces \u003d self.get_container_namespaces(headers\u003dheaders,"}],"source_content_type":"text/x-python","patch_set":4,"id":"42cdffe1_9b0b4b61","line":3009,"updated":"2023-11-29 14:59:23.000000000","message":"a little subtle, headers here includes the shard-name-filter: sharded so end_marker is ignored and backend returns all","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":true,"context_lines":[{"line_number":3006,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3007,"context_line":""},{"line_number":3008,"context_line":"        params \u003d {\u0027end_marker\u0027: all_obj_names[1]}"},{"line_number":3009,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":3010,"context_line":"                                                       params\u003dparams)"},{"line_number":3011,"context_line":"        check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3012,"context_line":"        namespaces \u003d self.get_container_namespaces(headers\u003dheaders,"}],"source_content_type":"text/x-python","patch_set":4,"id":"c24a68dd_1bdb297f","line":3009,"in_reply_to":"42cdffe1_9b0b4b61","updated":"2023-11-30 12:47:38.000000000","message":"it annoys me to have to track \u0027headers\u0027 as I work through the various assertions - I\u0027m going to inline headers and params so it\u0027s more obvious","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":false,"context_lines":[{"line_number":3006,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3007,"context_line":""},{"line_number":3008,"context_line":"        params \u003d {\u0027end_marker\u0027: all_obj_names[1]}"},{"line_number":3009,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":3010,"context_line":"                                                       params\u003dparams)"},{"line_number":3011,"context_line":"        check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3012,"context_line":"        namespaces \u003d self.get_container_namespaces(headers\u003dheaders,"}],"source_content_type":"text/x-python","patch_set":4,"id":"e5c3fbc2_87a5ae3c","line":3009,"in_reply_to":"c24a68dd_1bdb297f","updated":"2023-12-05 15:26:17.000000000","message":"Done","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":3020,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3021,"context_line":""},{"line_number":3022,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":3023,"context_line":"                                                       params\u003dparams)"},{"line_number":3024,"context_line":"        check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3025,"context_line":"        namespaces \u003d self.get_container_namespaces(headers\u003dheaders,"},{"line_number":3026,"context_line":"                                                   params\u003dparams)"}],"source_content_type":"text/x-python","patch_set":4,"id":"f602de68_c25bb05e","line":3023,"updated":"2023-11-29 14:59:23.000000000","message":"here again headers causes reverse to be ignored","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":false,"context_lines":[{"line_number":3020,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3021,"context_line":""},{"line_number":3022,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":3023,"context_line":"                                                       params\u003dparams)"},{"line_number":3024,"context_line":"        check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3025,"context_line":"        namespaces \u003d self.get_container_namespaces(headers\u003dheaders,"},{"line_number":3026,"context_line":"                                                   params\u003dparams)"}],"source_content_type":"text/x-python","patch_set":4,"id":"9b011a3e_54411135","line":3023,"in_reply_to":"96b56849_ae9f078a","updated":"2023-12-05 15:26:17.000000000","message":"Done","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":true,"context_lines":[{"line_number":3020,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3021,"context_line":""},{"line_number":3022,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges(headers\u003dheaders,"},{"line_number":3023,"context_line":"                                                       params\u003dparams)"},{"line_number":3024,"context_line":"        check_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":3025,"context_line":"        namespaces \u003d self.get_container_namespaces(headers\u003dheaders,"},{"line_number":3026,"context_line":"                                                   params\u003dparams)"}],"source_content_type":"text/x-python","patch_set":4,"id":"96b56849_ae9f078a","line":3023,"in_reply_to":"f602de68_c25bb05e","updated":"2023-11-30 12:47:38.000000000","message":"comment added","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":3026,"context_line":"                                                   params\u003dparams)"},{"line_number":3027,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3028,"context_line":""},{"line_number":3029,"context_line":"        # XXX NB get_container_objects is hacked to pass"},{"line_number":3030,"context_line":"        objs \u003d self.get_container_objects()"},{"line_number":3031,"context_line":"        self.assertEqual(all_obj_names, [obj[\u0027name\u0027] for obj in objs])"},{"line_number":3032,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"4027e802_c525715d","line":3029,"updated":"2023-11-29 14:59:23.000000000","message":"what is this comment about?","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":true,"context_lines":[{"line_number":3026,"context_line":"                                                   params\u003dparams)"},{"line_number":3027,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3028,"context_line":""},{"line_number":3029,"context_line":"        # XXX NB get_container_objects is hacked to pass"},{"line_number":3030,"context_line":"        objs \u003d self.get_container_objects()"},{"line_number":3031,"context_line":"        self.assertEqual(all_obj_names, [obj[\u0027name\u0027] for obj in objs])"},{"line_number":3032,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"a0e5ceb5_e544ef02","line":3029,"in_reply_to":"4027e802_c525715d","updated":"2023-11-30 12:47:38.000000000","message":"it\u0027s a reminder that this only passes because there is an XXX in get_container_objects. But this will go anyway once I move the record-type fix into this patch","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":false,"context_lines":[{"line_number":3026,"context_line":"                                                   params\u003dparams)"},{"line_number":3027,"context_line":"        check_equivalence(shard_ranges, namespaces)"},{"line_number":3028,"context_line":""},{"line_number":3029,"context_line":"        # XXX NB get_container_objects is hacked to pass"},{"line_number":3030,"context_line":"        objs \u003d self.get_container_objects()"},{"line_number":3031,"context_line":"        self.assertEqual(all_obj_names, [obj[\u0027name\u0027] for obj in objs])"},{"line_number":3032,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"25795b67_022de30e","line":3029,"in_reply_to":"a0e5ceb5_e544ef02","updated":"2023-12-05 15:26:17.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":3041,"context_line":"        # note: explicitly asking for the root object rows, but it has None"},{"line_number":3042,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3043,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3044,"context_line":"        self.assertEqual([], objs)"},{"line_number":3045,"context_line":""},{"line_number":3046,"context_line":""},{"line_number":3047,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":4,"id":"a9cfd85e_5f07d3d4","line":3044,"updated":"2023-11-29 14:59:23.000000000","message":"wow, I still keep expecting x-backend-record-type: auto to just mean \"default to object\" - but \"auto\" is in the proxy context is actually \"get a container listing\" and \"object\" is \"get an object listing from the root\"\n\nI know why we choose \u0027auto\u0027 for the backend container interface; but maybe the proxy interface the difference between record-type auto and record-type object is too subtle for me.","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":false,"context_lines":[{"line_number":3041,"context_line":"        # note: explicitly asking for the root object rows, but it has None"},{"line_number":3042,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3043,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3044,"context_line":"        self.assertEqual([], objs)"},{"line_number":3045,"context_line":""},{"line_number":3046,"context_line":""},{"line_number":3047,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":4,"id":"049dc67e_d3283850","line":3044,"in_reply_to":"2c6aa7c8_24b8931d","updated":"2023-12-11 14:19:17.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":false,"context_lines":[{"line_number":3041,"context_line":"        # note: explicitly asking for the root object rows, but it has None"},{"line_number":3042,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3043,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3044,"context_line":"        self.assertEqual([], objs)"},{"line_number":3045,"context_line":""},{"line_number":3046,"context_line":""},{"line_number":3047,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":4,"id":"13a9d3e8_89333b75","line":3044,"in_reply_to":"2c6aa7c8_24b8931d","updated":"2023-12-11 14:19:17.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a293fb649b150d8a7d807d047dd627637cc03f79","unresolved":true,"context_lines":[{"line_number":3041,"context_line":"        # note: explicitly asking for the root object rows, but it has None"},{"line_number":3042,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3043,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3044,"context_line":"        self.assertEqual([], objs)"},{"line_number":3045,"context_line":""},{"line_number":3046,"context_line":""},{"line_number":3047,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":4,"id":"2c6aa7c8_24b8931d","line":3044,"in_reply_to":"9da5c1bc_49d5d008","updated":"2023-12-05 15:26:17.000000000","message":"\u003e we need the proxy to support \u0027object\u0027 in order to override the recursive listing.\n\ndefinately; x-backend-record-type: object requests have a specific meaning that\u0027s different from auto/None\n\n\u003e the proxy (ContainerController) doesn\u0027t explicitly check for \u0027auto\u0027, so it\u0027s not really a client facing API \u0027feature\u0027\n\nwhere as x-backend-record-type: object *is* a [Internal]Client \"feature\u0027 - I think future authors may appreciate some consistency with the request and repsonse headers when dealing with record-type object|shard vs client object-listings\n\n\u003e The proxy just defaults to \u0027auto mode\u0027 in the absence of \u0027shard\u0027 or \u0027object\u0027.\n\nok, so *anything* to indicate a client object-listing response is better than shard|object - got it.","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":true,"context_lines":[{"line_number":3041,"context_line":"        # note: explicitly asking for the root object rows, but it has None"},{"line_number":3042,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3043,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3044,"context_line":"        self.assertEqual([], objs)"},{"line_number":3045,"context_line":""},{"line_number":3046,"context_line":""},{"line_number":3047,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":4,"id":"9da5c1bc_49d5d008","line":3044,"in_reply_to":"a9cfd85e_5f07d3d4","updated":"2023-11-30 12:47:38.000000000","message":"we need the *proxy* to support \u0027object\u0027 in order to override the recursive listing. \n\nWhen handling a client request (as opposed to a recursive GET) the proxy (ContainerController) doesn\u0027t explicitly check for \u0027auto\u0027, so it\u0027s not really a client facing API \u0027feature\u0027 (but obviously the header might take that value, hence the test at line 3038). The proxy just defaults to \u0027auto mode\u0027 in the absence of \u0027shard\u0027 or \u0027object\u0027.","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"19d836e73acf85fa44bd7d4f651919174297eb58","unresolved":true,"context_lines":[{"line_number":3560,"context_line":"        # because the sub-shards report their state to the root; we cannot make"},{"line_number":3561,"context_line":"        # assertions about shrunk states in shard_1\u0027s shard range table)"},{"line_number":3562,"context_line":"        root_shard_ranges \u003d self.get_container_shard_ranges("},{"line_number":3563,"context_line":"            headers\u003d{\u0027X-Backend-Include-Deleted\u0027: \u0027true\u0027})"},{"line_number":3564,"context_line":"        self.assertEqual(10, len(root_shard_ranges), root_shard_ranges)"},{"line_number":3565,"context_line":"        shrunk_shard_ranges \u003d [sr for sr in root_shard_ranges"},{"line_number":3566,"context_line":"                               if sr.state \u003d\u003d ShardRange.SHRUNK]"}],"source_content_type":"text/x-python","patch_set":4,"id":"c6f00dbd_db92c1f8","line":3563,"updated":"2023-11-29 14:59:23.000000000","message":"i didn\u0027t even notice this interface changed, the new one is more flexible - KUDOS!","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d6231016a2e048f10131768868c405708ee54119","unresolved":false,"context_lines":[{"line_number":3560,"context_line":"        # because the sub-shards report their state to the root; we cannot make"},{"line_number":3561,"context_line":"        # assertions about shrunk states in shard_1\u0027s shard range table)"},{"line_number":3562,"context_line":"        root_shard_ranges \u003d self.get_container_shard_ranges("},{"line_number":3563,"context_line":"            headers\u003d{\u0027X-Backend-Include-Deleted\u0027: \u0027true\u0027})"},{"line_number":3564,"context_line":"        self.assertEqual(10, len(root_shard_ranges), root_shard_ranges)"},{"line_number":3565,"context_line":"        shrunk_shard_ranges \u003d [sr for sr in root_shard_ranges"},{"line_number":3566,"context_line":"                               if sr.state \u003d\u003d ShardRange.SHRUNK]"}],"source_content_type":"text/x-python","patch_set":4,"id":"3be81455_17e9162a","line":3563,"in_reply_to":"c6f00dbd_db92c1f8","updated":"2023-11-30 12:47:38.000000000","message":"Acknowledged","commit_id":"f0194063b8e8923f49bdfe1d3511934e190e29ea"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ade21f3444a854583cded0338f60fb54f9d8b53","unresolved":true,"context_lines":[{"line_number":186,"context_line":"                              headers\u003dNone, params\u003dNone):"},{"line_number":187,"context_line":"        headers \u003d dict(headers) if headers else {}"},{"line_number":188,"context_line":"        resp \u003d self.get_container_listing(account, container, headers,"},{"line_number":189,"context_line":"                                          params\u003dparams)"},{"line_number":190,"context_line":"        return json.loads(resp.body)"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone,"}],"source_content_type":"text/x-python","patch_set":15,"id":"f76cb183_29a8dc97","line":189,"updated":"2023-12-08 16:48:11.000000000","message":"we could make add some assertions on x-backend-record-type here as well\n\ni guess it\u0027s either objects or None depending on what the client sent in?","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":false,"context_lines":[{"line_number":186,"context_line":"                              headers\u003dNone, params\u003dNone):"},{"line_number":187,"context_line":"        headers \u003d dict(headers) if headers else {}"},{"line_number":188,"context_line":"        resp \u003d self.get_container_listing(account, container, headers,"},{"line_number":189,"context_line":"                                          params\u003dparams)"},{"line_number":190,"context_line":"        return json.loads(resp.body)"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone,"}],"source_content_type":"text/x-python","patch_set":15,"id":"b1689cdf_29a1c136","line":189,"in_reply_to":"f76cb183_29a8dc97","updated":"2023-12-11 14:19:17.000000000","message":"Done","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ade21f3444a854583cded0338f60fb54f9d8b53","unresolved":true,"context_lines":[{"line_number":2944,"context_line":"        self.assertEqual(all_obj_names, [obj[\u0027name\u0027] for obj in objs])"},{"line_number":2945,"context_line":""},{"line_number":2946,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges()"},{"line_number":2947,"context_line":"        self.assertFalse(shard_ranges)"},{"line_number":2948,"context_line":""},{"line_number":2949,"context_line":"        # Shard the container"},{"line_number":2950,"context_line":"        client.post_container(self.url, self.admin_token, self.container_name,"}],"source_content_type":"text/x-python","patch_set":15,"id":"bd08b55b_3ed37680","line":2947,"updated":"2023-12-08 16:48:11.000000000","message":"these are helpful!","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":false,"context_lines":[{"line_number":2944,"context_line":"        self.assertEqual(all_obj_names, [obj[\u0027name\u0027] for obj in objs])"},{"line_number":2945,"context_line":""},{"line_number":2946,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges()"},{"line_number":2947,"context_line":"        self.assertFalse(shard_ranges)"},{"line_number":2948,"context_line":""},{"line_number":2949,"context_line":"        # Shard the container"},{"line_number":2950,"context_line":"        client.post_container(self.url, self.admin_token, self.container_name,"}],"source_content_type":"text/x-python","patch_set":15,"id":"03a42ee6_12f2b36e","line":2947,"in_reply_to":"bd08b55b_3ed37680","updated":"2023-12-11 14:19:17.000000000","message":"Acknowledged","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ade21f3444a854583cded0338f60fb54f9d8b53","unresolved":true,"context_lines":[{"line_number":2960,"context_line":"            additional_args\u003d\u0027--partitions\u003d%s\u0027 % self.brain.part)"},{"line_number":2961,"context_line":"        # sanity check"},{"line_number":2962,"context_line":"        for node in self.brain.nodes:"},{"line_number":2963,"context_line":"            self.assert_container_state(node, \u0027sharded\u0027, 2)"},{"line_number":2964,"context_line":""},{"line_number":2965,"context_line":"        def check_equivalence(this, that):"},{"line_number":2966,"context_line":"            self.assertEqual(len(this), len(that))"}],"source_content_type":"text/x-python","patch_set":15,"id":"42e40096_4aa5302b","line":2963,"updated":"2023-12-08 16:48:11.000000000","message":"this \"paragraph\" might look good extracted to some common base/helper_method","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":false,"context_lines":[{"line_number":2960,"context_line":"            additional_args\u003d\u0027--partitions\u003d%s\u0027 % self.brain.part)"},{"line_number":2961,"context_line":"        # sanity check"},{"line_number":2962,"context_line":"        for node in self.brain.nodes:"},{"line_number":2963,"context_line":"            self.assert_container_state(node, \u0027sharded\u0027, 2)"},{"line_number":2964,"context_line":""},{"line_number":2965,"context_line":"        def check_equivalence(this, that):"},{"line_number":2966,"context_line":"            self.assertEqual(len(this), len(that))"}],"source_content_type":"text/x-python","patch_set":15,"id":"ff8945e6_349dcf3a","line":2963,"in_reply_to":"42e40096_4aa5302b","updated":"2023-12-11 14:19:17.000000000","message":"Done","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9ade21f3444a854583cded0338f60fb54f9d8b53","unresolved":true,"context_lines":[{"line_number":2968,"context_line":"                [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":2969,"context_line":"                 for sr in this],"},{"line_number":2970,"context_line":"                [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":2971,"context_line":"                 for sr in that])"},{"line_number":2972,"context_line":""},{"line_number":2973,"context_line":"        orig_shard_ranges \u003d self.get_container_shard_ranges()"},{"line_number":2974,"context_line":"        self.assertEqual(2, len(orig_shard_ranges))"}],"source_content_type":"text/x-python","patch_set":15,"id":"87fe02df_8dc88f89","line":2971,"updated":"2023-12-08 16:48:11.000000000","message":"i know this is a pretty trivial closure but it would look better to me as a method.","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e3606ddcef3457344023ca37cacf8b98c196924e","unresolved":false,"context_lines":[{"line_number":2968,"context_line":"                [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":2969,"context_line":"                 for sr in this],"},{"line_number":2970,"context_line":"                [Namespace(sr.name, sr.lower, sr.upper)"},{"line_number":2971,"context_line":"                 for sr in that])"},{"line_number":2972,"context_line":""},{"line_number":2973,"context_line":"        orig_shard_ranges \u003d self.get_container_shard_ranges()"},{"line_number":2974,"context_line":"        self.assertEqual(2, len(orig_shard_ranges))"}],"source_content_type":"text/x-python","patch_set":15,"id":"3cf9b101_c8010b3c","line":2971,"in_reply_to":"87fe02df_8dc88f89","updated":"2023-12-11 14:19:17.000000000","message":"Done","commit_id":"5df6969462d5bbc53b50a74d71867acbcc77968a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":175,"context_line":"                conn.delete_object(self.container_name, obj)"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"    def get_container_listing(self, account\u003dNone, container\u003dNone,"},{"line_number":178,"context_line":"                              headers\u003dNone, params\u003dNone):"},{"line_number":179,"context_line":"        account \u003d account if account else self.account"},{"line_number":180,"context_line":"        container \u003d container if container else self.container_to_shard"},{"line_number":181,"context_line":"        path \u003d self.internal_client.make_path(account, container)"}],"source_content_type":"text/x-python","patch_set":17,"id":"4a68b413_628c8782","line":178,"updated":"2023-12-13 00:24:48.000000000","message":"more complete interface; better.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":193,"context_line":"        if req_record_type and req_record_type.lower() \u003d\u003d \u0027object\u0027:"},{"line_number":194,"context_line":"            self.assertEqual(\u0027object\u0027, resp_record_type)"},{"line_number":195,"context_line":"        else:"},{"line_number":196,"context_line":"            self.assertIsNone(resp_record_type)"},{"line_number":197,"context_line":"        return json.loads(resp.body)"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"    def get_container_shard_ranges(self, account\u003dNone, container\u003dNone,"}],"source_content_type":"text/x-python","patch_set":17,"id":"35fa202f_ba9d3f47","line":196,"updated":"2023-12-13 00:24:48.000000000","message":"tests cover req_record_type \u003d\u003d \u0027auto\u0027, \u0027banana\u0027, and None - and this is always the result.\n\nas expected if the test calls this method with x-backend-record-type: shard ...\n\n * on an unsharded container it gets an empty response with x-backend-record-type: shard\n * on a sharded container, it gets back a list of two shards with x-backend-record-type: shard","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":408,"context_line":""},{"line_number":409,"context_line":"    def assert_container_states(self, expected_state, num_shard_ranges):"},{"line_number":410,"context_line":"        for node in self.brain.nodes:"},{"line_number":411,"context_line":"            self.assert_container_state(node, expected_state, num_shard_ranges)"},{"line_number":412,"context_line":""},{"line_number":413,"context_line":"    def assert_subprocess_success(self, cmd_args):"},{"line_number":414,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":17,"id":"a45330f5_3297792b","line":411,"updated":"2023-12-13 00:24:48.000000000","message":"seems like a useful helper.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":465,"context_line":"            additional_args \u003d [additional_args]"},{"line_number":466,"context_line":"        additional_args.append(\u0027--no-auto-shard\u0027)"},{"line_number":467,"context_line":"        kwargs[\u0027additional_args\u0027] \u003d additional_args"},{"line_number":468,"context_line":"        self.sharders.once(**kwargs)"},{"line_number":469,"context_line":""},{"line_number":470,"context_line":""},{"line_number":471,"context_line":"class BaseAutoContainerSharding(BaseTestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":17,"id":"d5bfa4ce_b461bd87","line":468,"updated":"2023-12-13 00:24:48.000000000","message":"i\u0027m a little confused about the proliferation of this helper; a lot of these tests require configs to have auto_shard \u003d true, but now we\u0027re forcing the shards to ignore it\n\nthat\u0027s probably fine if we\u0027re not expecting them to do any auto sharding; but there\u0027s lots of places we still call self.sharders.once directly - so how did you choose which ones to migrate?","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b10a8c936c85fa363b985e46e45f37debf2fa7b4","unresolved":true,"context_lines":[{"line_number":465,"context_line":"            additional_args \u003d [additional_args]"},{"line_number":466,"context_line":"        additional_args.append(\u0027--no-auto-shard\u0027)"},{"line_number":467,"context_line":"        kwargs[\u0027additional_args\u0027] \u003d additional_args"},{"line_number":468,"context_line":"        self.sharders.once(**kwargs)"},{"line_number":469,"context_line":""},{"line_number":470,"context_line":""},{"line_number":471,"context_line":"class BaseAutoContainerSharding(BaseTestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":17,"id":"85be99d2_e6fb1ded","line":468,"in_reply_to":"d5bfa4ce_b461bd87","updated":"2023-12-13 11:23:34.000000000","message":"This is only used in TestManagedContainerSharding (all of those tests are deliberately non-auto) and the new test.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":2991,"context_line":"        # shard_ranges \u003d self.get_container_shard_ranges("},{"line_number":2992,"context_line":"        #     headers\u003d{\u0027X-Newest\u0027: \u0027true\u0027},"},{"line_number":2993,"context_line":"        #     params\u003d{\u0027states\u0027: \u0027listing\u0027})"},{"line_number":2994,"context_line":"        # self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)"},{"line_number":2995,"context_line":""},{"line_number":2996,"context_line":"        # this is what the sharder requests..."},{"line_number":2997,"context_line":"        shard_ranges \u003d self.get_container_shard_ranges("}],"source_content_type":"text/x-python","patch_set":17,"id":"e5b81140_2a11957b","line":2994,"updated":"2023-12-13 00:24:48.000000000","message":"both of these blow-up with KeyError on Timestamp when the synthesized-from-cache response tries to get converted to ShardRanges; very spooky.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3051,"context_line":"        # note: explicitly asking for the root object rows, but it has None"},{"line_number":3052,"context_line":"        objs \u003d self.get_container_objects("},{"line_number":3053,"context_line":"            headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027object\u0027})"},{"line_number":3054,"context_line":"        self.assertEqual([], objs)"},{"line_number":3055,"context_line":""},{"line_number":3056,"context_line":""},{"line_number":3057,"context_line":"class TestContainerShardingMoreUTF8(TestContainerSharding):"}],"source_content_type":"text/x-python","patch_set":17,"id":"58c861fa_2d912680","line":3054,"updated":"2023-12-13 00:24:48.000000000","message":"I love these tests and helpers; I can drop into a debugger and easily interogate the internal-client interface for x-backend-record-type","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":1246,"context_line":"    def test_fetch_shard_ranges_bad_data(self):"},{"line_number":1247,"context_line":"        def do_test(mock_resp_body):"},{"line_number":1248,"context_line":"            mock_resp_headers \u003d {\u0027x-backend-record-type\u0027: \u0027shard\u0027}"},{"line_number":1249,"context_line":"            with self._mock_sharder() as sharder:"},{"line_number":1250,"context_line":"                mock_make_request \u003d mock.MagicMock("},{"line_number":1251,"context_line":"                    return_value\u003dmock.MagicMock(headers\u003dmock_resp_headers,"},{"line_number":1252,"context_line":"                                                body\u003dmock_resp_body))"}],"source_content_type":"text/x-python","patch_set":17,"id":"235a8742_73763182","line":1249,"updated":"2023-12-13 00:24:48.000000000","message":"oic, _mock_sharder calls self.logger.clear()","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":1257,"context_line":"            self.assertFalse(lines[1:])"},{"line_number":1258,"context_line":""},{"line_number":1259,"context_line":"        broker \u003d self._make_broker()"},{"line_number":1260,"context_line":"        do_test({})"},{"line_number":1261,"context_line":"        do_test(\u0027\u0027)"},{"line_number":1262,"context_line":"        do_test(json.dumps({}))"},{"line_number":1263,"context_line":"        do_test(json.dumps([{\u0027account\u0027: \u0027a\u0027, \u0027container\u0027: \u0027c\u0027}]))"}],"source_content_type":"text/x-python","patch_set":17,"id":"f848f390_3c61ac26","line":1260,"updated":"2023-12-13 00:24:48.000000000","message":"TypeError(\u0027the JSON object must be str, bytes or bytearray, not dict\u0027),","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":1258,"context_line":""},{"line_number":1259,"context_line":"        broker \u003d self._make_broker()"},{"line_number":1260,"context_line":"        do_test({})"},{"line_number":1261,"context_line":"        do_test(\u0027\u0027)"},{"line_number":1262,"context_line":"        do_test(json.dumps({}))"},{"line_number":1263,"context_line":"        do_test(json.dumps([{\u0027account\u0027: \u0027a\u0027, \u0027container\u0027: \u0027c\u0027}]))"},{"line_number":1264,"context_line":"        do_test(json.dumps([dict(Namespace(\u0027a/c\u0027, \u0027l\u0027, \u0027u\u0027))]))"}],"source_content_type":"text/x-python","patch_set":17,"id":"d46627a5_3b36848d","line":1261,"updated":"2023-12-13 00:24:48.000000000","message":"JSONDecodeError(\u0027Expecting value: line 1 column 1 (char 0)\u0027)","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":1259,"context_line":"        broker \u003d self._make_broker()"},{"line_number":1260,"context_line":"        do_test({})"},{"line_number":1261,"context_line":"        do_test(\u0027\u0027)"},{"line_number":1262,"context_line":"        do_test(json.dumps({}))"},{"line_number":1263,"context_line":"        do_test(json.dumps([{\u0027account\u0027: \u0027a\u0027, \u0027container\u0027: \u0027c\u0027}]))"},{"line_number":1264,"context_line":"        do_test(json.dumps([dict(Namespace(\u0027a/c\u0027, \u0027l\u0027, \u0027u\u0027))]))"},{"line_number":1265,"context_line":"        sr_dict \u003d dict(ShardRange(\u0027a/c\u0027, next(self.ts_iter), \u0027l\u0027, \u0027u\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"d0a5f150_c6e8905f","line":1262,"updated":"2023-12-13 00:24:48.000000000","message":"invalid data: ValueError(\u0027not a list\u0027)","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":1260,"context_line":"        do_test({})"},{"line_number":1261,"context_line":"        do_test(\u0027\u0027)"},{"line_number":1262,"context_line":"        do_test(json.dumps({}))"},{"line_number":1263,"context_line":"        do_test(json.dumps([{\u0027account\u0027: \u0027a\u0027, \u0027container\u0027: \u0027c\u0027}]))"},{"line_number":1264,"context_line":"        do_test(json.dumps([dict(Namespace(\u0027a/c\u0027, \u0027l\u0027, \u0027u\u0027))]))"},{"line_number":1265,"context_line":"        sr_dict \u003d dict(ShardRange(\u0027a/c\u0027, next(self.ts_iter), \u0027l\u0027, \u0027u\u0027))"},{"line_number":1266,"context_line":"        sr_dict.pop(\u0027object_count\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"435c9e19_df1e3bc2","line":1263,"updated":"2023-12-13 00:24:48.000000000","message":"invalid data: KeyError(\u0027name\u0027)","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":1261,"context_line":"        do_test(\u0027\u0027)"},{"line_number":1262,"context_line":"        do_test(json.dumps({}))"},{"line_number":1263,"context_line":"        do_test(json.dumps([{\u0027account\u0027: \u0027a\u0027, \u0027container\u0027: \u0027c\u0027}]))"},{"line_number":1264,"context_line":"        do_test(json.dumps([dict(Namespace(\u0027a/c\u0027, \u0027l\u0027, \u0027u\u0027))]))"},{"line_number":1265,"context_line":"        sr_dict \u003d dict(ShardRange(\u0027a/c\u0027, next(self.ts_iter), \u0027l\u0027, \u0027u\u0027))"},{"line_number":1266,"context_line":"        sr_dict.pop(\u0027object_count\u0027)"},{"line_number":1267,"context_line":"        do_test(json.dumps([sr_dict]))"}],"source_content_type":"text/x-python","patch_set":17,"id":"083137a6_4fde4dca","line":1264,"updated":"2023-12-13 00:24:48.000000000","message":"invalid data: KeyError(\u0027timestamp\u0027)","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":1264,"context_line":"        do_test(json.dumps([dict(Namespace(\u0027a/c\u0027, \u0027l\u0027, \u0027u\u0027))]))"},{"line_number":1265,"context_line":"        sr_dict \u003d dict(ShardRange(\u0027a/c\u0027, next(self.ts_iter), \u0027l\u0027, \u0027u\u0027))"},{"line_number":1266,"context_line":"        sr_dict.pop(\u0027object_count\u0027)"},{"line_number":1267,"context_line":"        do_test(json.dumps([sr_dict]))"},{"line_number":1268,"context_line":""},{"line_number":1269,"context_line":"    def test_fetch_shard_ranges_ok(self):"},{"line_number":1270,"context_line":"        def do_test(mock_resp_body, params):"}],"source_content_type":"text/x-python","patch_set":17,"id":"6c862c3d_738b4d6d","line":1267,"updated":"2023-12-13 00:24:48.000000000","message":"invalid data: KeyError(\u0027object_count\u0027)","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":9704,"context_line":""},{"line_number":9705,"context_line":"        def assert_ok(conf):"},{"line_number":9706,"context_line":"            try:"},{"line_number":9707,"context_line":"                ContainerSharderConf.validate_conf(argparse.Namespace(**conf))"},{"line_number":9708,"context_line":"            except ValueError as err:"},{"line_number":9709,"context_line":"                self.fail(\u0027Unexpected ValueError: %s\u0027 % err)"},{"line_number":9710,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"7cc02293_c1a72331","line":9707,"updated":"2023-12-13 00:24:48.000000000","message":"trolololo, literal namespace collision!","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b10a8c936c85fa363b985e46e45f37debf2fa7b4","unresolved":true,"context_lines":[{"line_number":9704,"context_line":""},{"line_number":9705,"context_line":"        def assert_ok(conf):"},{"line_number":9706,"context_line":"            try:"},{"line_number":9707,"context_line":"                ContainerSharderConf.validate_conf(argparse.Namespace(**conf))"},{"line_number":9708,"context_line":"            except ValueError as err:"},{"line_number":9709,"context_line":"                self.fail(\u0027Unexpected ValueError: %s\u0027 % err)"},{"line_number":9710,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"e02fb0d9_e51098bb","line":9707,"in_reply_to":"7cc02293_c1a72331","updated":"2023-12-13 11:23:34.000000000","message":"I already got confused just seeing Namespace in these tests, so I wanted a bare Namespace to be *our* Namespace","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"}],"test/unit/proxy/controllers/test_container.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":2792,"context_line":"        self._check_response(resp, self.ns_dicts, {"},{"line_number":2793,"context_line":"            \u0027X-Backend-Recheck-Container-Existence\u0027: \u002760\u0027,"},{"line_number":2794,"context_line":"            \u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":2795,"context_line":"            \u0027X-Backend-Sharding-State\u0027: sharding_state})"},{"line_number":2796,"context_line":"        self.assertEqual("},{"line_number":2797,"context_line":"            [mock.call.get(\u0027container/a/c\u0027),"},{"line_number":2798,"context_line":"             mock.call.get(cache_key, raise_on_error\u003dTrue),"}],"source_content_type":"text/x-python","patch_set":17,"id":"3655a695_1d864a7d","side":"PARENT","line":2795,"updated":"2023-12-13 00:24:48.000000000","message":"ok, and sharding_state wasn\u0027t manipulated so you can re-ues the same dict.","commit_id":"93f812a5186f31bcdfcfb97f7f885605ada6819a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":505,"context_line":"        bodies \u003d iter([json.dumps(resp[1]).encode(\u0027ascii\u0027)"},{"line_number":506,"context_line":"                       for resp in mock_responses])"},{"line_number":507,"context_line":"        exp_headers \u003d [resp[2] for resp in mock_responses]"},{"line_number":508,"context_line":"        request \u003d Request.blank(container_path)"},{"line_number":509,"context_line":"        if req_headers:"},{"line_number":510,"context_line":"            request.headers.update(req_headers)"},{"line_number":511,"context_line":"        if memcache:"}],"source_content_type":"text/x-python","patch_set":17,"id":"23cfa06f_69c583b7","line":508,"updated":"2023-12-13 00:24:48.000000000","message":"blank takes a headers kwarg that defaults to None","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":false,"context_lines":[{"line_number":568,"context_line":"                         int(resp.headers[\u0027X-Container-Object-Count\u0027]))"},{"line_number":569,"context_line":"        self.assertEqual(exp_sharding_state,"},{"line_number":570,"context_line":"                         resp.headers[\u0027X-Backend-Sharding-State\u0027])"},{"line_number":571,"context_line":"        self.assertNotIn(\u0027X-Backend-Record-Type\u0027, resp.headers)"},{"line_number":572,"context_line":"        for k, v in root_resp_hdrs.items():"},{"line_number":573,"context_line":"            if k.lower().startswith(\u0027x-container-meta\u0027):"},{"line_number":574,"context_line":"                self.assertEqual(v, resp.headers[k])"}],"source_content_type":"text/x-python","patch_set":17,"id":"557a4f0f_afc54b79","line":571,"updated":"2023-12-13 00:24:48.000000000","message":"this hits a few tests\n\ntest_GET_sharded_container_with_memcache\ntest_GET_sharded_container_no_memcache\ntest_GET_sharded_container_with_delimiter\ntest_GET_sharded_container_with_delimiter_and_reverse\netc.\n\nall of which are making client-listing requests, and fail like:\n\n    swift/test/unit/proxy/controllers/test_container.py:569: in check_response\n        self.assertNotIn(\u0027X-Backend-Record-Type\u0027, resp.headers)\n    E   AssertionError: \u0027X-Backend-Record-Type\u0027 unexpectedly found in {\u0027Content-Type\u0027: \u0027x-application/test\u0027, \u0027Content-Length\u0027: \u0027151621\u0027, \u0027X-Timestamp\u0027: \u00271\u0027, \u0027X-Backend-Timestamp\u0027: \u002799\u0027, \u0027Last-Modified\u0027: \u00271\u0027, \u0027X-Object-Meta-Test\u0027: \u0027testing\u0027, \u0027X-Delete-At\u0027: \u00279876543210\u0027, \u0027Etag\u0027: \u00274060b1ef0b80615cf2a75a50762ef999\u0027, \u0027X-Works\u0027: \u0027yes\u0027, \u0027X-Account-Container-Count\u0027: \u002712345\u0027, \u0027X-Backend-Sharding-State\u0027: \u0027sharded\u0027, \u0027X-Container-Object-Count\u0027: \u00271032\u0027, \u0027X-Container-Bytes-Used\u0027: \u0027640356\u0027, \u0027X-Container-Meta-Flavour\u0027: \u0027peach\u0027, \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027, \u0027X-Backend-Record-Type\u0027: \u0027shard\u0027, \u0027Accept-Ranges\u0027: \u0027bytes\u0027, \u0027X-Storage-Policy\u0027: \u0027zero\u0027, \u0027X-Backend-Recheck-Container-Existence\u0027: \u002760\u0027}\n\n\n... with this change reverted.  Note the pre-existing `\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027`","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":675,"context_line":"            mock_responses, expected_objects, expected_requests,"},{"line_number":676,"context_line":"            req_headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027banana\u0027})"},{"line_number":677,"context_line":"        self.check_response(resp, root_resp_hdrs,"},{"line_number":678,"context_line":"                            expected_objects\u003dexpected_objects)"},{"line_number":679,"context_line":""},{"line_number":680,"context_line":"        # GET all objects - sharding, final shard range points back to root"},{"line_number":681,"context_line":"        root_range \u003d ShardRange(\u0027a/c\u0027, Timestamp.now(), \u0027pie\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"254046c6_3fd9fbfb","line":678,"updated":"2023-12-13 00:24:48.000000000","message":"all client listing responses; makes sense.\n\nFWIW the stubbed responses don\u0027t respect the requested record-type; besides getting errors about un-used requests if you call _check_GET_shard_listing with x-backend-record-type \u0027object\u0027 you get x-backend-record-type shard in the resp because we set root_shard_resp_hdrs[\u0027x-backend-record-type\u0027] \u003d shard","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b10a8c936c85fa363b985e46e45f37debf2fa7b4","unresolved":false,"context_lines":[{"line_number":675,"context_line":"            mock_responses, expected_objects, expected_requests,"},{"line_number":676,"context_line":"            req_headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027banana\u0027})"},{"line_number":677,"context_line":"        self.check_response(resp, root_resp_hdrs,"},{"line_number":678,"context_line":"                            expected_objects\u003dexpected_objects)"},{"line_number":679,"context_line":""},{"line_number":680,"context_line":"        # GET all objects - sharding, final shard range points back to root"},{"line_number":681,"context_line":"        root_range \u003d ShardRange(\u0027a/c\u0027, Timestamp.now(), \u0027pie\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"1dc305e8_4a3f74ec","line":678,"in_reply_to":"254046c6_3fd9fbfb","updated":"2023-12-13 11:23:34.000000000","message":"Acknowledged","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":1074,"context_line":"            mock_responses, expected_objects, expected_requests, memcache\u003dTrue,"},{"line_number":1075,"context_line":"            req_headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027banana\u0027})"},{"line_number":1076,"context_line":"        self.check_response(resp, root_resp_hdrs,"},{"line_number":1077,"context_line":"                            expected_objects\u003dexpected_objects)"},{"line_number":1078,"context_line":""},{"line_number":1079,"context_line":"        # GET all objects - sharding, final shard range points back to root"},{"line_number":1080,"context_line":"        root_range \u003d ShardRange(\u0027a/c\u0027, Timestamp.now(), \u0027pie\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"17da6ee4_6a1b22e3","line":1077,"updated":"2023-12-13 00:24:48.000000000","message":"same deal here; any x-backend-record-type not in (object, shard) means client-listing requests","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b10a8c936c85fa363b985e46e45f37debf2fa7b4","unresolved":false,"context_lines":[{"line_number":1074,"context_line":"            mock_responses, expected_objects, expected_requests, memcache\u003dTrue,"},{"line_number":1075,"context_line":"            req_headers\u003d{\u0027X-Backend-Record-Type\u0027: \u0027banana\u0027})"},{"line_number":1076,"context_line":"        self.check_response(resp, root_resp_hdrs,"},{"line_number":1077,"context_line":"                            expected_objects\u003dexpected_objects)"},{"line_number":1078,"context_line":""},{"line_number":1079,"context_line":"        # GET all objects - sharding, final shard range points back to root"},{"line_number":1080,"context_line":"        root_range \u003d ShardRange(\u0027a/c\u0027, Timestamp.now(), \u0027pie\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"f53e27e2_c6f7a75a","line":1077,"in_reply_to":"17da6ee4_6a1b22e3","updated":"2023-12-13 11:23:34.000000000","message":"Acknowledged","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":2770,"context_line":"            \u0027X-Backend-Cached-Results\u0027: \u0027true\u0027,"},{"line_number":2771,"context_line":"            \u0027X-Backend-Sharding-State\u0027: sharding_state}"},{"line_number":2772,"context_line":"        if exp_record_type:"},{"line_number":2773,"context_line":"            exp_cache_resp_headers[\u0027X-Backend-Record-Type\u0027] \u003d exp_record_type"},{"line_number":2774,"context_line":"        self.memcache.delete_all()"},{"line_number":2775,"context_line":"        # container is sharded but proxy does not have that state cached;"},{"line_number":2776,"context_line":"        # expect a backend request and expect shard ranges to be cached"}],"source_content_type":"text/x-python","patch_set":17,"id":"c88a6c4e_f481265d","line":2773,"updated":"2023-12-13 00:24:48.000000000","message":"ok, if we exp_record_type - we expect it in both sets of headers and to be the same.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3048,"context_line":"        self._do_test_caching(\u0027shard\u0027, 600, \u0027shard\u0027)"},{"line_number":3049,"context_line":"        # expect shard ranges cache time to be configured value of 120"},{"line_number":3050,"context_line":"        self.app.recheck_listing_shard_ranges \u003d 120"},{"line_number":3051,"context_line":"        self._do_test_caching(\u0027shard\u0027, 120, \u0027shard\u0027)"},{"line_number":3052,"context_line":""},{"line_number":3053,"context_line":"        def mock_get_from_shards(self, req, resp):"},{"line_number":3054,"context_line":"            # for the purposes of these tests we override _get_from_shards so"}],"source_content_type":"text/x-python","patch_set":17,"id":"a5e0082c_ff122199","line":3051,"updated":"2023-12-13 00:24:48.000000000","message":"ok, so both of these explicitly request x-backend-record-type: shard from the proxy","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b10a8c936c85fa363b985e46e45f37debf2fa7b4","unresolved":true,"context_lines":[{"line_number":3048,"context_line":"        self._do_test_caching(\u0027shard\u0027, 600, \u0027shard\u0027)"},{"line_number":3049,"context_line":"        # expect shard ranges cache time to be configured value of 120"},{"line_number":3050,"context_line":"        self.app.recheck_listing_shard_ranges \u003d 120"},{"line_number":3051,"context_line":"        self._do_test_caching(\u0027shard\u0027, 120, \u0027shard\u0027)"},{"line_number":3052,"context_line":""},{"line_number":3053,"context_line":"        def mock_get_from_shards(self, req, resp):"},{"line_number":3054,"context_line":"            # for the purposes of these tests we override _get_from_shards so"}],"source_content_type":"text/x-python","patch_set":17,"id":"5015877c_03b5454e","line":3051,"in_reply_to":"a5e0082c_ff122199","updated":"2023-12-13 11:23:34.000000000","message":"this is what we\u0027re going to fix later in the patch chain - if the client requests \u0027shard\u0027 then we shouldn\u0027t be going anywhere near the cache!! but these tests actually assert that cache is used 😞","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3056,"context_line":"            # record_type is \u0027auto\u0027; these tests are verifying the content and"},{"line_number":3057,"context_line":"            # caching of the backend shard range response so we\u0027re not"},{"line_number":3058,"context_line":"            # interested in gathering object from the shards"},{"line_number":3059,"context_line":"            return resp"},{"line_number":3060,"context_line":""},{"line_number":3061,"context_line":"        with mock.patch(\u0027swift.proxy.controllers.container.\u0027"},{"line_number":3062,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"}],"source_content_type":"text/x-python","patch_set":17,"id":"9e8f833f_22817a92","line":3059,"updated":"2023-12-13 00:24:48.000000000","message":"yikes and in _do_test_caching despite asking for x-backend-record-type: auto the helper asserts we get a shard response - which is only the case the proxy goes into this mocked code path and returns the shard listing response that was passed in...\n\nI added some code to capture the calls into this mocked function and I\u0027m quite sure I don\u0027t understand what\u0027s actually happening in this test.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b10a8c936c85fa363b985e46e45f37debf2fa7b4","unresolved":true,"context_lines":[{"line_number":3056,"context_line":"            # record_type is \u0027auto\u0027; these tests are verifying the content and"},{"line_number":3057,"context_line":"            # caching of the backend shard range response so we\u0027re not"},{"line_number":3058,"context_line":"            # interested in gathering object from the shards"},{"line_number":3059,"context_line":"            return resp"},{"line_number":3060,"context_line":""},{"line_number":3061,"context_line":"        with mock.patch(\u0027swift.proxy.controllers.container.\u0027"},{"line_number":3062,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"}],"source_content_type":"text/x-python","patch_set":17,"id":"ccd8c93f_fd8acae3","line":3059,"in_reply_to":"9e8f833f_22817a92","updated":"2023-12-13 11:23:34.000000000","message":"this is very dubious and I do try to straighten it out later in the patch chain.\n\nbut, FWIW, currently we are mocking get_from_shards to return the list of namespaces instead of a list of objects 😭","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3062,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"},{"line_number":3063,"context_line":"                        mock_get_from_shards):"},{"line_number":3064,"context_line":"            self.app.recheck_listing_shard_ranges \u003d 600"},{"line_number":3065,"context_line":"            self._do_test_caching(\u0027auto\u0027, 600, exp_record_type\u003dNone)"},{"line_number":3066,"context_line":""},{"line_number":3067,"context_line":"    def test_GET_shard_ranges_404_response(self):"},{"line_number":3068,"context_line":"        # pre-warm cache with container info but not shard ranges so that the"}],"source_content_type":"text/x-python","patch_set":17,"id":"6fe41279_e0b58316","line":3065,"updated":"2023-12-13 00:24:48.000000000","message":"this is kind of sneaky bonus coverage that\u0027s hi-jacking the stubs setup for the other cache tests so that it can assert \"we also cache when we do client-listings\" w/o having to actually do the listings in the shards.\n\nThe assertions all make sense tho, if a client sent x-backend-record-type: banana they\u0027d get back a resp w/o an record-type header as we\u0027ve seen in other tests.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3190,"context_line":"            return resp"},{"line_number":3191,"context_line":""},{"line_number":3192,"context_line":"        # request with record-type\u003dauto does not expect record-type in response"},{"line_number":3193,"context_line":"        del exp_hdrs[\u0027X-Backend-Record-Type\u0027]"},{"line_number":3194,"context_line":"        with mock.patch(\u0027swift.proxy.controllers.container.\u0027"},{"line_number":3195,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"},{"line_number":3196,"context_line":"                        mock_get_from_shards):"}],"source_content_type":"text/x-python","patch_set":17,"id":"f55dc32a_605a332d","line":3193,"updated":"2023-12-13 00:24:48.000000000","message":"ok, same pattern as before - and since client requests x-backend-record-type auto we expect resp header not to have the key x-backend-record-type","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3198,"context_line":"                {\u0027states\u0027: \u0027listing\u0027, \u0027reverse\u0027: \u0027true\u0027}, \u0027auto\u0027)"},{"line_number":3199,"context_line":"            exp_shards \u003d list(self.ns_dicts)"},{"line_number":3200,"context_line":"            exp_shards.reverse()"},{"line_number":3201,"context_line":"            self._check_response(resp, exp_shards, exp_hdrs)"},{"line_number":3202,"context_line":""},{"line_number":3203,"context_line":"            resp \u003d self._do_test_GET_shard_ranges_read_from_cache("},{"line_number":3204,"context_line":"                {\u0027states\u0027: \u0027listing\u0027, \u0027marker\u0027: \u0027jam\u0027}, \u0027auto\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"1b989735_22ee8825","line":3201,"updated":"2023-12-13 00:24:48.000000000","message":"in this case _do_test_GET_shard_ranges_read_from_cache is a bit more forgiving on what comes back from _get_from_shards; we probably don\u0027t need to use _check_response here.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3287,"context_line":"            return resp"},{"line_number":3288,"context_line":""},{"line_number":3289,"context_line":"        # request with record-type\u003dauto does not expect record-type in response"},{"line_number":3290,"context_line":"        del exp_hdrs[\u0027X-Backend-Record-Type\u0027]"},{"line_number":3291,"context_line":"        with mock.patch(\u0027swift.proxy.controllers.container.\u0027"},{"line_number":3292,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"},{"line_number":3293,"context_line":"                        mock_get_from_shards):"}],"source_content_type":"text/x-python","patch_set":17,"id":"19e9861f_4f082b4c","line":3290,"updated":"2023-12-13 00:24:48.000000000","message":"I\u0027m growing less sure of these tests the more time I spend with them, but that\u0027s pre-existing - this assertion makes perfect sense I\u0027m glad the tests despite their strange stubbing didn\u0027t get to avoid recognizing the change.","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a5f3f54e1e81aa38c99524fa987191de8c9e98b4","unresolved":true,"context_lines":[{"line_number":3308,"context_line":""},{"line_number":3309,"context_line":"            resp \u003d self._do_test_GET_shard_ranges_write_to_cache("},{"line_number":3310,"context_line":"                {\u0027states\u0027: \u0027listing\u0027, \u0027includes\u0027: \u0027egg\u0027}, \u0027auto\u0027)"},{"line_number":3311,"context_line":"            self._check_response(resp, self.ns_dicts[:1], exp_hdrs)"},{"line_number":3312,"context_line":""},{"line_number":3313,"context_line":"    def test_GET_shard_ranges_write_to_cache_with_x_newest(self):"},{"line_number":3314,"context_line":"        # when x-newest is sent, verify that there is no cache lookup to check"}],"source_content_type":"text/x-python","patch_set":17,"id":"ecac02de_78c77ad6","line":3311,"updated":"2023-12-13 00:24:48.000000000","message":"_get_from_shards takes namespaces and returns a client listing response right?\n\nwho\u0027s even doing this manipulation of this stubbed response based on query params?\n\nit would be bad if either\n\na) a client sent includes and got filtered shard ranges\nb) an internal-client asked for auto and got back filtered shard ranges","commit_id":"71ad062bc34e58fc7a909b0e7299d12bc47e6382"}]}