)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     Matthew Oliver \u003cmatt@oliver.net.au\u003e"},{"line_number":5,"context_line":"CommitDate: 2021-08-05 14:55:04 +1000"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"conatiner-server: return objects of a given policy"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"There is a tight coupling between a root container and it\u0027s shards and"},{"line_number":10,"context_line":"there can be a tail before a roots storage policy index is migrated to"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"d48100b4_80badcbc","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":9},"updated":"2021-08-05 11:57:11.000000000","message":"sp: container","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":7,"context_line":"conatiner-server: return objects of a given policy"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"There is a tight coupling between a root container and it\u0027s shards and"},{"line_number":10,"context_line":"there can be a tail before a roots storage policy index is migrated to"},{"line_number":11,"context_line":"it\u0027s shards. Especially in the case of a delete and creating the new"},{"line_number":12,"context_line":"container again with a new policy."},{"line_number":13,"context_line":"When we PUT objects in to a root container, it\u0027ll redirect them, with"},{"line_number":14,"context_line":"the roots policy to the shards. And the shards are happy to take them."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"d0a22281_f66d273f","line":11,"range":{"start_line":10,"start_character":29,"end_line":11,"end_character":11},"updated":"2021-08-05 11:57:11.000000000","message":"currently there is no policy migration so these statement don\u0027t really make sense for this patch","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":12,"context_line":"container again with a new policy."},{"line_number":13,"context_line":"When we PUT objects in to a root container, it\u0027ll redirect them, with"},{"line_number":14,"context_line":"the roots policy to the shards. And the shards are happy to take them."},{"line_number":15,"context_line":"But when it comes to GETs, the root forwards the GET onto it\u0027s shards"},{"line_number":16,"context_line":"whom currently wont respond with objects (which they probably took) becuase"},{"line_number":17,"context_line":"they are of a different policy. When getting objects from the container"},{"line_number":18,"context_line":"server, the policy used is always the brokers policy."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"8ae4366a_e8003b3c","line":15,"range":{"start_line":15,"start_character":31,"end_line":15,"end_character":44},"updated":"2021-08-05 11:57:11.000000000","message":"nit: it redirects rather than forwards","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":13,"context_line":"When we PUT objects in to a root container, it\u0027ll redirect them, with"},{"line_number":14,"context_line":"the roots policy to the shards. And the shards are happy to take them."},{"line_number":15,"context_line":"But when it comes to GETs, the root forwards the GET onto it\u0027s shards"},{"line_number":16,"context_line":"whom currently wont respond with objects (which they probably took) becuase"},{"line_number":17,"context_line":"they are of a different policy. When getting objects from the container"},{"line_number":18,"context_line":"server, the policy used is always the brokers policy."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"6d3c56d8_10c70681","line":16,"range":{"start_line":16,"start_character":68,"end_line":16,"end_character":75},"updated":"2021-08-05 11:57:11.000000000","message":"s/becuase/because/","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":20,"context_line":"This patch corrects this behaviour by allowing the ability to override"},{"line_number":21,"context_line":"the policy index to use. If the request to the container server"},{"line_number":22,"context_line":"contains an \u0027X-Backend-Storage-Policy-Index\u0027 header it\u0027ll be used"},{"line_number":23,"context_line":"instead of the policy index stored in the broker. This header isn\u0027t"},{"line_number":24,"context_line":"added between the Proxy -\u003e Container servers by default."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"This patch adds the root containers policy as this header to the proxy"},{"line_number":27,"context_line":"container controllers `_get_from_shards` method which is used to build up"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"7c060b63_c5523dbc","line":24,"range":{"start_line":23,"start_character":50,"end_line":24,"end_character":55},"updated":"2021-08-05 11:57:11.000000000","message":"not sure what this means - with this patch the header *is* added by default? does \u0027by default\u0027 mean for a non-sharded container GET.","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"}],"swift/container/server.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":773,"context_line":"                return HTTPNotFound(request\u003dreq, headers\u003dresp_headers)"},{"line_number":774,"context_line":"            resp_headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":775,"context_line":"            storage_policy_index \u003d info[\u0027storage_policy_index\u0027]"},{"line_number":776,"context_line":"            if req.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027):"},{"line_number":777,"context_line":"                storage_policy_index \u003d \\"},{"line_number":778,"context_line":"                    req.headers[\u0027X-Backend-Storage-Policy-Index\u0027]"},{"line_number":779,"context_line":"                resp_headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \\"}],"source_content_type":"text/x-python","patch_set":3,"id":"b2e5133d_7f1c0142","line":776,"range":{"start_line":776,"start_character":15,"end_line":776,"end_character":64},"updated":"2021-08-05 11:57:11.000000000","message":"we should validate this header using \n\n  self.get_and_validate_policy_index(req)","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":777,"context_line":"                storage_policy_index \u003d \\"},{"line_number":778,"context_line":"                    req.headers[\u0027X-Backend-Storage-Policy-Index\u0027]"},{"line_number":779,"context_line":"                resp_headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \\"},{"line_number":780,"context_line":"                    storage_policy_index"},{"line_number":781,"context_line":"            # Use the retired db while container is in process of sharding,"},{"line_number":782,"context_line":"            # otherwise use current db"},{"line_number":783,"context_line":"            src_broker \u003d broker.get_brokers()[0]"}],"source_content_type":"text/x-python","patch_set":3,"id":"91abadc5_84f57b23","line":780,"updated":"2021-08-05 11:57:11.000000000","message":"might be worth always returning this header in response even if not in request","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":70,"context_line":"            info.get(\u0027delete_timestamp\u0027, 0)).internal,"},{"line_number":71,"context_line":"        \u0027X-Backend-Status-Changed-At\u0027: Timestamp("},{"line_number":72,"context_line":"            info.get(\u0027status_changed_at\u0027, 0)).internal,"},{"line_number":73,"context_line":"        \u0027X-Backend-Storage-Policy-Index\u0027: info.get(\u0027storage_policy_index\u0027, 0),"},{"line_number":74,"context_line":"    }"},{"line_number":75,"context_line":"    if not is_deleted:"},{"line_number":76,"context_line":"        # base container info on deleted containers is not exposed to client"}],"source_content_type":"text/x-python","patch_set":6,"id":"b9fa3697_77d14779","line":73,"updated":"2021-08-10 18:16:37.000000000","message":"OK, so this has been included ever since gen_resp_headers was a thing: https://github.com/openstack/swift/commit/4321bb0","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":734,"context_line":"        if record_type \u003d\u003d \u0027shard\u0027:"},{"line_number":735,"context_line":"            override_deleted \u003d info and config_true_value("},{"line_number":736,"context_line":"                req.headers.get(\u0027x-backend-override-deleted\u0027, False))"},{"line_number":737,"context_line":"            resp_headers \u003d gen_resp_headers("},{"line_number":738,"context_line":"                info, is_deleted\u003dis_deleted and not override_deleted)"},{"line_number":739,"context_line":"            if is_deleted and not override_deleted:"},{"line_number":740,"context_line":"                return HTTPNotFound(request\u003dreq, headers\u003dresp_headers)"}],"source_content_type":"text/x-python","patch_set":6,"id":"bbf97955_23b79d47","line":737,"range":{"start_line":737,"start_character":0,"end_line":737,"end_character":43},"updated":"2021-08-10 18:16:37.000000000","message":"And shard responses have used this ever since the container learned how to make them: https://github.com/openstack/swift/commit/723eac9\n\nSo shard responses *always* send back SPI, and it\u0027s always the SPI *from DB metadata*.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":769,"context_line":"                include_own\u003dinclude_own)"},{"line_number":770,"context_line":"        else:"},{"line_number":771,"context_line":"            requested_policy_index \u003d self.get_and_validate_policy_index(req)"},{"line_number":772,"context_line":"            resp_headers \u003d gen_resp_headers(info, is_deleted\u003dis_deleted)"},{"line_number":773,"context_line":"            if is_deleted:"},{"line_number":774,"context_line":"                return HTTPNotFound(request\u003dreq, headers\u003dresp_headers)"},{"line_number":775,"context_line":"            resp_headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"}],"source_content_type":"text/x-python","patch_set":6,"id":"c86b3a32_a7cc80c5","line":772,"range":{"start_line":772,"start_character":0,"end_line":772,"end_character":43},"updated":"2021-08-10 18:16:37.000000000","message":"We do something similar with objects...","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":777,"context_line":"                requested_policy_index if requested_policy_index is not None"},{"line_number":778,"context_line":"                else info[\u0027storage_policy_index\u0027])"},{"line_number":779,"context_line":"            resp_headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \\"},{"line_number":780,"context_line":"                storage_policy_index"},{"line_number":781,"context_line":"            # Use the retired db while container is in process of sharding,"},{"line_number":782,"context_line":"            # otherwise use current db"},{"line_number":783,"context_line":"            src_broker \u003d broker.get_brokers()[0]"}],"source_content_type":"text/x-python","patch_set":6,"id":"785a5dad_c3ba6913","line":780,"updated":"2021-08-10 18:16:37.000000000","message":"... but then we\u0027re sure to update it to match what was actually queried if they differ. Maybe this would be better/more clear as\n\n if requested_policy_index is not None:\n     resp_headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d requested_policy_index\n # else, the container\u0027s policy was set in gen_resp_headers\n\n?\n\nMight even be worth using a different header; maybe\n\n X-Backend-Queried-Storage-Policy-Index\n\nor even\n\n X-Backend-Record-Storage-Policy-Index\n\nfor some degree of symmetry with X-Backend-Record-Type.\n\nThough then I\u0027d be torn about whether we ought to return it all the time (to be explicit) or only if it differs from X-Backend-Storage-Policy-Index (to push us a little more to keep the upgrade path in mind).","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0490cb434fedbfce1594f00a53e5dcc9987f85bf","unresolved":true,"context_lines":[{"line_number":777,"context_line":"                requested_policy_index if requested_policy_index is not None"},{"line_number":778,"context_line":"                else info[\u0027storage_policy_index\u0027])"},{"line_number":779,"context_line":"            resp_headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \\"},{"line_number":780,"context_line":"                storage_policy_index"},{"line_number":781,"context_line":"            # Use the retired db while container is in process of sharding,"},{"line_number":782,"context_line":"            # otherwise use current db"},{"line_number":783,"context_line":"            src_broker \u003d broker.get_brokers()[0]"}],"source_content_type":"text/x-python","patch_set":6,"id":"0ba0c4bd_a0acd79a","line":780,"in_reply_to":"023e878e_0b33a215","updated":"2021-08-11 07:14:21.000000000","message":"@tim: Maybe something like https://review.opendev.org/c/openstack/swift/+/804208","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5a1fee27d4d6fc006dea754e3266df76bff48081","unresolved":true,"context_lines":[{"line_number":777,"context_line":"                requested_policy_index if requested_policy_index is not None"},{"line_number":778,"context_line":"                else info[\u0027storage_policy_index\u0027])"},{"line_number":779,"context_line":"            resp_headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \\"},{"line_number":780,"context_line":"                storage_policy_index"},{"line_number":781,"context_line":"            # Use the retired db while container is in process of sharding,"},{"line_number":782,"context_line":"            # otherwise use current db"},{"line_number":783,"context_line":"            src_broker \u003d broker.get_brokers()[0]"}],"source_content_type":"text/x-python","patch_set":6,"id":"023e878e_0b33a215","line":780,"in_reply_to":"785a5dad_c3ba6913","updated":"2021-08-11 06:16:24.000000000","message":"As we talked about this morning. Having a different one when it isn\u0027t the same is warming on me. Esp if we ever want to use this feature for anything else. You said you managed to poison your cache while playing around because of a different spi returning from a shard.\n\nWe also take for granted that we use the resp from the root request and use its x-backend-storage-policy-index knowing it\u0027s what it must have in it\u0027s DB. And then we go write over the shards response, so now we can\u0027t trust shard return values. So it might be nice to know that the x-backend-storage-policy-index is the containers serialized version, but the object that are returned belong to x-backend-requested-storage-policy-index (which I like).\n\nI kinda like the requested-* but record-* also makes sense as it does have prior usage.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":783,"context_line":"            src_broker \u003d broker.get_brokers()[0]"},{"line_number":784,"context_line":"            container_list \u003d src_broker.list_objects_iter("},{"line_number":785,"context_line":"                limit, marker, end_marker, prefix, delimiter, path,"},{"line_number":786,"context_line":"                storage_policy_index\u003dstorage_policy_index,"},{"line_number":787,"context_line":"                reverse\u003dreverse, allow_reserved\u003dreq.allow_reserved_names)"},{"line_number":788,"context_line":"        return self.create_listing(req, out_content_type, info, resp_headers,"},{"line_number":789,"context_line":"                                   broker.metadata, container_list, container)"}],"source_content_type":"text/x-python","patch_set":6,"id":"cca0ed21_3036f9fa","line":786,"updated":"2021-08-09 20:30:53.000000000","message":"this is all obvious; kudos","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":783,"context_line":"            src_broker \u003d broker.get_brokers()[0]"},{"line_number":784,"context_line":"            container_list \u003d src_broker.list_objects_iter("},{"line_number":785,"context_line":"                limit, marker, end_marker, prefix, delimiter, path,"},{"line_number":786,"context_line":"                storage_policy_index\u003dstorage_policy_index,"},{"line_number":787,"context_line":"                reverse\u003dreverse, allow_reserved\u003dreq.allow_reserved_names)"},{"line_number":788,"context_line":"        return self.create_listing(req, out_content_type, info, resp_headers,"},{"line_number":789,"context_line":"                                   broker.metadata, container_list, container)"}],"source_content_type":"text/x-python","patch_set":6,"id":"becfd1e8_c0532e50","line":786,"in_reply_to":"cca0ed21_3036f9fa","updated":"2021-08-10 18:16:37.000000000","message":"I\u0027m getting less and less convinced of this -- at any rate, I needed to do more digging than I\u0027d originally expected ;-)","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eb5c486f76d21eac70c5a3c04e5975e55c21bef5","unresolved":true,"context_lines":[{"line_number":777,"context_line":"                requested_policy_index if requested_policy_index is not None"},{"line_number":778,"context_line":"                else info[\u0027storage_policy_index\u0027])"},{"line_number":779,"context_line":"            resp_headers[\u0027X-Backend-Record-Storage-Policy-Index\u0027] \u003d \\"},{"line_number":780,"context_line":"                storage_policy_index"},{"line_number":781,"context_line":"            # Use the retired db while container is in process of sharding,"},{"line_number":782,"context_line":"            # otherwise use current db"},{"line_number":783,"context_line":"            src_broker \u003d broker.get_brokers()[0]"}],"source_content_type":"text/x-python","patch_set":7,"id":"7649b4b0_928953f7","line":780,"updated":"2021-08-12 12:59:08.000000000","message":"+1 on balance I think I prefer this header being returned unconditionally for object records. We could then infer upgraded server from the presence of the header, as opposed to \u0027maybe not upgraded, or maybe same policy as requested\u0027.","commit_id":"5f519176ed2bad028cb272203886ae2c758cef93"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":373,"context_line":"                headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027object\u0027"},{"line_number":374,"context_line":"            if config_true_value(req.headers.get(\u0027x-newest\u0027, False)):"},{"line_number":375,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":376,"context_line":"            if resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027):"},{"line_number":377,"context_line":"                headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \\"},{"line_number":378,"context_line":"                    resp.headers[\u0027X-Backend-Storage-Policy-Index\u0027]"},{"line_number":379,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"1a38e7c3_7a816075","line":376,"range":{"start_line":376,"start_character":15,"end_line":376,"end_character":65},"updated":"2021-08-05 11:57:11.000000000","message":"this might be better written:\n\n  if \u0027X-Backend-Storage-Policy-Index\u0027 in resp.headers\n\nit\u0027s potentially confusing to think that \n\n  \u0027X-Backend-Storage-Policy-Index\u0027: 0\n\nwould not satisfy the condition \n\n  if resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)\n\nuntil you realise that headers are always stringified so\n\n   \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027\n\nwould satisfy.","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f30ddef51081b0296f14c15a8f31f4674dfc23d9","unresolved":true,"context_lines":[{"line_number":375,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":376,"context_line":"            if \u0027X-Backend-Storage-Policy-Index\u0027 in resp.headers:"},{"line_number":377,"context_line":"                headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \\"},{"line_number":378,"context_line":"                    resp.headers[\u0027X-Backend-Storage-Policy-Index\u0027]"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"            if prefix:"},{"line_number":381,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":4,"id":"eaed8dff_bb88544c","line":378,"updated":"2021-08-06 10:55:25.000000000","message":"if this method recurses (a shard redirects to subshards) then we\u0027ll lose the root policy index :( so I think we need to check that the request is not already specifying a policy index.","commit_id":"57242c1ab314791a1df7a2d7ff4e3ed411aef5b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c42c8cf20d7b8d5b0262a44451bfb699ed856d7d","unresolved":true,"context_lines":[{"line_number":141,"context_line":"        if (info and is_success(info[\u0027status\u0027]) and"},{"line_number":142,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":143,"context_line":"            # container is sharded so we may have the shard ranges cached"},{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"}],"source_content_type":"text/x-python","patch_set":6,"id":"86b91829_24e1b5c4","line":144,"updated":"2021-08-09 21:59:55.000000000","message":"Right, this will include X-Backend-Storage-Policy-Index.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0cc4075b0bcdcf2994305e6563e529c3e43989f4","unresolved":true,"context_lines":[{"line_number":141,"context_line":"        if (info and is_success(info[\u0027status\u0027]) and"},{"line_number":142,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":143,"context_line":"            # container is sharded so we may have the shard ranges cached"},{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"}],"source_content_type":"text/x-python","patch_set":6,"id":"ec8626c4_3481ba62","line":144,"updated":"2021-08-09 22:16:35.000000000","message":"in the case of a cached shard listing, we *construct* a resp object with the headers rebuilt from the cache info - which luckily apparently already included x-backend-storage-policy-index unmolested","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":141,"context_line":"        if (info and is_success(info[\u0027status\u0027]) and"},{"line_number":142,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":143,"context_line":"            # container is sharded so we may have the shard ranges cached"},{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"}],"source_content_type":"text/x-python","patch_set":6,"id":"67b16ff6_01ddc8bf","line":144,"in_reply_to":"ec8626c4_3481ba62","updated":"2021-08-10 18:16:37.000000000","message":"🤔 now I need to think more about rebuilt-from-cache responses for *shards*...","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":286,"context_line":"            resp.headers[\u0027X-Backend-Recheck-Container-Existence\u0027] \u003d str("},{"line_number":287,"context_line":"                self.app.recheck_container_existence)"},{"line_number":288,"context_line":"            set_info_cache(self.app, req.environ, self.account_name,"},{"line_number":289,"context_line":"                           self.container_name, resp)"},{"line_number":290,"context_line":"        if \u0027swift.authorize\u0027 in req.environ:"},{"line_number":291,"context_line":"            req.acl \u003d resp.headers.get(\u0027x-container-read\u0027)"},{"line_number":292,"context_line":"            aresp \u003d req.environ[\u0027swift.authorize\u0027](req)"}],"source_content_type":"text/x-python","patch_set":6,"id":"59bbfe2c_22b5573d","line":289,"range":{"start_line":289,"start_character":48,"end_line":289,"end_character":52},"updated":"2021-08-10 18:16:37.000000000","message":"Note that the requested SPI can get into cache here...\n\nI *think* I\u0027m OK with us having funky SPIs for shards in cache, though?\n\nBut it makes me worry a little if something like 1space or a bucket inventory feature tries to front-run the reconciler by requesting listings for other policies -- it\u0027ll poison the cache for the root and lead to a bunch of 404s. Using something like https://github.com/tipabu/no_op_gatekeeper to let me get around the gatekeeper, I see behavior like https://paste.opendev.org/show/807990/","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5a1fee27d4d6fc006dea754e3266df76bff48081","unresolved":true,"context_lines":[{"line_number":286,"context_line":"            resp.headers[\u0027X-Backend-Recheck-Container-Existence\u0027] \u003d str("},{"line_number":287,"context_line":"                self.app.recheck_container_existence)"},{"line_number":288,"context_line":"            set_info_cache(self.app, req.environ, self.account_name,"},{"line_number":289,"context_line":"                           self.container_name, resp)"},{"line_number":290,"context_line":"        if \u0027swift.authorize\u0027 in req.environ:"},{"line_number":291,"context_line":"            req.acl \u003d resp.headers.get(\u0027x-container-read\u0027)"},{"line_number":292,"context_line":"            aresp \u003d req.environ[\u0027swift.authorize\u0027](req)"}],"source_content_type":"text/x-python","patch_set":6,"id":"ca4b734a_43bedd7e","line":289,"range":{"start_line":289,"start_character":48,"end_line":289,"end_character":52},"in_reply_to":"59bbfe2c_22b5573d","updated":"2021-08-11 06:16:24.000000000","message":"ahh, cool, this is what you were talking about this morn. Yeah potential for cache poisoning isn\u0027t great.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0cc4075b0bcdcf2994305e6563e529c3e43989f4","unresolved":true,"context_lines":[{"line_number":366,"context_line":"            else:"},{"line_number":367,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(shard_range.end_marker)"},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"            headers \u003d {}"},{"line_number":370,"context_line":"            if ((shard_range.account, shard_range.container) in"},{"line_number":371,"context_line":"                    shard_listing_history):"},{"line_number":372,"context_line":"                # directed back to same container - force GET of objects"}],"source_content_type":"text/x-python","patch_set":6,"id":"0a4cbd2c_3f0a5074","line":369,"updated":"2021-08-09 22:16:35.000000000","message":"N.B. we build this dict fresh every time, and subrequests in _get_container_listing do a\n\n    subreq \u003d make_pre_auth_req(headers\u003dreq.headers)\n    subreq.headers.update(headers)\n\n... so even if we \"don\u0027t set x-backend-spi in headers\" we still forward the \"x-backend-spi\" from req.headers","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7ef2eea745880a2cec6be986767f63e9aa00bd47","unresolved":true,"context_lines":[{"line_number":366,"context_line":"            else:"},{"line_number":367,"context_line":"                params[\u0027end_marker\u0027] \u003d str_to_wsgi(shard_range.end_marker)"},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"            headers \u003d {}"},{"line_number":370,"context_line":"            if ((shard_range.account, shard_range.container) in"},{"line_number":371,"context_line":"                    shard_listing_history):"},{"line_number":372,"context_line":"                # directed back to same container - force GET of objects"}],"source_content_type":"text/x-python","patch_set":6,"id":"4582e3be_793d3f45","line":369,"in_reply_to":"0a4cbd2c_3f0a5074","updated":"2021-08-10 02:25:26.000000000","message":"Yeah, the point was the add it if it wasn\u0027t already there. See my comments to line 378 below.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":375,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":376,"context_line":"            policy_key \u003d \u0027X-Backend-Storage-Policy-Index\u0027"},{"line_number":377,"context_line":"            if policy_key in resp.headers and policy_key not in req.headers:"},{"line_number":378,"context_line":"                headers[policy_key] \u003d resp.headers[policy_key]"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"            if prefix:"},{"line_number":381,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":6,"id":"de29e787_2e5ba57c","line":378,"updated":"2021-08-09 20:30:53.000000000","message":"and not in req.headers?  it\u0027s like a propogation from the... first? ... response?  I expected to see something coming out of container_info getting forced into all subrequests","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7ef2eea745880a2cec6be986767f63e9aa00bd47","unresolved":true,"context_lines":[{"line_number":375,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":376,"context_line":"            policy_key \u003d \u0027X-Backend-Storage-Policy-Index\u0027"},{"line_number":377,"context_line":"            if policy_key in resp.headers and policy_key not in req.headers:"},{"line_number":378,"context_line":"                headers[policy_key] \u003d resp.headers[policy_key]"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"            if prefix:"},{"line_number":381,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":6,"id":"778d8232_85dbd258","line":378,"in_reply_to":"5a0a2463_605a7b00","updated":"2021-08-10 02:25:26.000000000","message":"Becasue these queries/sub reqs to the shards go via app.handle_request. This code can be somewhat recursive. So if we make a call to a root container, that then goes to a shard that also happens to be sharding, then it\u0027ll then make calls to the new shards to get the data.\n\nIn this case we have to be careful of the SPI to use. The only adding the policy index if isn\u0027t already in the req, ensures that we are only using the root spi, which is what we want.\n\nThe resp given here is the response from the root container, and so how we\u0027re getting the \"correct\" SPI.\n\nThe whole placing it in headers and not in the req directly is because that\u0027s what we are doing with other headers we want to override, then they get .update(headers) as clay mentions elsewhere. It could be added to the req, but was just matching the current way we did things.. but turns out makes it easy to test what we added/overwrote.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"950cba2212faf0bb94d16d35cc9e064cc822ae38","unresolved":true,"context_lines":[{"line_number":375,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":376,"context_line":"            policy_key \u003d \u0027X-Backend-Storage-Policy-Index\u0027"},{"line_number":377,"context_line":"            if policy_key in resp.headers and policy_key not in req.headers:"},{"line_number":378,"context_line":"                headers[policy_key] \u003d resp.headers[policy_key]"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"            if prefix:"},{"line_number":381,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":6,"id":"37226ff8_1de8c6d3","line":378,"in_reply_to":"778d8232_85dbd258","updated":"2021-08-10 09:07:30.000000000","message":"the confusion here might have been avoided if the var headers was named extra_headers: the headers dict is just gathering extra headers to add to the existing request headers when the subrequest is constructed.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c42c8cf20d7b8d5b0262a44451bfb699ed856d7d","unresolved":true,"context_lines":[{"line_number":375,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":376,"context_line":"            policy_key \u003d \u0027X-Backend-Storage-Policy-Index\u0027"},{"line_number":377,"context_line":"            if policy_key in resp.headers and policy_key not in req.headers:"},{"line_number":378,"context_line":"                headers[policy_key] \u003d resp.headers[policy_key]"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"            if prefix:"},{"line_number":381,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":6,"id":"55f8ae86_4b017ba5","line":378,"in_reply_to":"de29e787_2e5ba57c","updated":"2021-08-09 21:59:55.000000000","message":"Agreed -- this does feel a bit weird. If policy_key *is* in req.headers, why isn\u0027t *that* pulled across?\n\n\u003e I expected to see something coming out of container_info getting forced into all subrequests\n\nI think we get that on the listing-shard-ranges-from-cache path, if indirectly. If we didn\u0027t get shard ranges from cache, it seems reasonable to take it directly from the response headers rather than falling back to info cache.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0cc4075b0bcdcf2994305e6563e529c3e43989f4","unresolved":true,"context_lines":[{"line_number":375,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":376,"context_line":"            policy_key \u003d \u0027X-Backend-Storage-Policy-Index\u0027"},{"line_number":377,"context_line":"            if policy_key in resp.headers and policy_key not in req.headers:"},{"line_number":378,"context_line":"                headers[policy_key] \u003d resp.headers[policy_key]"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"            if prefix:"},{"line_number":381,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5a0a2463_605a7b00","line":378,"in_reply_to":"de29e787_2e5ba57c","updated":"2021-08-09 22:16:35.000000000","message":"I was confused, resp.headers is the root resp (or at least the shard range response, possibly from a shard returning sub-shards)\n\n\"if policy_key in resp.headers\" seems mostly defensive against upgrades, the meat is \"and policy_key not in req\" where req.headers is NOT the shard requests, but \"the request that responded with shard ranges\".  So at first req is the root req and it doesn\u0027t have the backend header unless a client set it behind gateway.  Then our sub-request machinary will copy \u0027x-backend-spi\u0027 from this headers dict we\u0027re building OVER what ever may be set on req\n\nthis is not obvious to me (and there may be a subtle bug during upgrade with confused sub-shards) - I think we should at consider if we could \"just\" plumb a root_spi kwarg from container_info","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0cc4075b0bcdcf2994305e6563e529c3e43989f4","unresolved":true,"context_lines":[{"line_number":392,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"},{"line_number":393,"context_line":"                i, shard_range, shard_range.name, headers)"},{"line_number":394,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":395,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":396,"context_line":"                headers\u003dheaders, params\u003dparams)"},{"line_number":397,"context_line":"            all_resp_status.append(shard_resp.status_int)"},{"line_number":398,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"ed3edf19_09706225","line":395,"updated":"2021-08-09 22:16:35.000000000","message":"so in this loop \"resp\" is the \"root_resp\" and \"shard_resp\" is the response from the shards\n\nwe do re-use the req, but it\u0027s \"mostly\" read-only (we pass in explicitly modified headers/params per shard subrequest)","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9effafb970bfac39d1c0a909a2b7b4e32b4e27b6","unresolved":true,"context_lines":[{"line_number":392,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"},{"line_number":393,"context_line":"                i, shard_range, shard_range.name, headers)"},{"line_number":394,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":395,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":396,"context_line":"                headers\u003dheaders, params\u003dparams)"},{"line_number":397,"context_line":"            all_resp_status.append(shard_resp.status_int)"},{"line_number":398,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"d1d84a64_2c676124","line":395,"in_reply_to":"147b77c0_ac4029d3","updated":"2021-08-10 18:16:37.000000000","message":"Note that resp *can be* from a shard, though -- if upon querying the shard we discover that it has split up.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"950cba2212faf0bb94d16d35cc9e064cc822ae38","unresolved":true,"context_lines":[{"line_number":392,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"},{"line_number":393,"context_line":"                i, shard_range, shard_range.name, headers)"},{"line_number":394,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":395,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":396,"context_line":"                headers\u003dheaders, params\u003dparams)"},{"line_number":397,"context_line":"            all_resp_status.append(shard_resp.status_int)"},{"line_number":398,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"147b77c0_ac4029d3","line":395,"in_reply_to":"ed3edf19_09706225","updated":"2021-08-10 09:07:30.000000000","message":"yes, and IIRC this is quite deliberate i.e. we do not \u0027pollute\u0027 the original request to root container with headers intended for the shards.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7e163fb2c72c1752be71727c07e500938034cdf6","unresolved":true,"context_lines":[{"line_number":386,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":387,"context_line":"            policy_key \u003d \u0027X-Backend-Storage-Policy-Index\u0027"},{"line_number":388,"context_line":"            if policy_key in resp.headers and policy_key not in req.headers:"},{"line_number":389,"context_line":"                headers[policy_key] \u003d resp.headers[policy_key]"},{"line_number":390,"context_line":""},{"line_number":391,"context_line":"            if prefix:"},{"line_number":392,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":8,"id":"ebfed784_ba4889e7","line":389,"updated":"2021-08-13 21:04:35.000000000","message":"Do we still need to set this here? Or is setting it in the base req @L331 enough?","commit_id":"91e1f3047bd616db61a75e8947bc26b7365a3552"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7bb5b680864de67c123e92b6ab1cd5cd68891447","unresolved":true,"context_lines":[{"line_number":386,"context_line":"                headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":387,"context_line":"            policy_key \u003d \u0027X-Backend-Storage-Policy-Index\u0027"},{"line_number":388,"context_line":"            if policy_key in resp.headers and policy_key not in req.headers:"},{"line_number":389,"context_line":"                headers[policy_key] \u003d resp.headers[policy_key]"},{"line_number":390,"context_line":""},{"line_number":391,"context_line":"            if prefix:"},{"line_number":392,"context_line":"                if prefix \u003e shard_range:"}],"source_content_type":"text/x-python","patch_set":8,"id":"f5dde4b0_7b02c0c4","line":389,"in_reply_to":"ebfed784_ba4889e7","updated":"2021-08-16 07:07:04.000000000","message":"no, a squash has left this here, should be removed. will fix.","commit_id":"91e1f3047bd616db61a75e8947bc26b7365a3552"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7e163fb2c72c1752be71727c07e500938034cdf6","unresolved":true,"context_lines":[{"line_number":416,"context_line":"                    \u0027Aborting listing from shards due to bad response: %r\u0027"},{"line_number":417,"context_line":"                    % all_resp_status)"},{"line_number":418,"context_line":"                return HTTPServiceUnavailable(request\u003dreq)"},{"line_number":419,"context_line":""},{"line_number":420,"context_line":"            self.app.logger.debug("},{"line_number":421,"context_line":"                \u0027Found %d objects in shard (state\u003d%s), total \u003d %d\u0027,"},{"line_number":422,"context_line":"                len(objs), sharding_state, len(objs) + len(objects))"}],"source_content_type":"text/x-python","patch_set":8,"id":"7d53321b_08930d87","line":419,"updated":"2021-08-13 21:04:35.000000000","message":"Should we have a check like\n\n            shard_policy \u003d shard_resp.headers.get(\n                \u0027X-Backend-Record-Storage-Policy-Index\u0027,\n                shard_resp.headers[policy_key]\n            )\n            if shard_policy !\u003d req.headers[policy_key]:\n                self.app.logger.error(\n                    \u0027Aborting listing from shards due to \u0027\n                    \u0027bad shard policy index: %s (expected %s)\u0027,\n                    shard_policy, req.headers[policy_key])\n                return HTTPServiceUnavailable(request\u003dreq)\n\n? Maybe even check for it in _get_container_listing...","commit_id":"91e1f3047bd616db61a75e8947bc26b7365a3552"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":2247,"context_line":"        # of that index out."},{"line_number":2248,"context_line":"        self.assert_container_listing(all_obj_names)"},{"line_number":2249,"context_line":"        # although a head request is getting object count for the shard spi"},{"line_number":2250,"context_line":"        self.assert_container_object_count(0)"},{"line_number":2251,"context_line":""},{"line_number":2252,"context_line":"    def test_object_update_redirection(self):"},{"line_number":2253,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"}],"source_content_type":"text/x-python","patch_set":3,"id":"47e9d7b4_5b5fe0c6","line":2250,"updated":"2021-08-05 11:57:11.000000000","message":"nice! might be fun to switch the policies in the other direction too just to be sure there\u0027s nothing amiss with handling policy index 0","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":2220,"context_line":"                                                          policy_idx,"},{"line_number":2221,"context_line":"                                                          new_policy_idx):"},{"line_number":2222,"context_line":"        # create sharded container"},{"line_number":2223,"context_line":"        client.delete_container(self.url, self.token, self.container_name)"},{"line_number":2224,"context_line":"        self.brain.put_container(policy_index\u003dint(policy_idx))"},{"line_number":2225,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"},{"line_number":2226,"context_line":"        self.put_objects(all_obj_names)"}],"source_content_type":"text/x-python","patch_set":6,"id":"f08e8141_3e7080d2","line":2223,"updated":"2021-08-09 20:30:53.000000000","message":"i guess in case it got created with the wrong policy index in setup or something?","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"950cba2212faf0bb94d16d35cc9e064cc822ae38","unresolved":true,"context_lines":[{"line_number":2220,"context_line":"                                                          policy_idx,"},{"line_number":2221,"context_line":"                                                          new_policy_idx):"},{"line_number":2222,"context_line":"        # create sharded container"},{"line_number":2223,"context_line":"        client.delete_container(self.url, self.token, self.container_name)"},{"line_number":2224,"context_line":"        self.brain.put_container(policy_index\u003dint(policy_idx))"},{"line_number":2225,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"},{"line_number":2226,"context_line":"        self.put_objects(all_obj_names)"}],"source_content_type":"text/x-python","patch_set":6,"id":"d4449c58_64260692","line":2223,"in_reply_to":"f08e8141_3e7080d2","updated":"2021-08-10 09:07:30.000000000","message":"yes. setup creates a container with the default policy, but the test may want the container to start life with a non-default policy","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":2246,"context_line":"        # The shard is still on the old policy index, but the root spi"},{"line_number":2247,"context_line":"        # is passed to shard container server and is used to pull objects"},{"line_number":2248,"context_line":"        # of that index out."},{"line_number":2249,"context_line":"        self.assert_container_listing(all_obj_names)"},{"line_number":2250,"context_line":"        # although a head request is getting object count for the shard spi"},{"line_number":2251,"context_line":"        self.assert_container_object_count(0)"},{"line_number":2252,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"0ca2069b_985ffef3","line":2249,"updated":"2021-08-09 20:30:53.000000000","message":"this is cool!","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":2248,"context_line":"        # of that index out."},{"line_number":2249,"context_line":"        self.assert_container_listing(all_obj_names)"},{"line_number":2250,"context_line":"        # although a head request is getting object count for the shard spi"},{"line_number":2251,"context_line":"        self.assert_container_object_count(0)"},{"line_number":2252,"context_line":""},{"line_number":2253,"context_line":"    @unittest.skipIf(len(ENABLED_POLICIES) \u003c 2, \"Need more than one policy\")"},{"line_number":2254,"context_line":"    def test_sharded_can_get_objects_different_policy(self):"}],"source_content_type":"text/x-python","patch_set":6,"id":"b900659b_4f13e279","line":2251,"updated":"2021-08-09 20:30:53.000000000","message":"this is a bummer","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7ef2eea745880a2cec6be986767f63e9aa00bd47","unresolved":true,"context_lines":[{"line_number":2248,"context_line":"        # of that index out."},{"line_number":2249,"context_line":"        self.assert_container_listing(all_obj_names)"},{"line_number":2250,"context_line":"        # although a head request is getting object count for the shard spi"},{"line_number":2251,"context_line":"        self.assert_container_object_count(0)"},{"line_number":2252,"context_line":""},{"line_number":2253,"context_line":"    @unittest.skipIf(len(ENABLED_POLICIES) \u003c 2, \"Need more than one policy\")"},{"line_number":2254,"context_line":"    def test_sharded_can_get_objects_different_policy(self):"}],"source_content_type":"text/x-python","patch_set":6,"id":"495c21ee_871251e4","line":2251,"in_reply_to":"51f4127a_4e16c537","updated":"2021-08-10 02:25:26.000000000","message":"Yup exactly.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c42c8cf20d7b8d5b0262a44451bfb699ed856d7d","unresolved":true,"context_lines":[{"line_number":2248,"context_line":"        # of that index out."},{"line_number":2249,"context_line":"        self.assert_container_listing(all_obj_names)"},{"line_number":2250,"context_line":"        # although a head request is getting object count for the shard spi"},{"line_number":2251,"context_line":"        self.assert_container_object_count(0)"},{"line_number":2252,"context_line":""},{"line_number":2253,"context_line":"    @unittest.skipIf(len(ENABLED_POLICIES) \u003c 2, \"Need more than one policy\")"},{"line_number":2254,"context_line":"    def test_sharded_can_get_objects_different_policy(self):"}],"source_content_type":"text/x-python","patch_set":6,"id":"51f4127a_4e16c537","line":2251,"in_reply_to":"b900659b_4f13e279","updated":"2021-08-09 21:59:55.000000000","message":"Am I right to think that https://review.opendev.org/c/openstack/swift/+/800748 would allow the sharder to eventually square it?\n\nI think I\u0027m OK with stats results lagging actual listings.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"}],"test/unit/container/test_server.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":4289,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":4290,"context_line":"        result \u003d [o for o in resp.body.split(b\u0027\\n\u0027) if o]"},{"line_number":4291,"context_line":"        self.assertEqual(len(result), 11)"},{"line_number":4292,"context_line":"        self.assertEqual(sorted(result), sorted(expected_pol_def_objs))"},{"line_number":4293,"context_line":""},{"line_number":4294,"context_line":"        # If we specify the policy 0 idx we should get the same"},{"line_number":4295,"context_line":"        req \u003d Request.blank("}],"source_content_type":"text/x-python","patch_set":3,"id":"ea23825d_ee25ca35","line":4292,"updated":"2021-08-05 11:57:11.000000000","message":"we should assert that the response headers now have \u0027X-Backend-Storage-Policy-Index\u0027","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":4319,"context_line":"        self.assertEqual(resp.status_int, 204)"},{"line_number":4320,"context_line":"        result \u003d [o for o in resp.body.split(b\u0027\\n\u0027) if o]"},{"line_number":4321,"context_line":"        self.assertEqual(len(result), 0)"},{"line_number":4322,"context_line":"        self.assertFalse(result)"},{"line_number":4323,"context_line":""},{"line_number":4324,"context_line":"    def test_GET_limit(self):"},{"line_number":4325,"context_line":"        # make a container"}],"source_content_type":"text/x-python","patch_set":3,"id":"92a6e2bb_ee0136db","line":4322,"updated":"2021-08-05 11:57:11.000000000","message":"nice!\n\n...and if we validate policy then test a non-existent policy","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":4320,"context_line":"        self.assertEqual(len(result), 10)"},{"line_number":4321,"context_line":"        self.assertEqual(sorted(result), sorted(expected_pol_1_objs))"},{"line_number":4322,"context_line":"        self.assertIn(\u0027X-Backend-Storage-Policy-Index\u0027, resp.headers)"},{"line_number":4323,"context_line":"        self.assertEqual(\u00271\u0027, resp.headers[\u0027X-Backend-Storage-Policy-Index\u0027])"},{"line_number":4324,"context_line":""},{"line_number":4325,"context_line":"        # And an index that the broker doesn\u0027t have any objects for"},{"line_number":4326,"context_line":"        req \u003d Request.blank("}],"source_content_type":"text/x-python","patch_set":6,"id":"9de5815e_83dc7896","line":4323,"updated":"2021-08-09 20:30:53.000000000","message":"ok, cool","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":4339,"context_line":"            \u0027/sda1/p/a/c\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":4340,"context_line":"        req.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d 3"},{"line_number":4341,"context_line":"        resp \u003d req.get_response(self.controller)"},{"line_number":4342,"context_line":"        self.assertEqual(resp.status_int, 400)"},{"line_number":4343,"context_line":""},{"line_number":4344,"context_line":"    def test_GET_limit(self):"},{"line_number":4345,"context_line":"        # make a container"}],"source_content_type":"text/x-python","patch_set":6,"id":"774dca27_4987b865","line":4342,"updated":"2021-08-09 20:30:53.000000000","message":"bad request!  nice","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"}],"test/unit/proxy/controllers/test_container.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3609c69cfdbaa62322195225ce79676210baa5aa","unresolved":true,"context_lines":[{"line_number":1949,"context_line":"            controller \u003d controller_cls(self.app, **d)"},{"line_number":1950,"context_line":"            controller._get_from_shards(req, resp)"},{"line_number":1951,"context_line":""},{"line_number":1952,"context_line":"        self.assertEqual("},{"line_number":1953,"context_line":"            shard_listing_headers[\u0027X-Backend-Storage-Policy-Index\u0027], \u00270\u0027)"},{"line_number":1954,"context_line":""},{"line_number":1955,"context_line":"        resp.headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d \u00271\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"5095b0de_61f3ab19","line":1952,"updated":"2021-08-05 11:57:11.000000000","message":"nit: precede this with\n\n  self.assertIn(\u0027X-Backend-Storage-Policy-Index\u0027, shard_listing_headers)\n\nto get a cleaner fail message","commit_id":"b54844b0bb1beca29b85bc5c666df018b49198cb"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":608,"context_line":"            (\u0027a/c\u0027, {\u0027X-Backend-Record-Type\u0027: \u0027auto\u0027},"},{"line_number":609,"context_line":"             dict(states\u003d\u0027listing\u0027)),  # 404"},{"line_number":610,"context_line":"            (\u0027a/c\u0027, {\u0027X-Backend-Record-Type\u0027: \u0027auto\u0027},"},{"line_number":611,"context_line":"             dict(states\u003d\u0027listing\u0027)),  # 200"},{"line_number":612,"context_line":"            (wsgi_quote(str_to_wsgi(shard_ranges[0].name)),"},{"line_number":613,"context_line":"             {\u0027X-Backend-Record-Type\u0027: \u0027auto\u0027,"},{"line_number":614,"context_line":"              \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027},"}],"source_content_type":"text/x-python","patch_set":6,"id":"c1ed35db_8b47fbea","line":611,"updated":"2021-08-09 20:30:53.000000000","message":"so, maybe this response included a x-backend-storage-policy-index when it returned shards?  what about when they\u0027re cached?","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"950cba2212faf0bb94d16d35cc9e064cc822ae38","unresolved":true,"context_lines":[{"line_number":608,"context_line":"            (\u0027a/c\u0027, {\u0027X-Backend-Record-Type\u0027: \u0027auto\u0027},"},{"line_number":609,"context_line":"             dict(states\u003d\u0027listing\u0027)),  # 404"},{"line_number":610,"context_line":"            (\u0027a/c\u0027, {\u0027X-Backend-Record-Type\u0027: \u0027auto\u0027},"},{"line_number":611,"context_line":"             dict(states\u003d\u0027listing\u0027)),  # 200"},{"line_number":612,"context_line":"            (wsgi_quote(str_to_wsgi(shard_ranges[0].name)),"},{"line_number":613,"context_line":"             {\u0027X-Backend-Record-Type\u0027: \u0027auto\u0027,"},{"line_number":614,"context_line":"              \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027},"}],"source_content_type":"text/x-python","patch_set":6,"id":"7538425e_19fa3bd7","line":611,"in_reply_to":"c1ed35db_8b47fbea","updated":"2021-08-10 09:07:30.000000000","message":"fortunately we took the trouble to ensure that responses from cache match responses form the backend :)\n\nhttps://github.com/openstack/swift/blob/077ba77ea6f735c98709b60e1fc9aa9f0f90676a/swift/proxy/controllers/base.py#L209","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":1793,"context_line":"            sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):] + sr_objs[2])"},{"line_number":1794,"context_line":"        resp \u003d self._check_GET_shard_listing("},{"line_number":1795,"context_line":"            mock_responses, expected_objects, expected_requests)"},{"line_number":1796,"context_line":"        # root object count will overridden by actual length of listing"},{"line_number":1797,"context_line":"        self.check_response(resp, root_resp_hdrs)"},{"line_number":1798,"context_line":""},{"line_number":1799,"context_line":"    @patch_policies(["}],"source_content_type":"text/x-python","patch_set":6,"id":"a389bc3b_ad32e791","line":1796,"updated":"2021-08-09 20:30:53.000000000","message":"what?","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"950cba2212faf0bb94d16d35cc9e064cc822ae38","unresolved":true,"context_lines":[{"line_number":1793,"context_line":"            sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):] + sr_objs[2])"},{"line_number":1794,"context_line":"        resp \u003d self._check_GET_shard_listing("},{"line_number":1795,"context_line":"            mock_responses, expected_objects, expected_requests)"},{"line_number":1796,"context_line":"        # root object count will overridden by actual length of listing"},{"line_number":1797,"context_line":"        self.check_response(resp, root_resp_hdrs)"},{"line_number":1798,"context_line":""},{"line_number":1799,"context_line":"    @patch_policies(["}],"source_content_type":"text/x-python","patch_set":6,"id":"729d4cef_770c7532","line":1796,"in_reply_to":"a389bc3b_ad32e791","updated":"2021-08-10 09:07:30.000000000","message":"this isn\u0027t new, the test diff just shows the change to end of this test when in fact the next test has been added\n\nwe\u0027ve always \u0027fixed\u0027 object count to match the length of the listing when we can:\n\nhttps://github.com/openstack/swift/blob/e40cf1ec561c78c8a02d40b581bf195939a6450a/swift/proxy/controllers/container.py#L431-L442","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29f7df80fca03401afc0c01b16eb731af880b999","unresolved":true,"context_lines":[{"line_number":1848,"context_line":"                          \u0027X-Container-Object-Count\u0027: num_all_objects,"},{"line_number":1849,"context_line":"                          \u0027X-Container-Bytes-Used\u0027: size_all_objects,"},{"line_number":1850,"context_line":"                          \u0027X-Container-Meta-Flavour\u0027: \u0027peach\u0027,"},{"line_number":1851,"context_line":"                          \u0027X-Backend-Storage-Policy-Index\u0027: 0}"},{"line_number":1852,"context_line":"        root_shard_resp_hdrs \u003d dict(root_resp_hdrs)"},{"line_number":1853,"context_line":"        root_shard_resp_hdrs[\u0027X-Backend-Record-Type\u0027] \u003d \u0027shard\u0027"},{"line_number":1854,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"4bc88e32_c0a8d18b","line":1851,"updated":"2021-08-09 20:30:53.000000000","message":"it looks like we always carry forward the policy index from the root resp","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7ef2eea745880a2cec6be986767f63e9aa00bd47","unresolved":true,"context_lines":[{"line_number":1848,"context_line":"                          \u0027X-Container-Object-Count\u0027: num_all_objects,"},{"line_number":1849,"context_line":"                          \u0027X-Container-Bytes-Used\u0027: size_all_objects,"},{"line_number":1850,"context_line":"                          \u0027X-Container-Meta-Flavour\u0027: \u0027peach\u0027,"},{"line_number":1851,"context_line":"                          \u0027X-Backend-Storage-Policy-Index\u0027: 0}"},{"line_number":1852,"context_line":"        root_shard_resp_hdrs \u003d dict(root_resp_hdrs)"},{"line_number":1853,"context_line":"        root_shard_resp_hdrs[\u0027X-Backend-Record-Type\u0027] \u003d \u0027shard\u0027"},{"line_number":1854,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"f240a9c4_e42fa5ae","line":1851,"in_reply_to":"4bc88e32_c0a8d18b","updated":"2021-08-10 02:25:26.000000000","message":"Yup thats what we want. otherwise a sharding shard on the wrong index could screw up our listing.","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"950cba2212faf0bb94d16d35cc9e064cc822ae38","unresolved":true,"context_lines":[{"line_number":1848,"context_line":"                          \u0027X-Container-Object-Count\u0027: num_all_objects,"},{"line_number":1849,"context_line":"                          \u0027X-Container-Bytes-Used\u0027: size_all_objects,"},{"line_number":1850,"context_line":"                          \u0027X-Container-Meta-Flavour\u0027: \u0027peach\u0027,"},{"line_number":1851,"context_line":"                          \u0027X-Backend-Storage-Policy-Index\u0027: 0}"},{"line_number":1852,"context_line":"        root_shard_resp_hdrs \u003d dict(root_resp_hdrs)"},{"line_number":1853,"context_line":"        root_shard_resp_hdrs[\u0027X-Backend-Record-Type\u0027] \u003d \u0027shard\u0027"},{"line_number":1854,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"7d14a675_dabd8002","line":1851,"in_reply_to":"f240a9c4_e42fa5ae","updated":"2021-08-10 09:07:30.000000000","message":"+1 what Matt said","commit_id":"63c6933f5d938bea47c365407ab7cea0b015b033"}]}