)]}'
{"etc/proxy-server.conf-sample":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":142,"context_line":"# the set is to be used for gathering object listings."},{"line_number":143,"context_line":"# Note that stale shard range info might result in incomplete object listings"},{"line_number":144,"context_line":"# so this value should be set less than recheck_updating_shard_ranges."},{"line_number":145,"context_line":"# recheck_listing_shard_ranges \u003d 600"},{"line_number":146,"context_line":"#"},{"line_number":147,"context_line":"# object_chunk_size \u003d 65536"},{"line_number":148,"context_line":"# client_chunk_size \u003d 65536"}],"source_content_type":"application/octet-stream","patch_set":5,"id":"96c0f785_482975f2","line":145,"updated":"2020-11-25 20:27:14.000000000","message":"or 60000!!! caching is awesome!","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":142,"context_line":"# the set is to be used for gathering object listings."},{"line_number":143,"context_line":"# Note that stale shard range info might result in incomplete object listings"},{"line_number":144,"context_line":"# so this value should be set less than recheck_updating_shard_ranges."},{"line_number":145,"context_line":"# recheck_listing_shard_ranges \u003d 600"},{"line_number":146,"context_line":"#"},{"line_number":147,"context_line":"# object_chunk_size \u003d 65536"},{"line_number":148,"context_line":"# client_chunk_size \u003d 65536"}],"source_content_type":"application/octet-stream","patch_set":5,"id":"0ddb7e9f_6e7e7ba8","line":145,"in_reply_to":"96c0f785_482975f2","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"swift/common/request_helpers.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":107,"context_line":"              appears in the request parameters"},{"line_number":108,"context_line":"    :raises HTTPBadRequest: if any parameter value is not a valid UTF-8 byte"},{"line_number":109,"context_line":"            sequence"},{"line_number":110,"context_line":"    \"\"\""},{"line_number":111,"context_line":"    params \u003d {}"},{"line_number":112,"context_line":"    for name in names:"},{"line_number":113,"context_line":"        value \u003d get_param(req, name)"}],"source_content_type":"text/x-python","patch_set":5,"id":"a0907dee_9a1f0944","line":110,"updated":"2020-11-25 20:27:14.000000000","message":"nice doc string","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":107,"context_line":"              appears in the request parameters"},{"line_number":108,"context_line":"    :raises HTTPBadRequest: if any parameter value is not a valid UTF-8 byte"},{"line_number":109,"context_line":"            sequence"},{"line_number":110,"context_line":"    \"\"\""},{"line_number":111,"context_line":"    params \u003d {}"},{"line_number":112,"context_line":"    for name in names:"},{"line_number":113,"context_line":"        value \u003d get_param(req, name)"}],"source_content_type":"text/x-python","patch_set":5,"id":"9b12f64f_72bc774b","line":110,"in_reply_to":"a0907dee_9a1f0944","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":133,"context_line":"                                   \u0027delimiter\u0027, \u0027path\u0027, \u0027format\u0027, \u0027reverse\u0027,"},{"line_number":134,"context_line":"                                   \u0027states\u0027, \u0027includes\u0027))"},{"line_number":135,"context_line":"    params[\u0027limit\u0027] \u003d constrain_req_limit(req, CONTAINER_LISTING_LIMIT)"},{"line_number":136,"context_line":"    return params"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"def _validate_internal_name(name, type_\u003d\u0027name\u0027):"}],"source_content_type":"text/x-python","patch_set":5,"id":"a1d8a4b4_0dd1d181","line":136,"updated":"2020-11-25 20:27:14.000000000","message":"these both seem reasonable","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":133,"context_line":"                                   \u0027delimiter\u0027, \u0027path\u0027, \u0027format\u0027, \u0027reverse\u0027,"},{"line_number":134,"context_line":"                                   \u0027states\u0027, \u0027includes\u0027))"},{"line_number":135,"context_line":"    params[\u0027limit\u0027] \u003d constrain_req_limit(req, CONTAINER_LISTING_LIMIT)"},{"line_number":136,"context_line":"    return params"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"def _validate_internal_name(name, type_\u003d\u0027name\u0027):"}],"source_content_type":"text/x-python","patch_set":5,"id":"4033ea9a_a9d9fe5c","line":136,"in_reply_to":"a1d8a4b4_0dd1d181","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":5441,"context_line":"    return None"},{"line_number":5442,"context_line":""},{"line_number":5443,"context_line":""},{"line_number":5444,"context_line":"def filter_shard_ranges(shard_ranges, includes, marker, end_marker):"},{"line_number":5445,"context_line":"    if includes:"},{"line_number":5446,"context_line":"        shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":5447,"context_line":"        return [shard_range] if shard_range else []"}],"source_content_type":"text/x-python","patch_set":5,"id":"04c45cf9_7908256c","line":5444,"range":{"start_line":5444,"start_character":48,"end_line":5444,"end_character":66},"updated":"2020-12-13 23:14:13.000000000","message":"Should we pass prefix here, too? Oh, or maybe we do that filtering elsewhere...","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":5441,"context_line":"    return None"},{"line_number":5442,"context_line":""},{"line_number":5443,"context_line":""},{"line_number":5444,"context_line":"def filter_shard_ranges(shard_ranges, includes, marker, end_marker):"},{"line_number":5445,"context_line":"    if includes:"},{"line_number":5446,"context_line":"        shard_range \u003d find_shard_range(includes, shard_ranges)"},{"line_number":5447,"context_line":"        return [shard_range] if shard_range else []"}],"source_content_type":"text/x-python","patch_set":5,"id":"88ad1355_d1df9ec1","line":5444,"range":{"start_line":5444,"start_character":48,"end_line":5444,"end_character":66},"in_reply_to":"04c45cf9_7908256c","updated":"2020-12-15 20:24:10.000000000","message":"prefix filtering is done in the proxy controller _get_from_shards when composing an object listing from shards, but isn\u0027t done for an (internal) shard range listing","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":5457,"context_line":"    if marker or end_marker:"},{"line_number":5458,"context_line":"        return list(filter(shard_range_filter, shard_ranges))"},{"line_number":5459,"context_line":""},{"line_number":5460,"context_line":"    return shard_ranges"},{"line_number":5461,"context_line":""},{"line_number":5462,"context_line":""},{"line_number":5463,"context_line":"def modify_priority(conf, logger):"}],"source_content_type":"text/x-python","patch_set":5,"id":"2378cb96_0301fb56","line":5460,"updated":"2020-11-25 20:27:14.000000000","message":"ok, find and filter are in utils... :+1:","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":5457,"context_line":"    if marker or end_marker:"},{"line_number":5458,"context_line":"        return list(filter(shard_range_filter, shard_ranges))"},{"line_number":5459,"context_line":""},{"line_number":5460,"context_line":"    return shard_ranges"},{"line_number":5461,"context_line":""},{"line_number":5462,"context_line":""},{"line_number":5463,"context_line":"def modify_priority(conf, logger):"}],"source_content_type":"text/x-python","patch_set":5,"id":"cc42e18f_1d72b2a3","line":5460,"in_reply_to":"2378cb96_0301fb56","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"swift/container/backend.py":[{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":28,"context_line":""},{"line_number":29,"context_line":"from swift.common.constraints import CONTAINER_LISTING_LIMIT"},{"line_number":30,"context_line":"from swift.common.exceptions import LockTimeout"},{"line_number":31,"context_line":"from swift.common.utils import Timestamp, encode_timestamps, \\"},{"line_number":32,"context_line":"    decode_timestamps, extract_swift_bytes, storage_directory, hash_path, \\"},{"line_number":33,"context_line":"    ShardRange, renamer, find_shard_range, MD5_OF_EMPTY_STRING, mkdirs, \\"},{"line_number":34,"context_line":"    get_db_files, parse_db_filename, make_db_file_path, split_path, \\"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_10102701","line":31,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: F401 \u0027swift.common.utils.find_shard_range\u0027 imported but unused","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":32,"context_line":"    decode_timestamps, extract_swift_bytes, storage_directory, hash_path, \\"},{"line_number":33,"context_line":"    ShardRange, renamer, MD5_OF_EMPTY_STRING, mkdirs, get_db_files, \\"},{"line_number":34,"context_line":"    parse_db_filename, make_db_file_path, split_path, RESERVED_BYTE, \\"},{"line_number":35,"context_line":"    filter_shard_ranges"},{"line_number":36,"context_line":"from swift.common.db import DatabaseBroker, utf8encode, BROKER_TIMEOUT, \\"},{"line_number":37,"context_line":"    zero_like, DatabaseAlreadyExists, SQLITE_ARG_LIMIT"},{"line_number":38,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"1e0a15a8_00f0dcf3","line":35,"updated":"2020-11-25 20:27:14.000000000","message":"at least filter_shard_ranges is used somewhere out side of the proxy!  👍","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":32,"context_line":"    decode_timestamps, extract_swift_bytes, storage_directory, hash_path, \\"},{"line_number":33,"context_line":"    ShardRange, renamer, MD5_OF_EMPTY_STRING, mkdirs, get_db_files, \\"},{"line_number":34,"context_line":"    parse_db_filename, make_db_file_path, split_path, RESERVED_BYTE, \\"},{"line_number":35,"context_line":"    filter_shard_ranges"},{"line_number":36,"context_line":"from swift.common.db import DatabaseBroker, utf8encode, BROKER_TIMEOUT, \\"},{"line_number":37,"context_line":"    zero_like, DatabaseAlreadyExists, SQLITE_ARG_LIMIT"},{"line_number":38,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"44f8714b_3771bece","line":35,"in_reply_to":"1e0a15a8_00f0dcf3","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"swift/container/server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":746,"context_line":"                    override_filter_hdr \u003d\u003d sharding_state):"},{"line_number":747,"context_line":"                resp_headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027true\u0027"},{"line_number":748,"context_line":"                marker \u003d end_marker \u003d includes \u003d None"},{"line_number":749,"context_line":"                reverse \u003d False"},{"line_number":750,"context_line":"            states \u003d get_param(req, \u0027states\u0027)"},{"line_number":751,"context_line":"            fill_gaps \u003d False"},{"line_number":752,"context_line":"            if states:"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_590c033b","line":749,"updated":"2020-11-17 23:28:09.000000000","message":"ok, yeah - we talked about this: when talking to a root the proxy wants full shard-range listings so it can cache them","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":648,"context_line":"          otherwise the response body will be a list of objects."},{"line_number":649,"context_line":""},{"line_number":650,"context_line":"        * Both shard range and object listings may be filtered according to"},{"line_number":651,"context_line":"          the constraints described below. However, the"},{"line_number":652,"context_line":"          ``X-Backend-Ignore-Shard-Name-Filter`` header may be used with a"},{"line_number":653,"context_line":"          value in :attr:`swift.common.utils.TRUE_VALUES` to override the"},{"line_number":654,"context_line":"          application of the ``marker``, ``end_marker``, ``includes`` and"},{"line_number":655,"context_line":"          ``reverse`` parameters to shard range listings. Note that this"},{"line_number":656,"context_line":"          headers does not override the ``states`` constraint on shard range"}],"source_content_type":"text/x-python","patch_set":5,"id":"6463bf9a_bc367c95","line":653,"range":{"start_line":651,"start_character":52,"end_line":653,"end_character":69},"updated":"2020-12-13 23:14:13.000000000","message":"... or one of \"sharded\" or \"sharding\".\n\n(Or, I suppose, \"not_found\", \"unsharded\", or \"collapsed\", but those don\u0027t really matter since there wouldn\u0027t be any shard ranges to return, anyway!)","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":648,"context_line":"          otherwise the response body will be a list of objects."},{"line_number":649,"context_line":""},{"line_number":650,"context_line":"        * Both shard range and object listings may be filtered according to"},{"line_number":651,"context_line":"          the constraints described below. However, the"},{"line_number":652,"context_line":"          ``X-Backend-Ignore-Shard-Name-Filter`` header may be used with a"},{"line_number":653,"context_line":"          value in :attr:`swift.common.utils.TRUE_VALUES` to override the"},{"line_number":654,"context_line":"          application of the ``marker``, ``end_marker``, ``includes`` and"},{"line_number":655,"context_line":"          ``reverse`` parameters to shard range listings. Note that this"},{"line_number":656,"context_line":"          headers does not override the ``states`` constraint on shard range"}],"source_content_type":"text/x-python","patch_set":5,"id":"12cbf33f_fd0c2dc3","line":653,"range":{"start_line":651,"start_character":52,"end_line":653,"end_character":69},"in_reply_to":"6463bf9a_bc367c95","updated":"2020-12-15 20:24:10.000000000","message":"good catch","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":717,"context_line":"        delimiter \u003d params.get(\u0027delimiter\u0027)"},{"line_number":718,"context_line":"        marker \u003d params.get(\u0027marker\u0027, \u0027\u0027)"},{"line_number":719,"context_line":"        end_marker \u003d params.get(\u0027end_marker\u0027)"},{"line_number":720,"context_line":"        limit \u003d params.get(\u0027limit\u0027, constraints.CONTAINER_LISTING_LIMIT)"},{"line_number":721,"context_line":"        reverse \u003d config_true_value(params.get(\u0027reverse\u0027))"},{"line_number":722,"context_line":"        out_content_type \u003d listing_formats.get_listing_content_type(req)"},{"line_number":723,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"d1c56b03_5571c93d","line":720,"range":{"start_line":720,"start_character":16,"end_line":720,"end_character":72},"updated":"2020-12-13 23:14:13.000000000","message":"nit: Can just be params[\u0027limit\u0027] -- the call to constrain_req_limit() already knows about the listing limit.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":717,"context_line":"        delimiter \u003d params.get(\u0027delimiter\u0027)"},{"line_number":718,"context_line":"        marker \u003d params.get(\u0027marker\u0027, \u0027\u0027)"},{"line_number":719,"context_line":"        end_marker \u003d params.get(\u0027end_marker\u0027)"},{"line_number":720,"context_line":"        limit \u003d params.get(\u0027limit\u0027, constraints.CONTAINER_LISTING_LIMIT)"},{"line_number":721,"context_line":"        reverse \u003d config_true_value(params.get(\u0027reverse\u0027))"},{"line_number":722,"context_line":"        out_content_type \u003d listing_formats.get_listing_content_type(req)"},{"line_number":723,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"d090e2d3_5504978c","line":720,"range":{"start_line":720,"start_character":16,"end_line":720,"end_character":72},"in_reply_to":"d1c56b03_5571c93d","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":729,"context_line":"                                            stale_reads_ok\u003dTrue)"},{"line_number":730,"context_line":"        info, is_deleted \u003d broker.get_info_is_deleted()"},{"line_number":731,"context_line":"        record_type \u003d req.headers.get(\u0027x-backend-record-type\u0027, \u0027\u0027).lower()"},{"line_number":732,"context_line":"        sharding_state \u003d info.get(\u0027db_state\u0027)"},{"line_number":733,"context_line":"        if record_type \u003d\u003d \u0027auto\u0027 and sharding_state in (SHARDING, SHARDED):"},{"line_number":734,"context_line":"            record_type \u003d \u0027shard\u0027"},{"line_number":735,"context_line":"        if record_type \u003d\u003d \u0027shard\u0027:"}],"source_content_type":"text/x-python","patch_set":5,"id":"81f0692c_a2b5e174","line":732,"range":{"start_line":732,"start_character":8,"end_line":732,"end_character":22},"updated":"2020-12-13 23:14:13.000000000","message":"I think I\u0027d prefer that we keep calling this db_state.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":729,"context_line":"                                            stale_reads_ok\u003dTrue)"},{"line_number":730,"context_line":"        info, is_deleted \u003d broker.get_info_is_deleted()"},{"line_number":731,"context_line":"        record_type \u003d req.headers.get(\u0027x-backend-record-type\u0027, \u0027\u0027).lower()"},{"line_number":732,"context_line":"        sharding_state \u003d info.get(\u0027db_state\u0027)"},{"line_number":733,"context_line":"        if record_type \u003d\u003d \u0027auto\u0027 and sharding_state in (SHARDING, SHARDED):"},{"line_number":734,"context_line":"            record_type \u003d \u0027shard\u0027"},{"line_number":735,"context_line":"        if record_type \u003d\u003d \u0027shard\u0027:"}],"source_content_type":"text/x-python","patch_set":5,"id":"39d3e1cb_ee53d012","line":732,"range":{"start_line":732,"start_character":8,"end_line":732,"end_character":22},"in_reply_to":"81f0692c_a2b5e174","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":743,"context_line":"            includes \u003d params.get(\u0027includes\u0027)"},{"line_number":744,"context_line":"            override_filter_hdr \u003d req.headers.get("},{"line_number":745,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, \u0027false\u0027).lower()"},{"line_number":746,"context_line":"            if (config_true_value(override_filter_hdr) or"},{"line_number":747,"context_line":"                    override_filter_hdr \u003d\u003d sharding_state):"},{"line_number":748,"context_line":"                resp_headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027true\u0027"},{"line_number":749,"context_line":"                marker \u003d end_marker \u003d includes \u003d None"}],"source_content_type":"text/x-python","patch_set":5,"id":"17159860_fc95ad98","line":746,"range":{"start_line":746,"start_character":16,"end_line":746,"end_character":33},"updated":"2020-12-13 23:14:13.000000000","message":"Does the proxy ever actually pass a truthy value? Does anyone else set this header, ever?\n\nI worry a little about maintaining this condition; do we actually anticipate a use-case where callers might send something other than \"sharded\"? Would it be better as something like\n\n if sharding_state \u003d\u003d \"sharded\" \u003d\u003d override_filter_hdr:\n\nuntil we find such a use-case?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":743,"context_line":"            includes \u003d params.get(\u0027includes\u0027)"},{"line_number":744,"context_line":"            override_filter_hdr \u003d req.headers.get("},{"line_number":745,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, \u0027false\u0027).lower()"},{"line_number":746,"context_line":"            if (config_true_value(override_filter_hdr) or"},{"line_number":747,"context_line":"                    override_filter_hdr \u003d\u003d sharding_state):"},{"line_number":748,"context_line":"                resp_headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027true\u0027"},{"line_number":749,"context_line":"                marker \u003d end_marker \u003d includes \u003d None"}],"source_content_type":"text/x-python","patch_set":5,"id":"4d0fa02d_5ea82763","line":746,"range":{"start_line":746,"start_character":16,"end_line":746,"end_character":33},"in_reply_to":"17159860_fc95ad98","updated":"2020-12-15 20:24:10.000000000","message":"I was thinking that if we ever wanted to cache shard ranges during \u0027sharding\u0027 state then the proxy would just send true. But I guess such future proofing isn\u0027t so important - worst case an upgrade proxy would just not get all the ranges from a non-upgraded server during sharding, and would know that from the lack of an x-backend-override-shard-name-filter in the response.\n\nAnd the test coverage needs improving so I\u0027m happy to simplify the cases to test!","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":745,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, \u0027false\u0027).lower()"},{"line_number":746,"context_line":"            if (config_true_value(override_filter_hdr) or"},{"line_number":747,"context_line":"                    override_filter_hdr \u003d\u003d sharding_state):"},{"line_number":748,"context_line":"                resp_headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027true\u0027"},{"line_number":749,"context_line":"                marker \u003d end_marker \u003d includes \u003d None"},{"line_number":750,"context_line":"                reverse \u003d False"},{"line_number":751,"context_line":"            states \u003d params.get(\u0027states\u0027)"}],"source_content_type":"text/x-python","patch_set":5,"id":"7f1f6834_a1bebc58","line":748,"range":{"start_line":748,"start_character":16,"end_line":748,"end_character":28},"updated":"2020-12-13 23:14:13.000000000","message":"This is a regular-old dict -- should we title-case the key for consistency with the other headers from gen_resp_headers()?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":745,"context_line":"                \u0027x-backend-override-shard-name-filter\u0027, \u0027false\u0027).lower()"},{"line_number":746,"context_line":"            if (config_true_value(override_filter_hdr) or"},{"line_number":747,"context_line":"                    override_filter_hdr \u003d\u003d sharding_state):"},{"line_number":748,"context_line":"                resp_headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027true\u0027"},{"line_number":749,"context_line":"                marker \u003d end_marker \u003d includes \u003d None"},{"line_number":750,"context_line":"                reverse \u003d False"},{"line_number":751,"context_line":"            states \u003d params.get(\u0027states\u0027)"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf87f4e9_9c1f6582","line":748,"range":{"start_line":748,"start_character":16,"end_line":748,"end_character":28},"in_reply_to":"7f1f6834_a1bebc58","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":746,"context_line":"            if (config_true_value(override_filter_hdr) or"},{"line_number":747,"context_line":"                    override_filter_hdr \u003d\u003d sharding_state):"},{"line_number":748,"context_line":"                resp_headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027true\u0027"},{"line_number":749,"context_line":"                marker \u003d end_marker \u003d includes \u003d None"},{"line_number":750,"context_line":"                reverse \u003d False"},{"line_number":751,"context_line":"            states \u003d params.get(\u0027states\u0027)"},{"line_number":752,"context_line":"            fill_gaps \u003d False"}],"source_content_type":"text/x-python","patch_set":5,"id":"ca5601c8_85514137","line":749,"updated":"2020-12-13 23:14:13.000000000","message":"Should we clear prefix/delimiter here, too?\n\nOh, right -- never mind. We don\u0027t send those down to the broker, so w/e. And https://review.opendev.org/c/openstack/swift/+/706010 was proxy-only... cool!","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":746,"context_line":"            if (config_true_value(override_filter_hdr) or"},{"line_number":747,"context_line":"                    override_filter_hdr \u003d\u003d sharding_state):"},{"line_number":748,"context_line":"                resp_headers[\u0027x-backend-override-shard-name-filter\u0027] \u003d \u0027true\u0027"},{"line_number":749,"context_line":"                marker \u003d end_marker \u003d includes \u003d None"},{"line_number":750,"context_line":"                reverse \u003d False"},{"line_number":751,"context_line":"            states \u003d params.get(\u0027states\u0027)"},{"line_number":752,"context_line":"            fill_gaps \u003d False"}],"source_content_type":"text/x-python","patch_set":5,"id":"ce6e9a79_ca219767","line":749,"in_reply_to":"ca5601c8_85514137","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":224,"context_line":"    headers \u003d {h: str(v) for h, v in headers.items() if v is not None}"},{"line_number":225,"context_line":"    return headers"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"def headers_to_object_info(headers, status_int\u003dHTTP_OK):"},{"line_number":228,"context_line":"    \"\"\""},{"line_number":229,"context_line":"    Construct a cacheable dict of object info based on response headers."},{"line_number":230,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_50f9ffc5","line":227,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":2272,"context_line":"        except ValueError as err:"},{"line_number":2273,"context_line":"            self.app.logger.error("},{"line_number":2274,"context_line":"                \u0027Problem with listing response from %s: %r\u0027,"},{"line_number":2275,"context_line":"                req.path_qs, err)"},{"line_number":2276,"context_line":"            return None"},{"line_number":2277,"context_line":""},{"line_number":2278,"context_line":"    def _get_container_listing(self, req, account, container, headers\u003dNone,"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_96082a2e","line":2275,"updated":"2020-11-17 23:28:09.000000000","message":"this is probably better than letting the ValueError leak - but what we\u0027re going to want is the name of the database or node so we can track it down and figure out why it\u0027s returning garbage","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":2272,"context_line":"        except ValueError as err:"},{"line_number":2273,"context_line":"            self.app.logger.error("},{"line_number":2274,"context_line":"                \u0027Problem with listing response from %s: %r\u0027,"},{"line_number":2275,"context_line":"                req.path_qs, err)"},{"line_number":2276,"context_line":"            return None"},{"line_number":2277,"context_line":""},{"line_number":2278,"context_line":"    def _get_container_listing(self, req, account, container, headers\u003dNone,"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_dcce3282","line":2275,"in_reply_to":"1f621f24_96082a2e","updated":"2020-11-19 17:12:01.000000000","message":"this is existing code refactored from _get_container_listing\n\nthere\u0027s a few other places that log errors with the container path but not node info - can we defer improving them to a separate patch?","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":198,"context_line":"        \u0027sysmeta\u0027: sysmeta,"},{"line_number":199,"context_line":"        \u0027sharding_state\u0027: headers.get(\u0027x-backend-sharding-state\u0027, \u0027unsharded\u0027),"},{"line_number":200,"context_line":"        # the \u0027internal\u0027 format version of timestamps is cached since the"},{"line_number":201,"context_line":"        # normal format can be derived from this when required"},{"line_number":202,"context_line":"        \u0027created_at\u0027: headers.get(\u0027x-backend-timestamp\u0027),"},{"line_number":203,"context_line":"        \u0027put_timestamp\u0027: headers.get(\u0027x-backend-put-timestamp\u0027),"},{"line_number":204,"context_line":"        \u0027delete_timestamp\u0027: headers.get(\u0027x-backend-delete-timestamp\u0027),"}],"source_content_type":"text/x-python","patch_set":5,"id":"d9ee1fe4_b85f3663","line":201,"updated":"2020-12-13 23:14:13.000000000","message":"Right, so: all of these came in with https://github.com/openstack/swift/commit/4321bb0 forever ago; should be pretty safe to assume they\u0027ll be present.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":198,"context_line":"        \u0027sysmeta\u0027: sysmeta,"},{"line_number":199,"context_line":"        \u0027sharding_state\u0027: headers.get(\u0027x-backend-sharding-state\u0027, \u0027unsharded\u0027),"},{"line_number":200,"context_line":"        # the \u0027internal\u0027 format version of timestamps is cached since the"},{"line_number":201,"context_line":"        # normal format can be derived from this when required"},{"line_number":202,"context_line":"        \u0027created_at\u0027: headers.get(\u0027x-backend-timestamp\u0027),"},{"line_number":203,"context_line":"        \u0027put_timestamp\u0027: headers.get(\u0027x-backend-put-timestamp\u0027),"},{"line_number":204,"context_line":"        \u0027delete_timestamp\u0027: headers.get(\u0027x-backend-delete-timestamp\u0027),"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf30baed_15ac9a05","line":201,"in_reply_to":"d9ee1fe4_b85f3663","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":242,"context_line":"    )"},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"    def lookup(info, key):"},{"line_number":245,"context_line":"        # raises TypeError, KeyError or ValueError"},{"line_number":246,"context_line":"        val \u003d info[key]"},{"line_number":247,"context_line":"        if val is None:"},{"line_number":248,"context_line":"            raise ValueError"}],"source_content_type":"text/x-python","patch_set":5,"id":"0b5cab17_71be12f3","line":245,"range":{"start_line":245,"start_character":17,"end_line":245,"end_character":26},"updated":"2020-12-13 23:14:13.000000000","message":"I guess -- if info isn\u0027t actually a dict? Is that the idea? Is it None sometimes, or something?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":242,"context_line":"    )"},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"    def lookup(info, key):"},{"line_number":245,"context_line":"        # raises TypeError, KeyError or ValueError"},{"line_number":246,"context_line":"        val \u003d info[key]"},{"line_number":247,"context_line":"        if val is None:"},{"line_number":248,"context_line":"            raise ValueError"}],"source_content_type":"text/x-python","patch_set":5,"id":"14051dc4_aa2904c1","line":245,"range":{"start_line":245,"start_character":17,"end_line":245,"end_character":26},"in_reply_to":"0b5cab17_71be12f3","updated":"2020-12-15 20:24:10.000000000","message":"I think I must have been allowing for info\u003d\u003dNone but on reflection that would be better handled explicitly earlier in the function","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":270,"context_line":"            pass"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"    policy_index \u003d info.get(\u0027storage_policy\u0027)"},{"line_number":273,"context_line":"    if policy_index is not None:"},{"line_number":274,"context_line":"        headers[\u0027x-storage-policy\u0027] \u003d POLICIES[int(policy_index)].name"},{"line_number":275,"context_line":"    prefix \u003d get_user_meta_prefix(\u0027container\u0027)"},{"line_number":276,"context_line":"    headers.update("}],"source_content_type":"text/x-python","patch_set":5,"id":"2ec4dd08_b85f0fb2","line":273,"updated":"2020-12-13 23:14:13.000000000","message":"\u0027storage_policy\u0027 is listed in required -- shouldn\u0027t it be a pretty safe bet that\n\n policy_index is not None\n\n?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":270,"context_line":"            pass"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"    policy_index \u003d info.get(\u0027storage_policy\u0027)"},{"line_number":273,"context_line":"    if policy_index is not None:"},{"line_number":274,"context_line":"        headers[\u0027x-storage-policy\u0027] \u003d POLICIES[int(policy_index)].name"},{"line_number":275,"context_line":"    prefix \u003d get_user_meta_prefix(\u0027container\u0027)"},{"line_number":276,"context_line":"    headers.update("}],"source_content_type":"text/x-python","patch_set":5,"id":"5210a445_5c0660c6","line":273,"in_reply_to":"2ec4dd08_b85f0fb2","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":286,"context_line":"        (prefix + k, v)"},{"line_number":287,"context_line":"        for k, v in info.get(\u0027sysmeta\u0027, {}).items())"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"    return headers"},{"line_number":290,"context_line":""},{"line_number":291,"context_line":""},{"line_number":292,"context_line":"def headers_to_object_info(headers, status_int\u003dHTTP_OK):"}],"source_content_type":"text/x-python","patch_set":5,"id":"7d5cdce6_0a52b9cb","line":289,"updated":"2020-11-25 20:27:14.000000000","message":"this seems pretty maintainable - and it\u0027s well tested","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":286,"context_line":"        (prefix + k, v)"},{"line_number":287,"context_line":"        for k, v in info.get(\u0027sysmeta\u0027, {}).items())"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"    return headers"},{"line_number":290,"context_line":""},{"line_number":291,"context_line":""},{"line_number":292,"context_line":"def headers_to_object_info(headers, status_int\u003dHTTP_OK):"}],"source_content_type":"text/x-python","patch_set":5,"id":"f79859c8_c8a8477d","line":289,"in_reply_to":"7d5cdce6_0a52b9cb","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":626,"context_line":"    :param  app: the application object"},{"line_number":627,"context_line":"    :param  account: the unquoted account name"},{"line_number":628,"context_line":"    :param  container: the unquoted container name or None"},{"line_number":629,"context_line":"    :param  resp: the response received or None if info cache should be cleared"},{"line_number":630,"context_line":""},{"line_number":631,"context_line":"    :returns: the info that was placed into the cache, or None if the"},{"line_number":632,"context_line":"              request status was not in (404, 410, 2xx)."}],"source_content_type":"text/x-python","patch_set":5,"id":"d9949a36_38825648","line":629,"range":{"start_line":629,"start_character":43,"end_line":629,"end_character":79},"updated":"2020-12-13 23:14:13.000000000","message":"Off-topic: this is such a weird semantic -- I\u0027m not sure why Sam did it this way (though I think I reviewed it, so maybe I *did*, once).","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":626,"context_line":"    :param  app: the application object"},{"line_number":627,"context_line":"    :param  account: the unquoted account name"},{"line_number":628,"context_line":"    :param  container: the unquoted container name or None"},{"line_number":629,"context_line":"    :param  resp: the response received or None if info cache should be cleared"},{"line_number":630,"context_line":""},{"line_number":631,"context_line":"    :returns: the info that was placed into the cache, or None if the"},{"line_number":632,"context_line":"              request status was not in (404, 410, 2xx)."}],"source_content_type":"text/x-python","patch_set":5,"id":"2fcbb184_4c6a6616","line":629,"range":{"start_line":629,"start_character":43,"end_line":629,"end_character":79},"in_reply_to":"d9949a36_38825648","updated":"2020-12-15 20:24:10.000000000","message":"I guess it was a convenience perhaps?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":201,"context_line":"                                           end_marker)"},{"line_number":202,"context_line":"        if reverse:"},{"line_number":203,"context_line":"            shard_ranges.reverse()"},{"line_number":204,"context_line":"        resp.body \u003d json.dumps([dict(sr) for sr in shard_ranges]).encode(\u0027ascii\u0027)"},{"line_number":205,"context_line":"        return resp"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"    def GETorHEAD(self, req):"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_f012cb08","line":204,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E501 line too long (81 \u003e 79 characters)","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":114,"context_line":"            record_type \u003d \u0027auto\u0027"},{"line_number":115,"context_line":"            req.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":116,"context_line":"            params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":117,"context_line":"        req.params \u003d params"},{"line_number":118,"context_line":"        resp \u003d self.GETorHEAD_base("},{"line_number":119,"context_line":"            req, _(\u0027Container\u0027), node_iter, part,"},{"line_number":120,"context_line":"            req.swift_entity_path, concurrency)"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_d64aa2aa","side":"PARENT","line":117,"updated":"2020-11-17 23:28:09.000000000","message":"oic, the _param_cache hack was pre-existing - no comment or nothing; wonky","commit_id":"90c737e3550a3fee1c7581c3050ea64f2a9255cd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":125,"context_line":"        params \u003d req.params"},{"line_number":126,"context_line":"        params[\u0027format\u0027] \u003d \u0027json\u0027"},{"line_number":127,"context_line":"        record_type \u003d req.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027).lower()"},{"line_number":128,"context_line":"        if not record_type:"},{"line_number":129,"context_line":"            record_type \u003d \u0027auto\u0027"},{"line_number":130,"context_line":"            req.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":131,"context_line":"            params[\u0027states\u0027] \u003d \u0027listing\u0027"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_9664aae7","line":128,"updated":"2020-11-17 23:28:09.000000000","message":"In what context do we expect _GET to be called with a req that has X-Backend-Record-Type already?\n\nsomehow this block doesn\u0027t seem to be covered by test_GET_shard_ranges (but it is covered by test_GET_sharded_container)","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":125,"context_line":"        params \u003d req.params"},{"line_number":126,"context_line":"        params[\u0027format\u0027] \u003d \u0027json\u0027"},{"line_number":127,"context_line":"        record_type \u003d req.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027).lower()"},{"line_number":128,"context_line":"        if not record_type:"},{"line_number":129,"context_line":"            record_type \u003d \u0027auto\u0027"},{"line_number":130,"context_line":"            req.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":131,"context_line":"            params[\u0027states\u0027] \u003d \u0027listing\u0027"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_9c6caa09","line":128,"in_reply_to":"1f621f24_9664aae7","updated":"2020-11-19 17:12:01.000000000","message":"the sharder uses internal client to fetch shard ranges from roots during audit and misplaced objects, and sets x-backend-record-type\u003d\u0027shard\u0027\n\nhmmm, we may not want those requests served from cache?? they also send x-newest\u003dtrue and it seems like x-newest ought to imply fetch from backend","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":129,"context_line":"            record_type \u003d \u0027auto\u0027"},{"line_number":130,"context_line":"            req.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":131,"context_line":"            params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":132,"context_line":"        req.params \u003d params"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"        memcache \u003d cache_from_env(req.environ, True)"},{"line_number":135,"context_line":"        if (self.app.recheck_listing_shard_ranges \u003c\u003d 0 or"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_af8a2ebb","line":132,"updated":"2020-11-17 23:28:09.000000000","message":"this is tricky, there\u0027s no copy, so I thought the above should be a modify in place and all this looked obfuscated\n\nthe \"problem\" seems to be the _params_cache and QUERY_STRING handling in the req.environ\n\nI\u0027d can\u0027t find any existing pattern that would make this more obvious; it kinda bothers me","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":129,"context_line":"            record_type \u003d \u0027auto\u0027"},{"line_number":130,"context_line":"            req.headers[\u0027X-Backend-Record-Type\u0027] \u003d \u0027auto\u0027"},{"line_number":131,"context_line":"            params[\u0027states\u0027] \u003d \u0027listing\u0027"},{"line_number":132,"context_line":"        req.params \u003d params"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"        memcache \u003d cache_from_env(req.environ, True)"},{"line_number":135,"context_line":"        if (self.app.recheck_listing_shard_ranges \u003c\u003d 0 or"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_bc724ee2","line":132,"in_reply_to":"1f621f24_af8a2ebb","updated":"2020-11-19 17:12:01.000000000","message":"yes. this is just moved existing code but I have a memory of finding this awkward when I first dealt with it.","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":141,"context_line":"            # \u0027x-backend-include-deleted\u0027 is not usually expected in requests"},{"line_number":142,"context_line":"            # to the proxy (it is used from sharder to container servers) but"},{"line_number":143,"context_line":"            # check just in case because we don\u0027t cache deleted shard ranges"},{"line_number":144,"context_line":"            return self._GETorHEAD_from_backend(req)"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"        info \u003d _get_info_from_caches(self.app, req.environ, self.account_name,"},{"line_number":147,"context_line":"                                     self.container_name)"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_f6e4fe3e","line":144,"updated":"2020-11-17 23:28:09.000000000","message":"generally I find the the \"guard return pattern\" simplifies things - but this is 1/3 ways we return from this method and boolean logic feels kind of like a \"list of bug-fixes\" anti-pattern","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":141,"context_line":"            # \u0027x-backend-include-deleted\u0027 is not usually expected in requests"},{"line_number":142,"context_line":"            # to the proxy (it is used from sharder to container servers) but"},{"line_number":143,"context_line":"            # check just in case because we don\u0027t cache deleted shard ranges"},{"line_number":144,"context_line":"            return self._GETorHEAD_from_backend(req)"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"        info \u003d _get_info_from_caches(self.app, req.environ, self.account_name,"},{"line_number":147,"context_line":"                                     self.container_name)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_9c45ca73","line":144,"in_reply_to":"1f621f24_f6e4fe3e","updated":"2020-11-19 17:12:01.000000000","message":"I to\u0027d and fro\u0027d on this - on the one hand I dislike too many returns from a method, on the other hand I liked reducing GETorHEAD to stuff that was genuinely common to GET and HEAD\n\nI will reconsider","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"        info \u003d _get_info_from_caches(self.app, req.environ, self.account_name,"},{"line_number":147,"context_line":"                                     self.container_name)"},{"line_number":148,"context_line":"        if (record_type in (\u0027shard\u0027, \u0027auto\u0027) and"},{"line_number":149,"context_line":"                info and is_success(info[\u0027status\u0027]) and"},{"line_number":150,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":151,"context_line":"            # container is sharded so we may have the shard ranges cached"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_b67ba627","line":148,"updated":"2020-11-17 23:28:09.000000000","message":"In what context would record_type be something else?  It doesn\u0027t seem to be tested - can we add it to the guard above?","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"        info \u003d _get_info_from_caches(self.app, req.environ, self.account_name,"},{"line_number":147,"context_line":"                                     self.container_name)"},{"line_number":148,"context_line":"        if (record_type in (\u0027shard\u0027, \u0027auto\u0027) and"},{"line_number":149,"context_line":"                info and is_success(info[\u0027status\u0027]) and"},{"line_number":150,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":151,"context_line":"            # container is sharded so we may have the shard ranges cached"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_af3ccebf","line":148,"in_reply_to":"1f621f24_b67ba627","updated":"2020-11-19 17:12:01.000000000","message":"record_type is set to \u0027object\u0027 in get_from_shard_ranges when we have to go back to the root container for some of the listing\n\nyes, we could flip the logic and route to backend earlier if record_type is object, since we\u0027re definitely not expecting the backend to return shards in that case!","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":163,"context_line":"                if cached_ranges is not None:"},{"line_number":164,"context_line":"                    # shard ranges can be returned from cache"},{"line_number":165,"context_line":"                    self.app.logger.debug(\u0027Found %d shards in cache for %s\u0027,"},{"line_number":166,"context_line":"                                          len(cached_ranges), req.path_qs)"},{"line_number":167,"context_line":"                    headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":168,"context_line":"                                    \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":169,"context_line":"                    shard_ranges \u003d self._filter_resp_shard_ranges("}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_16e09acc","line":166,"updated":"2020-11-17 23:28:09.000000000","message":"LOvE this message!","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":184,"context_line":"        # The request was not fulfilled from cache so send to the backend"},{"line_number":185,"context_line":"        # server, but instruct the backend server to ignore name constraints in"},{"line_number":186,"context_line":"        # request params if returning shard ranges so that the response can"},{"line_number":187,"context_line":"        # potentially be cached. Only do this if the container state is"},{"line_number":188,"context_line":"        # \u0027sharded\u0027. We don\u0027t attempt to cache shard ranges for a \u0027sharding\u0027"},{"line_number":189,"context_line":"        # container as they may include the container itself as a \u0027gap filler\u0027"},{"line_number":190,"context_line":"        # for shard ranges that have not yet cleaved; listings from \u0027gap"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_b939d7d9","line":187,"updated":"2020-11-17 23:28:09.000000000","message":"this explains the container.server change","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":200,"context_line":"        resp_record_type \u003d resp.headers.get("},{"line_number":201,"context_line":"            \u0027x-backend-record-type\u0027, \u0027\u0027).lower()"},{"line_number":202,"context_line":"        complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":203,"context_line":"            \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":204,"context_line":"        # given that we sent \u0027x-backend-override-shard-name-filter\u003dsharded\u0027 we"},{"line_number":205,"context_line":"        # should only receive back \u0027x-backend-override-shard-name-filter\u003dtrue\u0027"},{"line_number":206,"context_line":"        # if the sharding state is \u0027sharded\u0027, but check them both anyway..."}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_394b0760","line":203,"updated":"2020-11-17 23:28:09.000000000","message":"this protects us against old container.server responses","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":211,"context_line":"            # them and replace response body with filtered listing"},{"line_number":212,"context_line":"            cache_key \u003d get_cache_key(self.account_name, self.container_name,"},{"line_number":213,"context_line":"                                      shard\u003d\u0027listing\u0027)"},{"line_number":214,"context_line":"            data \u003d self._parse_listing_response(req, resp)"},{"line_number":215,"context_line":"            backend_shard_ranges \u003d self._parse_shard_ranges(req, data, resp)"},{"line_number":216,"context_line":"            if backend_shard_ranges is None:"},{"line_number":217,"context_line":"                # unexpected case: error parsing shard ranges; we have to"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_7612ae64","line":214,"updated":"2020-11-17 23:28:09.000000000","message":"I think most of the refactor over in base was to support code re-use here?","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":211,"context_line":"            # them and replace response body with filtered listing"},{"line_number":212,"context_line":"            cache_key \u003d get_cache_key(self.account_name, self.container_name,"},{"line_number":213,"context_line":"                                      shard\u003d\u0027listing\u0027)"},{"line_number":214,"context_line":"            data \u003d self._parse_listing_response(req, resp)"},{"line_number":215,"context_line":"            backend_shard_ranges \u003d self._parse_shard_ranges(req, data, resp)"},{"line_number":216,"context_line":"            if backend_shard_ranges is None:"},{"line_number":217,"context_line":"                # unexpected case: error parsing shard ranges; we have to"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_2f7a9e87","line":214,"in_reply_to":"1f621f24_7612ae64","updated":"2020-11-19 17:12:01.000000000","message":"yes","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":219,"context_line":"                # consumed, so return empty list without caching it"},{"line_number":220,"context_line":"                cached_ranges \u003d []"},{"line_number":221,"context_line":"            else:"},{"line_number":222,"context_line":"                cached_ranges \u003d [dict(sr) for sr in backend_shard_ranges]"},{"line_number":223,"context_line":"                if resp.headers.get(\u0027x-backend-sharding-state\u0027) \u003d\u003d \u0027sharded\u0027:"},{"line_number":224,"context_line":"                    # cache in infocache even if no shard ranges returned; this"},{"line_number":225,"context_line":"                    # is unexpected but use that result for this request"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_9622ea00","line":222,"updated":"2020-11-17 23:28:09.000000000","message":"from what I can tell, the parts that are going to get stale the quickest are the bits we don\u0027t need like \"object_count\"\n\n  {\n    \"object_count\": 50, \n    \"lower\": \"\", \n    \"name\": \".shards_AUTH_test/lots-of-files-9933d06bdb1de26a4adfa0a7692feae9-1605650820.78208-0\", \n    \"upper\": \"obj0049\", \n    \"deleted\": 0, \n    \"bytes_used\": 0, \n    \"meta_timestamp\": \"1605650860.13947\", \n    \"state_timestamp\": \"1605650820.78208\", \n    \"reported\": 0, \n    \"epoch\": null, \n    \"state\": 40, \n    \"timestamp\": \"1605650820.78208\"\n  },\n\nI\u0027d need to test how robust re-hydration is to missing attributes tho...","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":234,"context_line":"                            time\u003dself.app.recheck_listing_shard_ranges)"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"            # filter returned shard ranges according to request constraints"},{"line_number":237,"context_line":"            resp.body \u003d self._filter_resp_shard_ranges(req, cached_ranges)"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"        return resp"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_993b3bc7","line":237,"updated":"2020-11-17 23:28:09.000000000","message":"regardless of cache or backend resp - we always filter returned ranges as the existing \"get listing from shards\" code was dealing with legacy (filtered) responses","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":234,"context_line":"                            time\u003dself.app.recheck_listing_shard_ranges)"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"            # filter returned shard ranges according to request constraints"},{"line_number":237,"context_line":"            resp.body \u003d self._filter_resp_shard_ranges(req, cached_ranges)"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"        return resp"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_aff4ced0","line":237,"in_reply_to":"1f621f24_993b3bc7","updated":"2020-11-19 17:12:01.000000000","message":"also, a request from sharder internal client for shards might expect to be filtered (although, as commented above, we might not want to serve those from cache)\n\nI considered a possible optimisation to pass the whole list of shard ranges to get_from_shard_ranges and have it filter them on the fly (so only one iteration over the ranges) but so far I prefer keeping it simple and keeping the \u0027contract\u0027 of the GET unchanged. Also, in _filter_resp_shard_ranges I use the exact same filter_shard_ranges method as the backend so the two behaviours should stay in sync","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":257,"context_line":"        if req.method \u003d\u003d \u0027HEAD\u0027:"},{"line_number":258,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":259,"context_line":"        else:"},{"line_number":260,"context_line":"            resp \u003d self._GET(req)"},{"line_number":261,"context_line":""},{"line_number":262,"context_line":"        req_record_type \u003d req.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":263,"context_line":"        resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_765aae83","line":260,"updated":"2020-11-17 23:28:09.000000000","message":"I think this is the only entry point into _GET\n\nand it\u0027s a bit tricky, because the resp is *either*\n\n1) a virtual resp from cache\nor\n2) just exactly the same as the if branch above - a resp from  _GETorHEAD_from_backend\n\n\nmaybe:\n\n    if \"we_are_cool_with_a_virtual_cache_resp\":\n        resp \u003d get_from_cache_maybe()\n    if not is_good_enough(resp):\n        resp \u003d self._GETorHEAD_from_backend(req)","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":257,"context_line":"        if req.method \u003d\u003d \u0027HEAD\u0027:"},{"line_number":258,"context_line":"            resp \u003d self._GETorHEAD_from_backend(req)"},{"line_number":259,"context_line":"        else:"},{"line_number":260,"context_line":"            resp \u003d self._GET(req)"},{"line_number":261,"context_line":""},{"line_number":262,"context_line":"        req_record_type \u003d req.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":263,"context_line":"        resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_af1dee10","line":260,"in_reply_to":"1f621f24_765aae83","updated":"2020-11-19 17:12:01.000000000","message":"ok, will reconsider - I was 50/50 on where the GET path branching should live","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":263,"context_line":"        resp_record_type \u003d resp.headers.get(\u0027X-Backend-Record-Type\u0027, \u0027\u0027)"},{"line_number":264,"context_line":"        if all((req.method \u003d\u003d \"GET\", req_record_type.lower() \u003d\u003d \u0027auto\u0027,"},{"line_number":265,"context_line":"               resp_record_type.lower() \u003d\u003d \u0027shard\u0027)):"},{"line_number":266,"context_line":"            resp \u003d self._get_from_shards(req, resp)"},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"        if not config_true_value("},{"line_number":269,"context_line":"                resp.headers.get(\u0027X-Backend-Cached-Results\u0027)):"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_efc5365d","line":266,"updated":"2020-11-19 17:12:01.000000000","message":"you may wonder why I did not move this into _GET - that\u0027s because it struck me that we could move this to after the ACL auth check below, to avoid gathering shard listings when the request will be denied.\n\nBut IMHO I\u0027d like to do that in a separate patch, and if I move the branch to GET from backend back into this method then we need this to be in this method too!","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":268,"context_line":"        if not config_true_value("},{"line_number":269,"context_line":"                resp.headers.get(\u0027X-Backend-Cached-Results\u0027)):"},{"line_number":270,"context_line":"            # Cache this. We just made a request to a storage node and got"},{"line_number":271,"context_line":"            # up-to-date information for the container."},{"line_number":272,"context_line":"            resp.headers[\u0027X-Backend-Recheck-Container-Existence\u0027] \u003d str("},{"line_number":273,"context_line":"                self.app.recheck_container_existence)"},{"line_number":274,"context_line":"            set_info_cache(self.app, req.environ, self.account_name,"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_6fe8167b","line":271,"updated":"2020-11-17 23:28:09.000000000","message":"so this part in particular is useful to HEAD or GET - it\u0027s the container info headers","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":296,"context_line":"        shard_ranges \u003d [ShardRange.from_dict(data)"},{"line_number":297,"context_line":"                        for data in json.loads(resp.body)]"},{"line_number":298,"context_line":"        self.app.logger.debug(\u0027GET listing from %s shards for: %s\u0027,"},{"line_number":299,"context_line":"                              len(shard_ranges), req.path_qs)"},{"line_number":300,"context_line":"        if not shard_ranges:"},{"line_number":301,"context_line":"            # can\u0027t find ranges or there was a problem getting the ranges. So"},{"line_number":302,"context_line":"            # return what we have."}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_7633ce63","line":299,"updated":"2020-11-17 23:28:09.000000000","message":"don\u0027t love how we talk about \"from N shards\" here but the resp.body is already filtered...\n\nbut i guess in this context adding the index below makes more sense","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":358,"context_line":"                        continue"},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"            self.app.logger.debug(\u0027Getting from shard[%d] %s %s with %s\u0027,"},{"line_number":361,"context_line":"                                  i, shard_range, shard_range.name, headers)"},{"line_number":362,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":363,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":364,"context_line":"                headers\u003dheaders, params\u003dparams)"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_f6d87e1f","line":361,"updated":"2020-11-17 23:28:09.000000000","message":"is the index mis-leading when there\u0027s a marker or prefix or whatever?\n\nNov 17 22:36:22 saio proxy-server: Found 5 shards in cache for /v1/AUTH_test/lots-of-files?marker\u003dobj021\u0026states\u003dlisting\u0026limit\u003d10\u0026format\u003djson (txn: tx960b6e0f68d140d19d754-005fb45066) (client_ip: 127.0.0.1)\nNov 17 22:36:22 saio proxy-server: GET listing from 1 shards for: /v1/AUTH_test/lots-of-files?marker\u003dobj021\u0026states\u003dlisting\u0026limit\u003d10\u0026format\u003djson (txn: tx960b6e0f68d140d19d754-005fb45066) (client_ip: 127.0.0.1)\nNov 17 22:36:22 saio proxy-server: Getting from shard[0] ShardRange\u003c\u0027obj0199\u0027 to MaxBound as of 1605650820.78208, (34, 0) as of 1605650865.08684, active as of 1605650820.78208\u003e .shards_AUTH_test/lots-of-files-9933d06bdb1de26a4adfa0a7692feae9-1605650820.78208-4 with None (txn: tx960b6e0f68d140d19d754-005fb45066) (client_ip: 127.0.0.1)","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":358,"context_line":"                        continue"},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"            self.app.logger.debug(\u0027Getting from shard[%d] %s %s with %s\u0027,"},{"line_number":361,"context_line":"                                  i, shard_range, shard_range.name, headers)"},{"line_number":362,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":363,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":364,"context_line":"                headers\u003dheaders, params\u003dparams)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_8fe62ae8","line":361,"in_reply_to":"1f621f24_f6d87e1f","updated":"2020-11-19 17:12:01.000000000","message":"hmmm, I see what you mean, but not sure which way to go to make it better - not sure I want to figure out the relative offset of the filtered shard ranges\n\nI could add another debug message after the \u0027Found 5...\u0027: \u0027Filtered 1 ...\u0027 so there\u0027s a hint at the index\n\nOne reason I added the indexes was to help my eyes see nested GETs i.e. where indexes might go 0, 1, 0, 1, 2, 2, 3. I even considered passing a recursion depth so I could index 0, 1, 1.0, 1.1, 1.2, 2, 3\n\nPerhaps my mistake was to append the index to shard i.e. shard[n] which perhaps implies that is the shard index in the whole set?","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":361,"context_line":"                                  i, shard_range, shard_range.name, headers)"},{"line_number":362,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":363,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":364,"context_line":"                headers\u003dheaders, params\u003dparams)"},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"            shard_state \u003d \u0027unknown\u0027"},{"line_number":367,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_d6546224","line":364,"updated":"2020-11-17 23:28:09.000000000","message":"kinda fun, i forgot to run my container-updater and was surprised to see the account existence check failing:\n\nNov 17 23:07:31 saio account-6022: 127.0.0.1 - - [17/Nov/2020:23:07:31 +0000] \"HEAD /sdb2/61/.shards_AUTH_test\" 404 - \"HEAD http://saio:8080/v1/.shards_AUTH_test?format\u003djson\" \"txa3050e625fb14d90ab1bb-005fb457b3\" \"proxy-server 9980\" 0.0005 \"-\" 8698 -\n\n\nthe listing to the shard would still go through successfully tho","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":93,"context_line":"        clear_info_cache(self.app, req.environ,"},{"line_number":94,"context_line":"                         self.account_name, self.container_name)"},{"line_number":95,"context_line":"        clear_info_cache(self.app, req.environ,"},{"line_number":96,"context_line":"                         self.account_name, self.container_name, \u0027listing\u0027)"},{"line_number":97,"context_line":"        # TODO: should we also purge updating shards from cache?"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"    def _GETorHEAD_from_backend(self, req):"}],"source_content_type":"text/x-python","patch_set":5,"id":"3ce8ae67_39280d5f","line":96,"updated":"2020-12-13 23:14:13.000000000","message":"Client can\u0027t do anything directly to impact the state of shard ranges, though, right? I\u0027m not sure this is strictly necessary (though it probably(?) doesn\u0027t hurt much of anything).","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":93,"context_line":"        clear_info_cache(self.app, req.environ,"},{"line_number":94,"context_line":"                         self.account_name, self.container_name)"},{"line_number":95,"context_line":"        clear_info_cache(self.app, req.environ,"},{"line_number":96,"context_line":"                         self.account_name, self.container_name, \u0027listing\u0027)"},{"line_number":97,"context_line":"        # TODO: should we also purge updating shards from cache?"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"    def _GETorHEAD_from_backend(self, req):"}],"source_content_type":"text/x-python","patch_set":5,"id":"630af087_0b6b938c","line":96,"in_reply_to":"3ce8ae67_39280d5f","updated":"2020-12-15 20:24:10.000000000","message":"TBH I was being cautious/not wanting to think it through too much. I don\u0027t think PUT/POST/DELETE can change anything that wouldn\u0027t be reflected in the cached metadata anyway (like deleted) but decided to play it safe.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":122,"context_line":"                                           end_marker)"},{"line_number":123,"context_line":"        if reverse:"},{"line_number":124,"context_line":"            shard_ranges.reverse()"},{"line_number":125,"context_line":"        return json.dumps([dict(sr) for sr in shard_ranges]).encode(\u0027ascii\u0027)"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def _GET_using_cache(self, req):"},{"line_number":128,"context_line":"        # It may be possible to fulfil the request from cache: we only reach"}],"source_content_type":"text/x-python","patch_set":5,"id":"ea1f6f8d_8481d62f","line":125,"range":{"start_line":125,"start_character":15,"end_line":125,"end_character":25},"updated":"2020-12-13 23:14:13.000000000","message":"OK, so we have to re-serialize them here so we can deserialize them again down in _get_from_shards()... and even if we get a set of shard ranges from the backend, we\u0027ll still come through here to do the filtering 🤔\n\nAnd we\u0027ve also been seeing shard range sets exceeding the default memcached size limit... gotta keep that GC working hard, I guess!","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":122,"context_line":"                                           end_marker)"},{"line_number":123,"context_line":"        if reverse:"},{"line_number":124,"context_line":"            shard_ranges.reverse()"},{"line_number":125,"context_line":"        return json.dumps([dict(sr) for sr in shard_ranges]).encode(\u0027ascii\u0027)"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def _GET_using_cache(self, req):"},{"line_number":128,"context_line":"        # It may be possible to fulfil the request from cache: we only reach"}],"source_content_type":"text/x-python","patch_set":5,"id":"1b5285cd_ea256d7e","line":125,"range":{"start_line":125,"start_character":15,"end_line":125,"end_character":25},"in_reply_to":"ea1f6f8d_8481d62f","updated":"2020-12-15 20:24:10.000000000","message":"yeah, that\u0027s a little annoying","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def _GET_using_cache(self, req):"},{"line_number":128,"context_line":"        # It may be possible to fulfil the request from cache: we only reach"},{"line_number":129,"context_line":"        # here if record_type is \u0027shard\u0027 or \u0027auto\u0027, so if the container state"},{"line_number":130,"context_line":"        # is \u0027sharded\u0027 then look for cached shard ranges. However, if X-Newest"},{"line_number":131,"context_line":"        # is true then we always fetch from the backend servers."},{"line_number":132,"context_line":"        get_newest \u003d config_true_value(req.headers.get(\u0027x-newest\u0027, False))"}],"source_content_type":"text/x-python","patch_set":5,"id":"fdf03926_f47c2ac4","line":129,"range":{"start_line":129,"start_character":18,"end_line":129,"end_character":29},"updated":"2020-12-13 23:14:13.000000000","message":"...*request* record type, yeah? Just to be clear.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def _GET_using_cache(self, req):"},{"line_number":128,"context_line":"        # It may be possible to fulfil the request from cache: we only reach"},{"line_number":129,"context_line":"        # here if record_type is \u0027shard\u0027 or \u0027auto\u0027, so if the container state"},{"line_number":130,"context_line":"        # is \u0027sharded\u0027 then look for cached shard ranges. However, if X-Newest"},{"line_number":131,"context_line":"        # is true then we always fetch from the backend servers."},{"line_number":132,"context_line":"        get_newest \u003d config_true_value(req.headers.get(\u0027x-newest\u0027, False))"}],"source_content_type":"text/x-python","patch_set":5,"id":"3584d0e1_ded4f9d1","line":129,"range":{"start_line":129,"start_character":18,"end_line":129,"end_character":29},"in_reply_to":"fdf03926_f47c2ac4","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":132,"context_line":"        get_newest \u003d config_true_value(req.headers.get(\u0027x-newest\u0027, False))"},{"line_number":133,"context_line":"        if get_newest:"},{"line_number":134,"context_line":"            self.app.logger.debug("},{"line_number":135,"context_line":"                \u0027Skipping shard cache lookup (x-newest) for %s\u0027, req.path_qs)"},{"line_number":136,"context_line":"            info \u003d None"},{"line_number":137,"context_line":"        else:"},{"line_number":138,"context_line":"            info \u003d _get_info_from_caches(self.app, req.environ,"}],"source_content_type":"text/x-python","patch_set":5,"id":"987aaa76_a1409523","line":135,"updated":"2020-11-25 20:27:14.000000000","message":"noice","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":132,"context_line":"        get_newest \u003d config_true_value(req.headers.get(\u0027x-newest\u0027, False))"},{"line_number":133,"context_line":"        if get_newest:"},{"line_number":134,"context_line":"            self.app.logger.debug("},{"line_number":135,"context_line":"                \u0027Skipping shard cache lookup (x-newest) for %s\u0027, req.path_qs)"},{"line_number":136,"context_line":"            info \u003d None"},{"line_number":137,"context_line":"        else:"},{"line_number":138,"context_line":"            info \u003d _get_info_from_caches(self.app, req.environ,"}],"source_content_type":"text/x-python","patch_set":5,"id":"d33bf0e4_06ba12b3","line":135,"in_reply_to":"987aaa76_a1409523","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":141,"context_line":"        if (info and is_success(info[\u0027status\u0027]) and"},{"line_number":142,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":143,"context_line":"            # container is sharded so we may have the shard ranges cached"},{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":148,"context_line":"                memcache \u003d cache_from_env(req.environ, True)"}],"source_content_type":"text/x-python","patch_set":5,"id":"049a8a29_26c184b3","line":145,"range":{"start_line":144,"start_character":12,"end_line":145,"end_character":23},"updated":"2020-12-13 23:14:13.000000000","message":"There aren\u0027t *that* many places where I\u0027d like to be able to use assignment expressions, but getting to move this up into the preceding `if` and reclaim some indent is one of them. 😉","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":141,"context_line":"        if (info and is_success(info[\u0027status\u0027]) and"},{"line_number":142,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":143,"context_line":"            # container is sharded so we may have the shard ranges cached"},{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":148,"context_line":"                memcache \u003d cache_from_env(req.environ, True)"}],"source_content_type":"text/x-python","patch_set":5,"id":"b6121dc5_132fbc38","line":145,"range":{"start_line":144,"start_character":12,"end_line":145,"end_character":23},"in_reply_to":"049a8a29_26c184b3","updated":"2020-12-15 20:24:10.000000000","message":"+1","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":148,"context_line":"                memcache \u003d cache_from_env(req.environ, True)"},{"line_number":149,"context_line":"                cache_key \u003d get_cache_key(self.account_name,"},{"line_number":150,"context_line":"                                          self.container_name,"}],"source_content_type":"text/x-python","patch_set":5,"id":"927f2fb7_6bed499d","line":147,"range":{"start_line":147,"start_character":39,"end_line":147,"end_character":73},"updated":"2020-12-13 23:14:13.000000000","message":"Surely if we get here, this key already exists, right? _get_info_from_caches would have stuffed the info dict that it returned in there.\n\nEh, belt \u0026 bracers, I suppose.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":148,"context_line":"                memcache \u003d cache_from_env(req.environ, True)"},{"line_number":149,"context_line":"                cache_key \u003d get_cache_key(self.account_name,"},{"line_number":150,"context_line":"                                          self.container_name,"}],"source_content_type":"text/x-python","patch_set":5,"id":"a2a7fc5e_2ed09cb0","line":147,"range":{"start_line":147,"start_character":39,"end_line":147,"end_character":73},"in_reply_to":"927f2fb7_6bed499d","updated":"2020-12-15 20:24:10.000000000","message":"agree but having typed it I\u0027ll stick with belt\u0026braces","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":148,"context_line":"                memcache \u003d cache_from_env(req.environ, True)"},{"line_number":149,"context_line":"                cache_key \u003d get_cache_key(self.account_name,"},{"line_number":150,"context_line":"                                          self.container_name,"},{"line_number":151,"context_line":"                                          shard\u003d\u0027listing\u0027)"},{"line_number":152,"context_line":"                cached_ranges \u003d infocache.get(cache_key)"},{"line_number":153,"context_line":"                if cached_ranges is None and memcache:"},{"line_number":154,"context_line":"                    cached_ranges \u003d memcache.get(cache_key)"},{"line_number":155,"context_line":"                if cached_ranges is not None:"},{"line_number":156,"context_line":"                    # shard ranges can be returned from cache"},{"line_number":157,"context_line":"                    self.app.logger.debug(\u0027Found %d shards in cache for %s\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba110dab_60daac07","line":154,"range":{"start_line":148,"start_character":16,"end_line":154,"end_character":59},"updated":"2020-12-13 23:14:13.000000000","message":"We can\u0027t just use _get_info_from_caches? I suppose it doesn\u0027t have the shard kwarg -- should we fix that?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":145,"context_line":"            if headers:"},{"line_number":146,"context_line":"                # only use cached values if all required headers available"},{"line_number":147,"context_line":"                infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":148,"context_line":"                memcache \u003d cache_from_env(req.environ, True)"},{"line_number":149,"context_line":"                cache_key \u003d get_cache_key(self.account_name,"},{"line_number":150,"context_line":"                                          self.container_name,"},{"line_number":151,"context_line":"                                          shard\u003d\u0027listing\u0027)"},{"line_number":152,"context_line":"                cached_ranges \u003d infocache.get(cache_key)"},{"line_number":153,"context_line":"                if cached_ranges is None and memcache:"},{"line_number":154,"context_line":"                    cached_ranges \u003d memcache.get(cache_key)"},{"line_number":155,"context_line":"                if cached_ranges is not None:"},{"line_number":156,"context_line":"                    # shard ranges can be returned from cache"},{"line_number":157,"context_line":"                    self.app.logger.debug(\u0027Found %d shards in cache for %s\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"865d3194_bada8020","line":154,"range":{"start_line":148,"start_character":16,"end_line":154,"end_character":59},"in_reply_to":"ba110dab_60daac07","updated":"2020-12-15 20:24:10.000000000","message":"_get_info_from_caches returns a dict, cached_ranges is a list. There\u0027s a similar pattern here swift.proxy.controllers.base.Controller._get_update_shard","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":153,"context_line":"                if cached_ranges is None and memcache:"},{"line_number":154,"context_line":"                    cached_ranges \u003d memcache.get(cache_key)"},{"line_number":155,"context_line":"                if cached_ranges is not None:"},{"line_number":156,"context_line":"                    # shard ranges can be returned from cache"},{"line_number":157,"context_line":"                    self.app.logger.debug(\u0027Found %d shards in cache for %s\u0027,"},{"line_number":158,"context_line":"                                          len(cached_ranges), req.path_qs)"},{"line_number":159,"context_line":"                    headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"07844cb0_978e1528","line":156,"updated":"2020-12-13 23:14:13.000000000","message":"Should we do a spot-check that there are no gaps in the ranges?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":true,"context_lines":[{"line_number":153,"context_line":"                if cached_ranges is None and memcache:"},{"line_number":154,"context_line":"                    cached_ranges \u003d memcache.get(cache_key)"},{"line_number":155,"context_line":"                if cached_ranges is not None:"},{"line_number":156,"context_line":"                    # shard ranges can be returned from cache"},{"line_number":157,"context_line":"                    self.app.logger.debug(\u0027Found %d shards in cache for %s\u0027,"},{"line_number":158,"context_line":"                                          len(cached_ranges), req.path_qs)"},{"line_number":159,"context_line":"                    headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"37129317_c4299272","line":156,"in_reply_to":"07844cb0_978e1528","updated":"2020-12-15 20:24:10.000000000","message":"good question!\n\nif we were to do a check perhaps it should be before writing shard ranges *to* cache - but we don\u0027t require shard ranges from backend to be without gaps, so we\u0027d potentially just be hitting the backend more for the same result. On the other hand, maybe the backend fixes the gap while gappy data is in cache.\n\nI\u0027ll ponder this some more","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":155,"context_line":"                if cached_ranges is not None:"},{"line_number":156,"context_line":"                    # shard ranges can be returned from cache"},{"line_number":157,"context_line":"                    self.app.logger.debug(\u0027Found %d shards in cache for %s\u0027,"},{"line_number":158,"context_line":"                                          len(cached_ranges), req.path_qs)"},{"line_number":159,"context_line":"                    headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":160,"context_line":"                                    \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":161,"context_line":"                    shard_ranges \u003d self._filter_resp_shard_ranges("}],"source_content_type":"text/x-python","patch_set":5,"id":"69aadef7_aee652e8","line":158,"updated":"2020-11-25 20:27:14.000000000","message":"still love this message","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":155,"context_line":"                if cached_ranges is not None:"},{"line_number":156,"context_line":"                    # shard ranges can be returned from cache"},{"line_number":157,"context_line":"                    self.app.logger.debug(\u0027Found %d shards in cache for %s\u0027,"},{"line_number":158,"context_line":"                                          len(cached_ranges), req.path_qs)"},{"line_number":159,"context_line":"                    headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":160,"context_line":"                                    \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":161,"context_line":"                    shard_ranges \u003d self._filter_resp_shard_ranges("}],"source_content_type":"text/x-python","patch_set":5,"id":"81275bc1_55955736","line":158,"in_reply_to":"69aadef7_aee652e8","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":158,"context_line":"                                          len(cached_ranges), req.path_qs)"},{"line_number":159,"context_line":"                    headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":160,"context_line":"                                    \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":161,"context_line":"                    shard_ranges \u003d self._filter_resp_shard_ranges("},{"line_number":162,"context_line":"                        req, cached_ranges)"},{"line_number":163,"context_line":"                    # mimic GetOrHeadHandler.get_working_response..."},{"line_number":164,"context_line":"                    # note: server sets charset with content_type but proxy"}],"source_content_type":"text/x-python","patch_set":5,"id":"2f0b6a9b_cd54c807","line":161,"range":{"start_line":161,"start_character":20,"end_line":161,"end_character":32},"updated":"2020-12-13 23:14:13.000000000","message":"Note to self: not a list; they\u0027ve been serialized already.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":158,"context_line":"                                          len(cached_ranges), req.path_qs)"},{"line_number":159,"context_line":"                    headers.update({\u0027x-backend-record-type\u0027: \u0027shard\u0027,"},{"line_number":160,"context_line":"                                    \u0027x-backend-cached-results\u0027: \u0027true\u0027})"},{"line_number":161,"context_line":"                    shard_ranges \u003d self._filter_resp_shard_ranges("},{"line_number":162,"context_line":"                        req, cached_ranges)"},{"line_number":163,"context_line":"                    # mimic GetOrHeadHandler.get_working_response..."},{"line_number":164,"context_line":"                    # note: server sets charset with content_type but proxy"}],"source_content_type":"text/x-python","patch_set":5,"id":"cc5bbce9_ef737b25","line":161,"range":{"start_line":161,"start_character":20,"end_line":161,"end_character":32},"in_reply_to":"2f0b6a9b_cd54c807","updated":"2020-12-15 20:24:10.000000000","message":"hmm, I\u0027ll tweak the var name","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":162,"context_line":"                        req, cached_ranges)"},{"line_number":163,"context_line":"                    # mimic GetOrHeadHandler.get_working_response..."},{"line_number":164,"context_line":"                    # note: server sets charset with content_type but proxy"},{"line_number":165,"context_line":"                    # GETorHEAD_base does not, so don\u0027t set it here either"},{"line_number":166,"context_line":"                    resp \u003d Response(request\u003dreq, body\u003dshard_ranges)"},{"line_number":167,"context_line":"                    update_headers(resp, headers)"},{"line_number":168,"context_line":"                    resp.last_modified \u003d math.ceil("}],"source_content_type":"text/x-python","patch_set":5,"id":"92ce29be_20733bd9","line":165,"updated":"2020-12-13 23:14:13.000000000","message":"This kind of attention to detail is why I\u0027m not worried about this change at all 😊","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":162,"context_line":"                        req, cached_ranges)"},{"line_number":163,"context_line":"                    # mimic GetOrHeadHandler.get_working_response..."},{"line_number":164,"context_line":"                    # note: server sets charset with content_type but proxy"},{"line_number":165,"context_line":"                    # GETorHEAD_base does not, so don\u0027t set it here either"},{"line_number":166,"context_line":"                    resp \u003d Response(request\u003dreq, body\u003dshard_ranges)"},{"line_number":167,"context_line":"                    update_headers(resp, headers)"},{"line_number":168,"context_line":"                    resp.last_modified \u003d math.ceil("}],"source_content_type":"text/x-python","patch_set":5,"id":"3b3b3da5_ebf15e30","line":165,"in_reply_to":"92ce29be_20733bd9","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":171,"context_line":"                        \u0027x-timestamp\u0027)"},{"line_number":172,"context_line":"                    resp.accept_ranges \u003d \u0027bytes\u0027"},{"line_number":173,"context_line":"                    resp.content_type \u003d \u0027application/json\u0027"},{"line_number":174,"context_line":"                    return resp"},{"line_number":175,"context_line":""},{"line_number":176,"context_line":"        # The request was not fulfilled from cache so send to the backend"},{"line_number":177,"context_line":"        # server, but instruct the backend server to ignore name constraints in"}],"source_content_type":"text/x-python","patch_set":5,"id":"9905f2ec_7b02a275","line":174,"updated":"2020-11-25 20:27:14.000000000","message":"if we\u0027re going to return a cached response it\u0027s right here","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":171,"context_line":"                        \u0027x-timestamp\u0027)"},{"line_number":172,"context_line":"                    resp.accept_ranges \u003d \u0027bytes\u0027"},{"line_number":173,"context_line":"                    resp.content_type \u003d \u0027application/json\u0027"},{"line_number":174,"context_line":"                    return resp"},{"line_number":175,"context_line":""},{"line_number":176,"context_line":"        # The request was not fulfilled from cache so send to the backend"},{"line_number":177,"context_line":"        # server, but instruct the backend server to ignore name constraints in"}],"source_content_type":"text/x-python","patch_set":5,"id":"5faab580_bc8a115d","line":174,"in_reply_to":"9905f2ec_7b02a275","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":191,"context_line":"            \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":192,"context_line":"        resp_record_type \u003d resp.headers.get("},{"line_number":193,"context_line":"            \u0027x-backend-record-type\u0027, \u0027\u0027).lower()"},{"line_number":194,"context_line":"        complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":195,"context_line":"            \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":196,"context_line":"        # given that we sent \u0027x-backend-override-shard-name-filter\u003dsharded\u0027 we"},{"line_number":197,"context_line":"        # should only receive back \u0027x-backend-override-shard-name-filter\u003dtrue\u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"e8d0dd31_3790e5c2","line":194,"range":{"start_line":194,"start_character":8,"end_line":194,"end_character":24},"updated":"2020-12-13 23:14:13.000000000","message":"What if this replica has missing ranges? It might override the filters but *still* not return a set that covers the namespace...","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":true,"context_lines":[{"line_number":191,"context_line":"            \u0027x-backend-sharding-state\u0027, \u0027\u0027).lower()"},{"line_number":192,"context_line":"        resp_record_type \u003d resp.headers.get("},{"line_number":193,"context_line":"            \u0027x-backend-record-type\u0027, \u0027\u0027).lower()"},{"line_number":194,"context_line":"        complete_listing \u003d config_true_value(resp.headers.pop("},{"line_number":195,"context_line":"            \u0027x-backend-override-shard-name-filter\u0027, False))"},{"line_number":196,"context_line":"        # given that we sent \u0027x-backend-override-shard-name-filter\u003dsharded\u0027 we"},{"line_number":197,"context_line":"        # should only receive back \u0027x-backend-override-shard-name-filter\u003dtrue\u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"8c0bd2e5_3a889f2f","line":194,"range":{"start_line":194,"start_character":8,"end_line":194,"end_character":24},"in_reply_to":"e8d0dd31_3790e5c2","updated":"2020-12-15 20:24:10.000000000","message":"yeah,, I\u0027m beginning to feel like we should do a check on the ranges before caching so that we don\u0027t sit on bad data for 600secs","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":209,"context_line":"                # unexpected case: error parsing shard ranges; we have to"},{"line_number":210,"context_line":"                # return something, but backend response body has been"},{"line_number":211,"context_line":"                # consumed, so return empty list without caching it"},{"line_number":212,"context_line":"                cached_ranges \u003d []"},{"line_number":213,"context_line":"            else:"},{"line_number":214,"context_line":"                cached_ranges \u003d [dict(sr) for sr in backend_shard_ranges]"},{"line_number":215,"context_line":"                if resp.headers.get(\u0027x-backend-sharding-state\u0027) \u003d\u003d \u0027sharded\u0027:"}],"source_content_type":"text/x-python","patch_set":5,"id":"9570face_7633624d","line":212,"updated":"2020-12-13 23:14:13.000000000","message":"So wait, who\u0027s going to wind up looking at this body? What are they going to do when it\u0027s empty?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":209,"context_line":"                # unexpected case: error parsing shard ranges; we have to"},{"line_number":210,"context_line":"                # return something, but backend response body has been"},{"line_number":211,"context_line":"                # consumed, so return empty list without caching it"},{"line_number":212,"context_line":"                cached_ranges \u003d []"},{"line_number":213,"context_line":"            else:"},{"line_number":214,"context_line":"                cached_ranges \u003d [dict(sr) for sr in backend_shard_ranges]"},{"line_number":215,"context_line":"                if resp.headers.get(\u0027x-backend-sharding-state\u0027) \u003d\u003d \u0027sharded\u0027:"}],"source_content_type":"text/x-python","patch_set":5,"id":"7d7b1a30_6e638de3","line":212,"in_reply_to":"9570face_7633624d","updated":"2020-12-15 20:24:10.000000000","message":"really wasn\u0027t sure how to handle this case - the non-cache path will blow up if the parsed shard ranges is None - maybe I should have just left the response body empty here rather than pretending it was valid-but-empty-list??\n\nohh, but the response body has not been consumed, so I can just return resp","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":229,"context_line":"            # filter returned shard ranges according to request constraints"},{"line_number":230,"context_line":"            resp.body \u003d self._filter_resp_shard_ranges(req, cached_ranges)"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        return resp"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"    def GETorHEAD(self, req):"},{"line_number":235,"context_line":"        \"\"\"Handler for HTTP GET/HEAD requests.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"03f0378c_1a0ed3a9","line":232,"updated":"2020-11-25 20:27:14.000000000","message":"the second and only other return from this method, either shard resp from the backend (filtered and equivalent resp synthesized from cached value above) - OR this could be an object listing","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":229,"context_line":"            # filter returned shard ranges according to request constraints"},{"line_number":230,"context_line":"            resp.body \u003d self._filter_resp_shard_ranges(req, cached_ranges)"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        return resp"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"    def GETorHEAD(self, req):"},{"line_number":235,"context_line":"        \"\"\"Handler for HTTP GET/HEAD requests.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"10f8011d_018e048c","line":232,"in_reply_to":"03f0378c_1a0ed3a9","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":249,"context_line":""},{"line_number":250,"context_line":"        # The read-modify-write of params here is because the Request.params"},{"line_number":251,"context_line":"        # getter dynamically generates a dict of params from the query string;"},{"line_number":252,"context_line":"        # the setter must be called for new params to update the query string."},{"line_number":253,"context_line":"        params \u003d req.params"},{"line_number":254,"context_line":"        params[\u0027format\u0027] \u003d \u0027json\u0027"},{"line_number":255,"context_line":"        # x-backend-record-type may be sent via internal client e.g. from"}],"source_content_type":"text/x-python","patch_set":5,"id":"4eaa9bab_cecbd549","line":252,"updated":"2020-11-25 20:27:14.000000000","message":"aww, thanks","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":249,"context_line":""},{"line_number":250,"context_line":"        # The read-modify-write of params here is because the Request.params"},{"line_number":251,"context_line":"        # getter dynamically generates a dict of params from the query string;"},{"line_number":252,"context_line":"        # the setter must be called for new params to update the query string."},{"line_number":253,"context_line":"        params \u003d req.params"},{"line_number":254,"context_line":"        params[\u0027format\u0027] \u003d \u0027json\u0027"},{"line_number":255,"context_line":"        # x-backend-record-type may be sent via internal client e.g. from"}],"source_content_type":"text/x-python","patch_set":5,"id":"64957f22_60d3f52b","line":252,"in_reply_to":"4eaa9bab_cecbd549","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":268,"context_line":"                memcache and"},{"line_number":269,"context_line":"                get_param(req, \u0027states\u0027) \u003d\u003d \u0027listing\u0027 and not"},{"line_number":270,"context_line":"                config_true_value("},{"line_number":271,"context_line":"                    req.headers.get(\u0027x-backend-include-deleted\u0027, False))):"},{"line_number":272,"context_line":"            # This GET might be served from cache or might populate cache."},{"line_number":273,"context_line":"            # \u0027x-backend-include-deleted\u0027 is not usually expected in requests"},{"line_number":274,"context_line":"            # to the proxy (it is used from sharder to container servers) but"}],"source_content_type":"text/x-python","patch_set":5,"id":"9aec8ff0_7fbda5a6","line":271,"updated":"2020-11-25 20:27:14.000000000","message":"this really isn\u0027t too bad - the last one gets me tno:\n\nand not true deleted false","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":268,"context_line":"                memcache and"},{"line_number":269,"context_line":"                get_param(req, \u0027states\u0027) \u003d\u003d \u0027listing\u0027 and not"},{"line_number":270,"context_line":"                config_true_value("},{"line_number":271,"context_line":"                    req.headers.get(\u0027x-backend-include-deleted\u0027, False))):"},{"line_number":272,"context_line":"            # This GET might be served from cache or might populate cache."},{"line_number":273,"context_line":"            # \u0027x-backend-include-deleted\u0027 is not usually expected in requests"},{"line_number":274,"context_line":"            # to the proxy (it is used from sharder to container servers) but"}],"source_content_type":"text/x-python","patch_set":5,"id":"408eaa6d_5b6c10bb","line":271,"in_reply_to":"755de997_70953b31","updated":"2020-12-15 20:24:10.000000000","message":"do we need config_false_value() ? I\u0027ll move the \u0027not\u0027 _ IIRC pep8 has a thing about the logical operators being at end of line but I think not is free to move, or I am remembering some other style guide, not pep8.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":268,"context_line":"                memcache and"},{"line_number":269,"context_line":"                get_param(req, \u0027states\u0027) \u003d\u003d \u0027listing\u0027 and not"},{"line_number":270,"context_line":"                config_true_value("},{"line_number":271,"context_line":"                    req.headers.get(\u0027x-backend-include-deleted\u0027, False))):"},{"line_number":272,"context_line":"            # This GET might be served from cache or might populate cache."},{"line_number":273,"context_line":"            # \u0027x-backend-include-deleted\u0027 is not usually expected in requests"},{"line_number":274,"context_line":"            # to the proxy (it is used from sharder to container servers) but"}],"source_content_type":"text/x-python","patch_set":5,"id":"755de997_70953b31","line":271,"in_reply_to":"9aec8ff0_7fbda5a6","updated":"2020-12-13 23:14:13.000000000","message":"Yeah, I might at least move the \"not\" to be on the same line as \"config_true_value(\" -- I *think* that might help make it a little easier to read?","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":374,"context_line":"                    pass"},{"line_number":375,"context_line":"                else:"},{"line_number":376,"context_line":"                    if just_past \u003c shard_range:"},{"line_number":377,"context_line":"                        continue"},{"line_number":378,"context_line":""},{"line_number":379,"context_line":"            self.app.logger.debug("},{"line_number":380,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"81ab175a_009bdba6","line":377,"updated":"2020-11-25 20:27:14.000000000","message":"I had a prefix, so the requests makes sense the requests skipped ahead:\n\n\tNov 25 18:19:27 saio proxy-server: GET listing from 6 shards for: /v1/AUTH_test/test?states\u003dlisting\u0026prefix\u003dobj269\u0026format\u003djson (txn: tx80459d75c7324fd4ab511-005fbea02f) (client_ip: 127.0.0.1)\n\tNov 25 18:19:27 saio proxy-server: Getting listing part 5 from shard ShardRange\u003c\u0027obj249\u0027 to MaxBound as of 1606325494.67246, (150, 0) as of 1606326413.66229, active as of 1606325494.67246\u003e .shards_AUTH_test/test-098f6bcd4621d373cade4e832627b4f6-1606325494.67246-5 with {} (txn: tx80459d75c7324fd4ab511-005fbea02f) (client_ip: 127.0.0.1)","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":374,"context_line":"                    pass"},{"line_number":375,"context_line":"                else:"},{"line_number":376,"context_line":"                    if just_past \u003c shard_range:"},{"line_number":377,"context_line":"                        continue"},{"line_number":378,"context_line":""},{"line_number":379,"context_line":"            self.app.logger.debug("},{"line_number":380,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"8e0bfcef_e900305d","line":377,"in_reply_to":"81ab175a_009bdba6","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":378,"context_line":""},{"line_number":379,"context_line":"            self.app.logger.debug("},{"line_number":380,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"},{"line_number":381,"context_line":"                i, shard_range, shard_range.name, headers)"},{"line_number":382,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":383,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":384,"context_line":"                headers\u003dheaders, params\u003dparams)"}],"source_content_type":"text/x-python","patch_set":5,"id":"88b6bb90_ac8b7e80","line":381,"updated":"2020-11-25 20:27:14.000000000","message":"this looks great!  You can even see the sub-shard recursion:\n\n\tNov 25 18:19:27 saio proxy-server: Getting listing part 5 from shard ShardRange\u003c\u0027obj249\u0027 to MaxBound as of 1606325494.67246, (150, 0) as of 1606326413.66229, active as of 1606325494.67246\u003e .shards_AUTH_test/test-098f6bcd4621d373cade4e832627b4f6-1606325494.67246-5 with {} (txn: tx80459d75c7324fd4ab511-005fbea02f) (client_ip: 127.0.0.1)\n\tNov 25 18:19:27 saio proxy-server: Get listing from /v1/.shards_AUTH_test/test-098f6bcd4621d373cade4e832627b4f6-1606325494.67246-5?marker\u003d\u0026prefix\u003dobj269\u0026limit\u003d10000\u0026end_marker\u003d\u0026format\u003djson {} (txn: tx80459d75c7324fd4ab511-005fbea02f) (client_ip: 127.0.0.1)\n\tNov 25 18:19:27 saio proxy-server: Found 3 shards in cache for /v1/.shards_AUTH_test/test-098f6bcd4621d373cade4e832627b4f6-1606325494.67246-5?end_marker\u003d\u0026format\u003djson\u0026states\u003dlisting\u0026prefix\u003dobj269\u0026limit\u003d10000\u0026marker\u003d (txn: tx80459d75c7324fd4ab511-005fbea02f)\n\tNov 25 18:19:27 saio proxy-server: GET listing from 3 shards for: /v1/.shards_AUTH_test/test-098f6bcd4621d373cade4e832627b4f6-1606325494.67246-5?end_marker\u003d\u0026format\u003djson\u0026states\u003dlisting\u0026prefix\u003dobj269\u0026limit\u003d10000\u0026marker\u003d (txn: tx80459d75c7324fd4ab511-005fbea02f)","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":378,"context_line":""},{"line_number":379,"context_line":"            self.app.logger.debug("},{"line_number":380,"context_line":"                \u0027Getting listing part %d from shard %s %s with %s\u0027,"},{"line_number":381,"context_line":"                i, shard_range, shard_range.name, headers)"},{"line_number":382,"context_line":"            objs, shard_resp \u003d self._get_container_listing("},{"line_number":383,"context_line":"                req, shard_range.account, shard_range.container,"},{"line_number":384,"context_line":"                headers\u003dheaders, params\u003dparams)"}],"source_content_type":"text/x-python","patch_set":5,"id":"369f2c35_04a606a4","line":381,"in_reply_to":"88b6bb90_ac8b7e80","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":399,"context_line":""},{"line_number":400,"context_line":"            self.app.logger.debug("},{"line_number":401,"context_line":"                \u0027Found %d objects in shard (state\u003d%s), total \u003d %d\u0027,"},{"line_number":402,"context_line":"                len(objs), shard_state, len(objs) + len(objects))"},{"line_number":403,"context_line":""},{"line_number":404,"context_line":"            if not objs:"},{"line_number":405,"context_line":"                # tolerate empty shard containers"}],"source_content_type":"text/x-python","patch_set":5,"id":"fbdc8fdd_9dcd2fab","line":402,"updated":"2020-11-25 20:27:14.000000000","message":"i like this message too!\n\nit\u0027s interesting when there\u0027s cached-sub-shards\n\n\tNov 25 18:19:27 saio proxy-server: Found 20 objects in shard (state\u003dunsharded), total \u003d 101 (txn: tx80459d75c7324fd4ab511-005fbea02f)\n\tNov 25 18:19:27 saio proxy-server: Found 101 objects in shard (state\u003d(70, \u0027sharded\u0027)), total \u003d 101 (txn: tx80459d75c7324fd4ab511-005fbea02f)","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":399,"context_line":""},{"line_number":400,"context_line":"            self.app.logger.debug("},{"line_number":401,"context_line":"                \u0027Found %d objects in shard (state\u003d%s), total \u003d %d\u0027,"},{"line_number":402,"context_line":"                len(objs), shard_state, len(objs) + len(objects))"},{"line_number":403,"context_line":""},{"line_number":404,"context_line":"            if not objs:"},{"line_number":405,"context_line":"                # tolerate empty shard containers"}],"source_content_type":"text/x-python","patch_set":5,"id":"831cc8e0_5dd48dad","line":402,"in_reply_to":"fbdc8fdd_9dcd2fab","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4c8b4596f446d0d1e30e58c0512064a8a70dad6d","unresolved":true,"context_lines":[{"line_number":139,"context_line":"                                         self.account_name,"},{"line_number":140,"context_line":"                                         self.container_name)"},{"line_number":141,"context_line":"        if (info and is_success(info[\u0027status\u0027]) and"},{"line_number":142,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":143,"context_line":"            # container is sharded so we may have the shard ranges cached"},{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"}],"source_content_type":"text/x-python","patch_set":9,"id":"387866ac_34bc7476","line":142,"updated":"2021-01-07 03:30:58.000000000","message":"does this mean if the info isn\u0027t cached or has fallen out of the cache we alwasy to get the shard list from the backend?\n\nI guess that makes sense, if we\u0027re relying on the info for current shard state. Just confirming my understanding.\n\nUPDATE: Seems to be confirmed in tests, nice :)","commit_id":"077ba77ea6f735c98709b60e1fc9aa9f0f90676a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf1db11b8a4f5e0daf871821d5d3d3c62f7f45ea","unresolved":true,"context_lines":[{"line_number":139,"context_line":"                                         self.account_name,"},{"line_number":140,"context_line":"                                         self.container_name)"},{"line_number":141,"context_line":"        if (info and is_success(info[\u0027status\u0027]) and"},{"line_number":142,"context_line":"                info.get(\u0027sharding_state\u0027) \u003d\u003d \u0027sharded\u0027):"},{"line_number":143,"context_line":"            # container is sharded so we may have the shard ranges cached"},{"line_number":144,"context_line":"            headers \u003d headers_from_container_info(info)"},{"line_number":145,"context_line":"            if headers:"}],"source_content_type":"text/x-python","patch_set":9,"id":"910f59ac_e186dd63","line":142,"in_reply_to":"387866ac_34bc7476","updated":"2021-01-07 09:15:48.000000000","message":"yes, we need this guard so that we don\u0027t just take shard ranges from cache, build a listing and return it when the container may actually have been deleted. Also, we don\u0027t want to even try looking in cache unless the container db is sharded.","commit_id":"077ba77ea6f735c98709b60e1fc9aa9f0f90676a"}],"test/unit/__init__.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":397,"context_line":""},{"line_number":398,"context_line":"    def __init__(self):"},{"line_number":399,"context_line":"        self.store \u003d {}"},{"line_number":400,"context_line":"        self.calls \u003d []"},{"line_number":401,"context_line":""},{"line_number":402,"context_line":"    def clear_calls(self):"},{"line_number":403,"context_line":"        self.calls \u003d []"}],"source_content_type":"text/x-python","patch_set":5,"id":"12c9e9c9_e1f37492","line":400,"updated":"2020-11-25 20:27:14.000000000","message":"gettin\u0027 fancy now","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":397,"context_line":""},{"line_number":398,"context_line":"    def __init__(self):"},{"line_number":399,"context_line":"        self.store \u003d {}"},{"line_number":400,"context_line":"        self.calls \u003d []"},{"line_number":401,"context_line":""},{"line_number":402,"context_line":"    def clear_calls(self):"},{"line_number":403,"context_line":"        self.calls \u003d []"}],"source_content_type":"text/x-python","patch_set":5,"id":"54122905_7ac7ad32","line":400,"in_reply_to":"12c9e9c9_e1f37492","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":400,"context_line":"        self.calls \u003d []"},{"line_number":401,"context_line":""},{"line_number":402,"context_line":"    def clear_calls(self):"},{"line_number":403,"context_line":"        self.calls \u003d []"},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"    def _called(self, method, key\u003dNone, value\u003dNone, time\u003dNone):"},{"line_number":406,"context_line":"        self.calls.append((method, key, value, time))"}],"source_content_type":"text/x-python","patch_set":5,"id":"03122540_1e712b52","line":403,"updated":"2020-12-13 23:14:13.000000000","message":"Do we care that this changes the reference? I guess not. But we might want\n\n self.calls[:] \u003d []\n\ninstead, just in case.","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":400,"context_line":"        self.calls \u003d []"},{"line_number":401,"context_line":""},{"line_number":402,"context_line":"    def clear_calls(self):"},{"line_number":403,"context_line":"        self.calls \u003d []"},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"    def _called(self, method, key\u003dNone, value\u003dNone, time\u003dNone):"},{"line_number":406,"context_line":"        self.calls.append((method, key, value, time))"}],"source_content_type":"text/x-python","patch_set":5,"id":"3532e885_d3433b8d","line":403,"in_reply_to":"03122540_1e712b52","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":403,"context_line":"        self.calls \u003d []"},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"    def _called(self, method, key\u003dNone, value\u003dNone, time\u003dNone):"},{"line_number":406,"context_line":"        self.calls.append((method, key, value, time))"},{"line_number":407,"context_line":""},{"line_number":408,"context_line":"    def get(self, key):"},{"line_number":409,"context_line":"        self._called(\u0027get\u0027, key)"}],"source_content_type":"text/x-python","patch_set":5,"id":"e3c43819_725b182e","line":406,"updated":"2020-12-13 23:14:13.000000000","message":"Heh -- I\u0027ve often found myself doing a similar thing with the *real* memcache client (only logging instead of appending to a list).","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":403,"context_line":"        self.calls \u003d []"},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"    def _called(self, method, key\u003dNone, value\u003dNone, time\u003dNone):"},{"line_number":406,"context_line":"        self.calls.append((method, key, value, time))"},{"line_number":407,"context_line":""},{"line_number":408,"context_line":"    def get(self, key):"},{"line_number":409,"context_line":"        self._called(\u0027get\u0027, key)"}],"source_content_type":"text/x-python","patch_set":5,"id":"bbb18e0c_506a9fcc","line":406,"in_reply_to":"e3c43819_725b182e","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":436,"context_line":"        return True"},{"line_number":437,"context_line":""},{"line_number":438,"context_line":"    def delete_all(self):"},{"line_number":439,"context_line":"        self.store \u003d {}"},{"line_number":440,"context_line":""},{"line_number":441,"context_line":""},{"line_number":442,"context_line":"def readuntil2crlfs(fd):"}],"source_content_type":"text/x-python","patch_set":5,"id":"e9365aa9_7419f485","line":439,"updated":"2020-12-13 23:14:13.000000000","message":"...and this one could be\n\n self.store.clear()\n\n(In fact, I\u0027m somewhat surprised if we didn\u0027t already have some tests that did that...)","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":436,"context_line":"        return True"},{"line_number":437,"context_line":""},{"line_number":438,"context_line":"    def delete_all(self):"},{"line_number":439,"context_line":"        self.store \u003d {}"},{"line_number":440,"context_line":""},{"line_number":441,"context_line":""},{"line_number":442,"context_line":"def readuntil2crlfs(fd):"}],"source_content_type":"text/x-python","patch_set":5,"id":"ed0e0ff9_da39af14","line":439,"in_reply_to":"e9365aa9_7419f485","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"test/unit/common/test_request_helpers.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":104,"context_line":"        req \u003d Request.blank(\u0027\u0027, query_string\u003d\u0027limit\u003d10001\u0027)"},{"line_number":105,"context_line":"        with self.assertRaises(HTTPException) as raised:"},{"line_number":106,"context_line":"            rh.validate_container_params(req)"},{"line_number":107,"context_line":"        self.assertEqual(raised.exception.status_int, 412)"},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"    def test_is_user_meta(self):"},{"line_number":110,"context_line":"        m_type \u003d \u0027meta\u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"20b5bbe4_8df1402b","line":107,"updated":"2020-11-25 20:27:14.000000000","message":"superb coverage!!!","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":104,"context_line":"        req \u003d Request.blank(\u0027\u0027, query_string\u003d\u0027limit\u003d10001\u0027)"},{"line_number":105,"context_line":"        with self.assertRaises(HTTPException) as raised:"},{"line_number":106,"context_line":"            rh.validate_container_params(req)"},{"line_number":107,"context_line":"        self.assertEqual(raised.exception.status_int, 412)"},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"    def test_is_user_meta(self):"},{"line_number":110,"context_line":"        m_type \u003d \u0027meta\u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"8b55bf2d_114b748c","line":107,"in_reply_to":"20b5bbe4_8df1402b","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"test/unit/container/test_server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6e8d41962ce758b293045a60b8869f7b7d437fa6","unresolved":true,"context_lines":[{"line_number":2717,"context_line":"        def check_shard_GET_override_filter("},{"line_number":2718,"context_line":"                expected_shard_ranges, path, params\u003d\u0027\u0027):"},{"line_number":2719,"context_line":"            req_headers \u003d {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":2720,"context_line":"                           \u0027X-Backend-Override-Shard-Name-Filter\u0027: \u0027sharded\u0027}"},{"line_number":2721,"context_line":"            req \u003d Request.blank(\u0027/sda1/p/%s?format\u003djson%s\u0027 %"},{"line_number":2722,"context_line":"                                (path, params), method\u003d\u0027GET\u0027,"},{"line_number":2723,"context_line":"                                headers\u003dreq_headers)"}],"source_content_type":"text/x-python","patch_set":5,"id":"2f192a20_b4115509","line":2720,"range":{"start_line":2720,"start_character":68,"end_line":2720,"end_character":75},"updated":"2020-12-13 23:14:13.000000000","message":"So we only ever *test* sending \"sharded\"...","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":2717,"context_line":"        def check_shard_GET_override_filter("},{"line_number":2718,"context_line":"                expected_shard_ranges, path, params\u003d\u0027\u0027):"},{"line_number":2719,"context_line":"            req_headers \u003d {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":2720,"context_line":"                           \u0027X-Backend-Override-Shard-Name-Filter\u0027: \u0027sharded\u0027}"},{"line_number":2721,"context_line":"            req \u003d Request.blank(\u0027/sda1/p/%s?format\u003djson%s\u0027 %"},{"line_number":2722,"context_line":"                                (path, params), method\u003d\u0027GET\u0027,"},{"line_number":2723,"context_line":"                                headers\u003dreq_headers)"}],"source_content_type":"text/x-python","patch_set":5,"id":"a9737106_9144ec09","line":2720,"range":{"start_line":2720,"start_character":68,"end_line":2720,"end_character":75},"in_reply_to":"2f192a20_b4115509","updated":"2020-12-15 20:24:10.000000000","message":"will fix","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"}],"test/unit/proxy/controllers/test_base.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":790,"context_line":"            \u0027X-Container-Meta-Access-Control-Max-Age\u0027: \u0027my-cors-age\u0027,"},{"line_number":791,"context_line":"        }"},{"line_number":792,"context_line":""},{"line_number":793,"context_line":"        self.assertEqual(expected, res)"},{"line_number":794,"context_line":""},{"line_number":795,"context_line":"        for required in ("},{"line_number":796,"context_line":"                \u0027created_at\u0027, \u0027put_timestamp\u0027, \u0027delete_timestamp\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_79073f1a","line":793,"updated":"2020-11-17 23:28:09.000000000","message":"nice!  i was hoping for a test like this","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"}],"test/unit/proxy/controllers/test_container.py":[{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1528,"context_line":"                self.assertEqual(\u0027a/c\u0027, backend_req[\u0027path\u0027][7:])"},{"line_number":1529,"context_line":"                self.assertEqual(req.headers[\u0027X-Trans-Id\u0027],"},{"line_number":1530,"context_line":"                                 backend_req[\u0027headers\u0027][\u0027X-Trans-Id\u0027])"},{"line_number":1531,"context_line":"                self.assertTrue(backend_req[\u0027headers\u0027][\u0027User-Agent\u0027].startswith("},{"line_number":1532,"context_line":"                    \u0027proxy-server\u0027))"},{"line_number":1533,"context_line":"            return backend_req, resp"},{"line_number":1534,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_30f443cf","line":1531,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E501 line too long (80 \u003e 79 characters)","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1558,"context_line":"            self.assertNotIn(\u0027X-Backend-Cached-Results\u0027, resp.headers)"},{"line_number":1559,"context_line":"            self.assertEqual("},{"line_number":1560,"context_line":"                [(\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":1561,"context_line":"                 memcache.calls)"},{"line_number":1562,"context_line":"            self.assertEqual(\u0027unsharded\u0027,"},{"line_number":1563,"context_line":"                             memcache.calls[0][2][\u0027sharding_state\u0027])"},{"line_number":1564,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_900397b8","line":1561,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E131 continuation line unaligned for hanging indent","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1582,"context_line":"            self.assertEqual("},{"line_number":1583,"context_line":"                [(\u0027get\u0027, \u0027container/a/c\u0027, None, None),"},{"line_number":1584,"context_line":"                 (\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":1585,"context_line":"                 memcache.calls)"},{"line_number":1586,"context_line":"            self.assertEqual(\u0027unsharded\u0027,"},{"line_number":1587,"context_line":"                             memcache.calls[1][2][\u0027sharding_state\u0027])"},{"line_number":1588,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_70febbae","line":1585,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E131 continuation line unaligned for hanging indent","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1606,"context_line":"            self.assertEqual("},{"line_number":1607,"context_line":"                [(\u0027get\u0027, \u0027container/a/c\u0027, None, None),"},{"line_number":1608,"context_line":"                 (\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":1609,"context_line":"                 memcache.calls)"},{"line_number":1610,"context_line":"            self.assertEqual(sharding_state,"},{"line_number":1611,"context_line":"                             memcache.calls[1][2][\u0027sharding_state\u0027])"},{"line_number":1612,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_d02eaf41","line":1609,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E131 continuation line unaligned for hanging indent","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1633,"context_line":"                 (\u0027get\u0027, \u0027shard-listing/a/c\u0027, None, None),"},{"line_number":1634,"context_line":"                 (\u0027set\u0027, \u0027shard-listing/a/c\u0027, sr_dicts, exp_recheck_listing),"},{"line_number":1635,"context_line":"                 (\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":1636,"context_line":"                 memcache.calls)"},{"line_number":1637,"context_line":"            self.assertEqual(sharding_state,"},{"line_number":1638,"context_line":"                             memcache.calls[3][2][\u0027sharding_state\u0027])"},{"line_number":1639,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_b03173a5","line":1636,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E131 continuation line unaligned for hanging indent","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1673,"context_line":"            self.assertNotIn(\u0027X-Backend-Cached-Results\u0027, resp.headers)"},{"line_number":1674,"context_line":"            self.assertEqual("},{"line_number":1675,"context_line":"                [(\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":1676,"context_line":"                 memcache.calls)"},{"line_number":1677,"context_line":"            self.assertEqual(sharding_state,"},{"line_number":1678,"context_line":"                             memcache.calls[0][2][\u0027sharding_state\u0027])"},{"line_number":1679,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_10294737","line":1676,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E131 continuation line unaligned for hanging indent","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1755,"context_line":"                 (\u0027get\u0027, \u0027shard-listing/a/c\u0027, None, None),"},{"line_number":1756,"context_line":"                 (\u0027set\u0027, \u0027shard-listing/a/c\u0027, sr_dicts, exp_recheck_listing),"},{"line_number":1757,"context_line":"                 (\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":1758,"context_line":"                 memcache.calls)"},{"line_number":1759,"context_line":"            self.assertEqual(sharding_state,"},{"line_number":1760,"context_line":"                             memcache.calls[3][2][\u0027sharding_state\u0027])"},{"line_number":1761,"context_line":"            self.assertEqual(\u0027jam\u0027, req.params[\u0027marker\u0027])"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_f02beb2c","line":1758,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E131 continuation line unaligned for hanging indent","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"33a40107b5c3d8b6b9ff48c4823f7d79627f8f37","unresolved":false,"context_lines":[{"line_number":1777,"context_line":"                [(\u0027get\u0027, \u0027container/a/c\u0027, None, None),"},{"line_number":1778,"context_line":"                 (\u0027get\u0027, \u0027shard-listing/a/c\u0027, None, None),"},{"line_number":1779,"context_line":"                 (\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 6.0)],"},{"line_number":1780,"context_line":"                 memcache.calls)"},{"line_number":1781,"context_line":"            self.assertEqual(404, memcache.calls[2][2][\u0027status\u0027])"},{"line_number":1782,"context_line":"            self.assertEqual(b\u0027\u0027, resp.body)"},{"line_number":1783,"context_line":"            self.assertEqual(404, resp.status_int)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1f621f24_50229f53","line":1780,"updated":"2020-11-05 21:57:26.000000000","message":"pep8: E131 continuation line unaligned for hanging indent","commit_id":"6d8bf951082e5472349e1509de04cc59d8273b62"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":1529,"context_line":"            # returned shards"},{"line_number":1530,"context_line":"            container_path \u003d \u0027/v1/a/c\u0027 + query_string"},{"line_number":1531,"context_line":"            request \u003d Request.blank(container_path, headers\u003dheaders)"},{"line_number":1532,"context_line":"            request.environ[\u0027swift.cache\u0027] \u003d memcache"},{"line_number":1533,"context_line":"            request.environ[\u0027swift.infocache\u0027] \u003d infocache"},{"line_number":1534,"context_line":"            return request"},{"line_number":1535,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_53f6a4fb","line":1532,"updated":"2020-11-17 23:28:09.000000000","message":"mostly these closures seem to be dealing with wrapping over \"memcache\" - maybe better as a self._build_request helper?","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":1529,"context_line":"            # returned shards"},{"line_number":1530,"context_line":"            container_path \u003d \u0027/v1/a/c\u0027 + query_string"},{"line_number":1531,"context_line":"            request \u003d Request.blank(container_path, headers\u003dheaders)"},{"line_number":1532,"context_line":"            request.environ[\u0027swift.cache\u0027] \u003d memcache"},{"line_number":1533,"context_line":"            request.environ[\u0027swift.infocache\u0027] \u003d infocache"},{"line_number":1534,"context_line":"            return request"},{"line_number":1535,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_aff5ee85","line":1532,"in_reply_to":"1f621f24_53f6a4fb","updated":"2020-11-19 17:12:01.000000000","message":"yes, agree, this mega-test could be improved!","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":1542,"context_line":"            with mocked_http_conn("},{"line_number":1543,"context_line":"                    *resp_status, body_iter\u003d[resp_body] * num_resp,"},{"line_number":1544,"context_line":"                    headers\u003d[resp_hdrs] * num_resp) as fake_conn:"},{"line_number":1545,"context_line":"                resp \u003d req.get_response(self.app)"},{"line_number":1546,"context_line":"            self.assertEqual(resp_status[0], resp.status_int)"},{"line_number":1547,"context_line":"            self.assertEqual(num_resp, len(fake_conn.requests))"},{"line_number":1548,"context_line":"            return fake_conn.requests[0], resp"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_332fa8a8","line":1545,"updated":"2020-11-17 23:28:09.000000000","message":"this makes it look like req is an incoming unmodified client request?","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":1542,"context_line":"            with mocked_http_conn("},{"line_number":1543,"context_line":"                    *resp_status, body_iter\u003d[resp_body] * num_resp,"},{"line_number":1544,"context_line":"                    headers\u003d[resp_hdrs] * num_resp) as fake_conn:"},{"line_number":1545,"context_line":"                resp \u003d req.get_response(self.app)"},{"line_number":1546,"context_line":"            self.assertEqual(resp_status[0], resp.status_int)"},{"line_number":1547,"context_line":"            self.assertEqual(num_resp, len(fake_conn.requests))"},{"line_number":1548,"context_line":"            return fake_conn.requests[0], resp"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_8fff4a66","line":1545,"in_reply_to":"1f621f24_332fa8a8","updated":"2020-11-19 17:12:01.000000000","message":"I think so - have I got it wrong?","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":1609,"context_line":"            # cached because sharding state is not \u0027sharded\u0027; check that"},{"line_number":1610,"context_line":"            # marker, end_marker etc are passed to backend"},{"line_number":1611,"context_line":"            memcache.delete_all()"},{"line_number":1612,"context_line":"            memcache.clear_calls()"},{"line_number":1613,"context_line":"            req \u003d make_request({\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027},"},{"line_number":1614,"context_line":"                               \u0027?states\u003dlisting;marker\u003degg;end_marker\u003djam\u0027, {})"},{"line_number":1615,"context_line":"            resp_shards \u003d sr_dicts[:2]"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_736e40ca","line":1612,"updated":"2020-11-17 23:28:09.000000000","message":"to me, this is a smell - it means we have a test doing more than one thing.\n\nI like tests to \"just do one thing\" because when you have a test that does N things and it fails on step \"N - 1\" there\u0027s a whole lot of commenting out stuff to get to the failure you\u0027re interested in debugging.","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":1609,"context_line":"            # cached because sharding state is not \u0027sharded\u0027; check that"},{"line_number":1610,"context_line":"            # marker, end_marker etc are passed to backend"},{"line_number":1611,"context_line":"            memcache.delete_all()"},{"line_number":1612,"context_line":"            memcache.clear_calls()"},{"line_number":1613,"context_line":"            req \u003d make_request({\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027},"},{"line_number":1614,"context_line":"                               \u0027?states\u003dlisting;marker\u003degg;end_marker\u003djam\u0027, {})"},{"line_number":1615,"context_line":"            resp_shards \u003d sr_dicts[:2]"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_8f346a4c","line":1612,"in_reply_to":"1f621f24_736e40ca","updated":"2020-11-19 17:12:01.000000000","message":"fair comment - the test kind of grew incrementally until it became a monster :)","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":1653,"context_line":"                \u0027?states\u003dlisting;marker\u003degg;end_marker\u003djam;reverse\u003dtrue\u0027, {})"},{"line_number":1654,"context_line":"            resp_shards \u003d sr_dicts[:2]"},{"line_number":1655,"context_line":"            resp_shards.reverse()"},{"line_number":1656,"context_line":"            backend_req, resp \u003d capture_backend_request("},{"line_number":1657,"context_line":"                req, 200, json.dumps(resp_shards).encode(\u0027ascii\u0027),"},{"line_number":1658,"context_line":"                {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":1659,"context_line":"                 \u0027X-Backend-Sharding-State\u0027: sharding_state})"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_f37ed0ad","line":1656,"updated":"2020-11-17 23:28:09.000000000","message":"make a docstring on capture_backend_request would make this more clear - is the req param a *client* request?\n\nbecause in this case it has x-backend headers and ?states\u003dlisting query params","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c72e81f0c0873abbe4eaf5b524bf5cdc9f6f9ef0","unresolved":false,"context_lines":[{"line_number":1653,"context_line":"                \u0027?states\u003dlisting;marker\u003degg;end_marker\u003djam;reverse\u003dtrue\u0027, {})"},{"line_number":1654,"context_line":"            resp_shards \u003d sr_dicts[:2]"},{"line_number":1655,"context_line":"            resp_shards.reverse()"},{"line_number":1656,"context_line":"            backend_req, resp \u003d capture_backend_request("},{"line_number":1657,"context_line":"                req, 200, json.dumps(resp_shards).encode(\u0027ascii\u0027),"},{"line_number":1658,"context_line":"                {\u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":1659,"context_line":"                 \u0027X-Backend-Sharding-State\u0027: sharding_state})"}],"source_content_type":"text/x-python","patch_set":4,"id":"fffc6b78_af3a0e31","line":1656,"in_reply_to":"1f621f24_f37ed0ad","updated":"2020-11-19 17:12:01.000000000","message":"+1 to improve the test commentary\n\nI think the backend header and params are ok though? e.g. an internal client can set x-backend-*","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4a02c0be75bd5c2e7d2fb69addc7054eeb94d261","unresolved":false,"context_lines":[{"line_number":1969,"context_line":"                        \u0027ContainerController._get_from_shards\u0027,"},{"line_number":1970,"context_line":"                        mock_get_from_shards):"},{"line_number":1971,"context_line":"            self.app.recheck_listing_shard_ranges \u003d 600"},{"line_number":1972,"context_line":"            do_test_caching(\u0027auto\u0027, 600)"},{"line_number":1973,"context_line":""},{"line_number":1974,"context_line":"    def test_cache_clearing(self):"},{"line_number":1975,"context_line":"        # verify that both metadata and shard ranges are purged form memcache"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f621f24_539ce45c","line":1972,"updated":"2020-11-17 23:28:09.000000000","message":"I don\u0027t think I\u0027m smart enough to to debug unittests over a few hundred lines","commit_id":"6850a4383ef1661082f976832cf5e5c3482c276d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":1993,"context_line":""},{"line_number":1994,"context_line":"        stale_info \u003d dict(info)"},{"line_number":1995,"context_line":"        stale_info.pop(\u0027status_changed_at\u0027)"},{"line_number":1996,"context_line":"        do_test(stale_info)"},{"line_number":1997,"context_line":""},{"line_number":1998,"context_line":"    def test_GET_shard_ranges_no_cache_write_for_non_sharded_states(self):"},{"line_number":1999,"context_line":"        # verify that shard ranges are not written to cache when container"}],"source_content_type":"text/x-python","patch_set":5,"id":"5210a0a5_a282c34b","line":1996,"updated":"2020-11-25 20:27:14.000000000","message":"great coverage here","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":1993,"context_line":""},{"line_number":1994,"context_line":"        stale_info \u003d dict(info)"},{"line_number":1995,"context_line":"        stale_info.pop(\u0027status_changed_at\u0027)"},{"line_number":1996,"context_line":"        do_test(stale_info)"},{"line_number":1997,"context_line":""},{"line_number":1998,"context_line":"    def test_GET_shard_ranges_no_cache_write_for_non_sharded_states(self):"},{"line_number":1999,"context_line":"        # verify that shard ranges are not written to cache when container"}],"source_content_type":"text/x-python","patch_set":5,"id":"8031f01b_e44f46c2","line":1996,"in_reply_to":"5210a0a5_a282c34b","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":2104,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2105,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2106,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2107,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2108,"context_line":""},{"line_number":2109,"context_line":"    def test_GET_shard_ranges_no_cache_when_include_deleted_shards(self):"},{"line_number":2110,"context_line":"        # verify that a GET for shards in listing states does not lookup or"}],"source_content_type":"text/x-python","patch_set":5,"id":"53d9877b_d970a251","line":2107,"updated":"2020-11-25 20:27:14.000000000","message":"i best we wanted to perturb the sharding_state param here","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":2104,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2105,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2106,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2107,"context_line":"        self._do_test_GET_shards_no_cache(\u0027sharded\u0027, {\u0027states\u0027: \u0027updating\u0027})"},{"line_number":2108,"context_line":""},{"line_number":2109,"context_line":"    def test_GET_shard_ranges_no_cache_when_include_deleted_shards(self):"},{"line_number":2110,"context_line":"        # verify that a GET for shards in listing states does not lookup or"}],"source_content_type":"text/x-python","patch_set":5,"id":"30bacfd0_dd24a84b","line":2107,"in_reply_to":"53d9877b_d970a251","updated":"2020-12-15 20:24:10.000000000","message":"Done","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":2153,"context_line":"            [(\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":2154,"context_line":"            self.memcache.calls)"},{"line_number":2155,"context_line":"        self.assertEqual(\u0027sharded\u0027,"},{"line_number":2156,"context_line":"                         self.memcache.calls[0][2][\u0027sharding_state\u0027])"},{"line_number":2157,"context_line":""},{"line_number":2158,"context_line":"    def test_GET_shard_ranges_no_memcache_available(self):"},{"line_number":2159,"context_line":"        self._setup_shard_range_stubs()"}],"source_content_type":"text/x-python","patch_set":5,"id":"ab640bf5_451e87e6","line":2156,"updated":"2020-11-25 20:27:14.000000000","message":"quite nice!","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":2153,"context_line":"            [(\u0027set\u0027, \u0027container/a/c\u0027, mock.ANY, 60)],"},{"line_number":2154,"context_line":"            self.memcache.calls)"},{"line_number":2155,"context_line":"        self.assertEqual(\u0027sharded\u0027,"},{"line_number":2156,"context_line":"                         self.memcache.calls[0][2][\u0027sharding_state\u0027])"},{"line_number":2157,"context_line":""},{"line_number":2158,"context_line":"    def test_GET_shard_ranges_no_memcache_available(self):"},{"line_number":2159,"context_line":"        self._setup_shard_range_stubs()"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8f7a28_a3d9352c","line":2156,"in_reply_to":"ab640bf5_451e87e6","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"22154d50eabb125c157e9264d937433d9adf6e86","unresolved":true,"context_lines":[{"line_number":2173,"context_line":"            \u0027X-Backend-Recheck-Container-Existence\u0027: \u002760\u0027,"},{"line_number":2174,"context_line":"            \u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":2175,"context_line":"            \u0027X-Backend-Sharding-State\u0027: \u0027sharded\u0027})"},{"line_number":2176,"context_line":"        self.assertEqual([], self.memcache.calls)  # sanity check"},{"line_number":2177,"context_line":""},{"line_number":2178,"context_line":"    def test_cache_clearing(self):"},{"line_number":2179,"context_line":"        # verify that both metadata and shard ranges are purged form memcache"}],"source_content_type":"text/x-python","patch_set":5,"id":"e76a4a41_d2c6f2f6","line":2176,"updated":"2020-11-25 20:27:14.000000000","message":"covering all the bases!","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc7ef464d2ed3eaac4f5951f0a3f25cd5501afba","unresolved":false,"context_lines":[{"line_number":2173,"context_line":"            \u0027X-Backend-Recheck-Container-Existence\u0027: \u002760\u0027,"},{"line_number":2174,"context_line":"            \u0027X-Backend-Record-Type\u0027: \u0027shard\u0027,"},{"line_number":2175,"context_line":"            \u0027X-Backend-Sharding-State\u0027: \u0027sharded\u0027})"},{"line_number":2176,"context_line":"        self.assertEqual([], self.memcache.calls)  # sanity check"},{"line_number":2177,"context_line":""},{"line_number":2178,"context_line":"    def test_cache_clearing(self):"},{"line_number":2179,"context_line":"        # verify that both metadata and shard ranges are purged form memcache"}],"source_content_type":"text/x-python","patch_set":5,"id":"0216bc14_1bd504d0","line":2176,"in_reply_to":"e76a4a41_d2c6f2f6","updated":"2020-12-15 20:24:10.000000000","message":"Ack","commit_id":"4609bce98f6ee1d1aab06517b6f933657624a411"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4c8b4596f446d0d1e30e58c0512064a8a70dad6d","unresolved":true,"context_lines":[{"line_number":1773,"context_line":"        self.memcache.clear_calls()"},{"line_number":1774,"context_line":"        # container is sharded but proxy does not have that state cached;"},{"line_number":1775,"context_line":"        # expect a backend request and expect shard ranges to be cached"},{"line_number":1776,"context_line":"        self.memcache.clear_calls()"},{"line_number":1777,"context_line":"        req \u003d self._build_request({\u0027X-Backend-Record-Type\u0027: record_type},"},{"line_number":1778,"context_line":"                                  {\u0027states\u0027: \u0027listing\u0027}, {})"},{"line_number":1779,"context_line":"        backend_req, resp \u003d self._capture_backend_request("}],"source_content_type":"text/x-python","patch_set":9,"id":"b9ea375b_254a98b6","line":1776,"updated":"2021-01-07 03:30:58.000000000","message":"Repeated clear, I hope we don\u0027t have to do it twice :P","commit_id":"077ba77ea6f735c98709b60e1fc9aa9f0f90676a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cf1db11b8a4f5e0daf871821d5d3d3c62f7f45ea","unresolved":true,"context_lines":[{"line_number":1773,"context_line":"        self.memcache.clear_calls()"},{"line_number":1774,"context_line":"        # container is sharded but proxy does not have that state cached;"},{"line_number":1775,"context_line":"        # expect a backend request and expect shard ranges to be cached"},{"line_number":1776,"context_line":"        self.memcache.clear_calls()"},{"line_number":1777,"context_line":"        req \u003d self._build_request({\u0027X-Backend-Record-Type\u0027: record_type},"},{"line_number":1778,"context_line":"                                  {\u0027states\u0027: \u0027listing\u0027}, {})"},{"line_number":1779,"context_line":"        backend_req, resp \u003d self._capture_backend_request("}],"source_content_type":"text/x-python","patch_set":9,"id":"c93a6e3c_4b63f50b","line":1776,"in_reply_to":"b9ea375b_254a98b6","updated":"2021-01-07 09:15:48.000000000","message":":( let\u0027s see if there\u0027s any other comments before I push a fix","commit_id":"077ba77ea6f735c98709b60e1fc9aa9f0f90676a"}]}