)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0a83744198e1a3aa1092b93dcaf498db9f2c6e06","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"91c3e285_a9e4eaaa","updated":"2022-08-04 19:42:38.000000000","message":"Probe test failure seems to be caused by the container-server returning a set of shard ranges like\n\n [ShardRange\u003cMinBound to \u0027beta024\u0027 as of 1659640785.18315, (25, 0) as of 1659640786.48467, cleaved as of 1659640785.18315\u003e,\n  ShardRange\u003c\u0027beta024\u0027 to \u0027beta049\u0027 as of 1659640785.18315, (25, 0) as of 1659640786.51794, cleaved as of 1659640785.18315\u003e,\n  ShardRange\u003c\u0027beta049\u0027 to \u0027obj-0024\u0027 as of 1659640785.18315, (25, 0) as of 1659640785.18315, created as of 1659640785.18315\u003e,\n  ShardRange\u003cMinBound to \u0027obj-0024\u0027 as of 1659640775.90179, (75, 0) as of 1659640787.58121, sharding as of 1659640784.02609\u003e,\n  ShardRange\u003c\u0027obj-0024\u0027 to MaxBound as of 1659640775.90179, (25, 0) as of 1659640776.11280, active as of 1659640775.90179\u003e]\n\nwhich sure smells like a bug -- that sharding shard should have been replaced by the CLEAVED/CREATED ones.","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"aa60baeb46f7dc2ff5728739bcd4529752998099","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"abbe4fc9_a69ce4f1","in_reply_to":"91c3e285_a9e4eaaa","updated":"2022-08-04 23:37:53.000000000","message":"Needs tests, but https://review.opendev.org/c/openstack/swift/+/852221 should resolve it.","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"42811cdb71626457b07587b9b3d84ea81e186cc4","unresolved":true,"context_lines":[{"line_number":2391,"context_line":"                return None"},{"line_number":2392,"context_line":"            return shard_ranges[0]"},{"line_number":2393,"context_line":""},{"line_number":2394,"context_line":"        cache_key \u003d get_cache_key(account, container, shard\u003d\u0027updating-v2\u0027)"},{"line_number":2395,"context_line":"        infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":2396,"context_line":"        memcache \u003d cache_from_env(req.environ, True)"},{"line_number":2397,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"acd8b183_cafbc1e7","line":2394,"range":{"start_line":2394,"start_character":60,"end_line":2394,"end_character":72},"updated":"2022-08-22 15:40:44.000000000","message":"hmmm, so during upgrade we\u0027ll effectively lose the current cache content.\n\nI wonder if we could keep the same key and use a try/except to detect the old format. \n\nIt might be worth wrapping the shard range list in a dict and including a version field (and an expiry time!).\n\n  cache_value \u003d {\n      \"version\": 2,\n      \"expires\": \u003ctime\u003e,\n      \"ranges\": [...],\n      }","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0c5edc3ccb681525eb06f241e7e589a7e2315508","unresolved":true,"context_lines":[{"line_number":2391,"context_line":"                return None"},{"line_number":2392,"context_line":"            return shard_ranges[0]"},{"line_number":2393,"context_line":""},{"line_number":2394,"context_line":"        cache_key \u003d get_cache_key(account, container, shard\u003d\u0027updating-v2\u0027)"},{"line_number":2395,"context_line":"        infocache \u003d req.environ.setdefault(\u0027swift.infocache\u0027, {})"},{"line_number":2396,"context_line":"        memcache \u003d cache_from_env(req.environ, True)"},{"line_number":2397,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"b3ecf09b_ce46657c","line":2394,"range":{"start_line":2394,"start_character":60,"end_line":2394,"end_character":72},"in_reply_to":"acd8b183_cafbc1e7","updated":"2022-08-24 20:52:18.000000000","message":"Will need a config option to decide whether to write the new format or not -- otherwise, during a rolling upgrade you could have new code writing the new format to cache, then old code blowing up upon reading it. And we should immediately be thinking about how to deprecate/remove it -- it took us something like 5 years (!) to remove pickle support from our memcache client.\n\nHow bad is it really to flop over to a new key? If you\u0027re sure to upgrade just one proxy as part of a canary rollout ahead of the main upgrade, that slice of client traffic will cause the new key to get populated quickly -- the main question becomes whether that 1/N slice of traffic would still be too much and overwhelm the container servers.\n\nNot opposed to having a slightly richer data structure, though!","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"aa60baeb46f7dc2ff5728739bcd4529752998099","unresolved":true,"context_lines":[{"line_number":2404,"context_line":"            else:"},{"line_number":2405,"context_line":"                try:"},{"line_number":2406,"context_line":"                    cached_ranges \u003d memcache.get("},{"line_number":2407,"context_line":"                        cache_key, raise_on_error\u003dTrue)"},{"line_number":2408,"context_line":"                    cache_state \u003d \u0027hit\u0027 if cached_ranges else \u0027miss\u0027"},{"line_number":2409,"context_line":"                except MemcacheConnectionError:"},{"line_number":2410,"context_line":"                    cache_state \u003d \u0027error\u0027"}],"source_content_type":"text/x-python","patch_set":1,"id":"ca9a453c_e8b616cb","line":2407,"updated":"2022-08-04 23:37:53.000000000","message":"Trouble on py2:\n\n UnicodeDecodeError: \u0027ascii\u0027 codec can\u0027t decode byte 0xc3 in position 4: ordinal not in range(128)\n\nWhen we\u0027ve got a cache hit, the bounds are coming out as unicode, but obj is bytes.","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"42811cdb71626457b07587b9b3d84ea81e186cc4","unresolved":true,"context_lines":[{"line_number":2416,"context_line":"            self.logger.increment("},{"line_number":2417,"context_line":"                \u0027shard_updating.backend.%s\u0027 % response.status_int)"},{"line_number":2418,"context_line":"            if shard_ranges:"},{"line_number":2419,"context_line":"                cached_ranges \u003d [[str(sr.lower), sr.name]"},{"line_number":2420,"context_line":"                                 for sr in shard_ranges]"},{"line_number":2421,"context_line":"                # went to disk; cache it"},{"line_number":2422,"context_line":"                if memcache:"}],"source_content_type":"text/x-python","patch_set":1,"id":"19e7706a_4678cf79","line":2419,"range":{"start_line":2419,"start_character":34,"end_line":2419,"end_character":47},"updated":"2022-08-22 15:40:44.000000000","message":"sr.lower_str()","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0c5edc3ccb681525eb06f241e7e589a7e2315508","unresolved":true,"context_lines":[{"line_number":2416,"context_line":"            self.logger.increment("},{"line_number":2417,"context_line":"                \u0027shard_updating.backend.%s\u0027 % response.status_int)"},{"line_number":2418,"context_line":"            if shard_ranges:"},{"line_number":2419,"context_line":"                cached_ranges \u003d [[str(sr.lower), sr.name]"},{"line_number":2420,"context_line":"                                 for sr in shard_ranges]"},{"line_number":2421,"context_line":"                # went to disk; cache it"},{"line_number":2422,"context_line":"                if memcache:"}],"source_content_type":"text/x-python","patch_set":1,"id":"2fb1ce7f_f4e0b54c","line":2419,"range":{"start_line":2419,"start_character":34,"end_line":2419,"end_character":47},"in_reply_to":"19e7706a_4678cf79","updated":"2022-08-24 20:52:18.000000000","message":"Good call.","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"42811cdb71626457b07587b9b3d84ea81e186cc4","unresolved":true,"context_lines":[{"line_number":2416,"context_line":"            self.logger.increment("},{"line_number":2417,"context_line":"                \u0027shard_updating.backend.%s\u0027 % response.status_int)"},{"line_number":2418,"context_line":"            if shard_ranges:"},{"line_number":2419,"context_line":"                cached_ranges \u003d [[str(sr.lower), sr.name]"},{"line_number":2420,"context_line":"                                 for sr in shard_ranges]"},{"line_number":2421,"context_line":"                # went to disk; cache it"},{"line_number":2422,"context_line":"                if memcache:"},{"line_number":2423,"context_line":"                    memcache.set(cache_key, cached_ranges,"}],"source_content_type":"text/x-python","patch_set":1,"id":"b9692a4c_584df61a","line":2420,"range":{"start_line":2419,"start_character":32,"end_line":2420,"end_character":56},"updated":"2022-08-22 15:40:44.000000000","message":"we ought to be able to extract the a Namespace superclass with just name, lower, upper and then encapsulate some compact serialization method which could be re-used in other places that we cache shard ranges. There are also places where we abuse ShardRange to represent simply a namespace, for which the superclass would be appropriate.","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"42811cdb71626457b07587b9b3d84ea81e186cc4","unresolved":true,"context_lines":[{"line_number":2429,"context_line":"        infocache[cache_key] \u003d tuple(cached_ranges)"},{"line_number":2430,"context_line":"        pos \u003d bisect.bisect(cached_ranges, [obj]) - 1"},{"line_number":2431,"context_line":"        lower, name \u003d cached_ranges[pos]"},{"line_number":2432,"context_line":"        upper \u003d (\u0027\u0027 if pos + 1 \u003d\u003d len(cached_ranges)"},{"line_number":2433,"context_line":"                 else cached_ranges[pos + 1][0])"},{"line_number":2434,"context_line":"        return ShardRange(name, Timestamp.now(), lower, upper)"}],"source_content_type":"text/x-python","patch_set":1,"id":"70730d7f_42b61007","line":2433,"range":{"start_line":2432,"start_character":8,"end_line":2433,"end_character":48},"updated":"2022-08-22 15:40:44.000000000","message":"we can\u0027t make that leap - shard ranges returned from the backend can contain gaps, so I think we need to cache the upper as well as lower, or at least cache the upper when there is a gap.\n\n  let range[n] \u003d (lower, name, upper) where upper \u003d\u003d None -\u003e upper \u003d range[n+1].upper\n  \nI guess the value of compaction depends on whether we\u0027re primarily seeking faster decodes, and whether decode speed is a function of value size and/or item count.","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0c5edc3ccb681525eb06f241e7e589a7e2315508","unresolved":true,"context_lines":[{"line_number":2429,"context_line":"        infocache[cache_key] \u003d tuple(cached_ranges)"},{"line_number":2430,"context_line":"        pos \u003d bisect.bisect(cached_ranges, [obj]) - 1"},{"line_number":2431,"context_line":"        lower, name \u003d cached_ranges[pos]"},{"line_number":2432,"context_line":"        upper \u003d (\u0027\u0027 if pos + 1 \u003d\u003d len(cached_ranges)"},{"line_number":2433,"context_line":"                 else cached_ranges[pos + 1][0])"},{"line_number":2434,"context_line":"        return ShardRange(name, Timestamp.now(), lower, upper)"}],"source_content_type":"text/x-python","patch_set":1,"id":"d4384861_bd35b959","line":2433,"range":{"start_line":2432,"start_character":8,"end_line":2433,"end_character":48},"in_reply_to":"70730d7f_42b61007","updated":"2022-08-24 20:52:18.000000000","message":"\u003e shard ranges returned from the backend can contain gaps\n\nThat sounds like a bug -- I think whenever we\u0027re getting updating shards (and possibly whenever we\u0027ve got fill_gaps\u003dTrue, though I haven\u0027t thought as much about the listing case yet), it\u0027s appropriate for the container server to create more synthetic shards pointing at the root.\n\nI mean, it\u0027s *also* a bug that those gaps happen *at all* -- but hopefully we\u0027re getting better and better at\n\n* avoiding creating gaps,\n* noticing when a gap happens, and\n* quickly repairing the gaps, even if it currently requires some manual intervention.","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0c5edc3ccb681525eb06f241e7e589a7e2315508","unresolved":true,"context_lines":[{"line_number":2429,"context_line":"        infocache[cache_key] \u003d tuple(cached_ranges)"},{"line_number":2430,"context_line":"        pos \u003d bisect.bisect(cached_ranges, [obj]) - 1"},{"line_number":2431,"context_line":"        lower, name \u003d cached_ranges[pos]"},{"line_number":2432,"context_line":"        upper \u003d (\u0027\u0027 if pos + 1 \u003d\u003d len(cached_ranges)"},{"line_number":2433,"context_line":"                 else cached_ranges[pos + 1][0])"},{"line_number":2434,"context_line":"        return ShardRange(name, Timestamp.now(), lower, upper)"}],"source_content_type":"text/x-python","patch_set":1,"id":"68937ab2_7a67b8be","line":2433,"range":{"start_line":2432,"start_character":8,"end_line":2433,"end_character":48},"in_reply_to":"70730d7f_42b61007","updated":"2022-08-24 20:52:18.000000000","message":"My thought is that for updating ranges in particular, it doesn\u0027t matter *that much* if we get the shard wrong. Let it get moved via misplaced-objects.\n\nMeanwhile, including upper adds ~50% to our cache value length: looking at a 5900 shard-range DB from prod, I see\n\n* 3.1M when storing the whole container-server response;\n* 1.3M when storing lower, upper, and name; and\n* 905K when only storing lower and name.\n\nIt *also* impacts deserialization time: on py2, that last one loads up in ~5.3ms, compared to ~8.2ms for the middle one. Way better than the 43ms from the top one, of course -- but still!","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ec4f8c5e262ed080d7de212e9b1696132e0d6666","unresolved":true,"context_lines":[{"line_number":2429,"context_line":"        infocache[cache_key] \u003d tuple(cached_ranges)"},{"line_number":2430,"context_line":"        pos \u003d bisect.bisect(cached_ranges, [obj]) - 1"},{"line_number":2431,"context_line":"        lower, name \u003d cached_ranges[pos]"},{"line_number":2432,"context_line":"        upper \u003d (\u0027\u0027 if pos + 1 \u003d\u003d len(cached_ranges)"},{"line_number":2433,"context_line":"                 else cached_ranges[pos + 1][0])"},{"line_number":2434,"context_line":"        return ShardRange(name, Timestamp.now(), lower, upper)"}],"source_content_type":"text/x-python","patch_set":1,"id":"d190e7f2_8c7ef7f9","line":2433,"range":{"start_line":2432,"start_character":8,"end_line":2433,"end_character":48},"in_reply_to":"d4384861_bd35b959","updated":"2022-08-24 21:19:10.000000000","message":"for updating shard ranges I think it makes little difference if a gap filler is synthesised or not: the outcome is that the update goes to the root\n\nIIRC the gap filler appendix may have been to deal with partially cleaved containers, rather than unintended gaps. But I\u0027m not sure.\n\nI need to remind myself what happens with misplaced object rows in shard containers - how does the shard know where to move them?","commit_id":"824f132933d32903acd5c2733865c23dccdf01b6"}]}