)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7b7a3a5c9c606d75a8616718964e62699b3c6687","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"0a232f5f_33d307aa","updated":"2024-04-23 01:57:19.000000000","message":"I\u0027m getting more comfortable with what this patch does and less comfortable with the idea it addresses our memcache problem - but the sooner we know if it\u0027s going to help us the sooner we can direct our limited resources/review-bandwidth to the most valuable/impactful work.","commit_id":"6892eb2c74cd3aad576f76c40346c6c6afd0df59"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ced1f037169b4ae6f90d1da3b111c734bb4f972d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"366a5512_43657d24","updated":"2024-04-30 05:30:24.000000000","message":"the new test looks good to me, I will have it squashed. thanks for the help!","commit_id":"6892eb2c74cd3aad576f76c40346c6c6afd0df59"}],"test/unit/proxy/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7b7a3a5c9c606d75a8616718964e62699b3c6687","unresolved":true,"context_lines":[{"line_number":5272,"context_line":"                def get(self, key, raise_on_error\u003dFalse):"},{"line_number":5273,"context_line":"                    if key !\u003d cache_key:"},{"line_number":5274,"context_line":"                        return super(CustomizedFakeCache, self).get(key)"},{"line_number":5275,"context_line":"                    # all fail forever - just like real memcache!"},{"line_number":5276,"context_line":"                    return super(CustomizedFakeCache, self).get("},{"line_number":5277,"context_line":"                        \"NOT_EXISTED_YET\")"},{"line_number":5278,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"ae40431a_9d35d5ce","line":5275,"updated":"2024-04-23 01:57:19.000000000","message":"as best I can tell the problem we\u0027re having in memcache is sometimes for hours we will KEEP trying to set a value but no matter how much set we send subsequent gets will just keep missing.\n\nIt\u0027s possible the reason this state doesn\u0027t correct sooner is because once we get a premature eviction we start setting the value \"too aggressively\" and the continuous storm of sets causes the slab reallocator to *rerepeatedly* fail and memcache constantly OOMs - so if it\u0027s our own sets that cause the storm and doing LESS sets during some small window (3s?) will allow the slab reallocator to \"make room\" then this patch may already have everything we need.\n\nOTOH, if it\u0027s reasoable to expect memcache to continue to return a miss even after set when it gets full this will probably just arbitrarily batch backend requests for 3s, unleash a herd, and then start over again and again until whatever actually FIXES memcache happens on it\u0027s own.  In which case this new feature will probably mostly work as described at least the cold start case which could still be useful on it\u0027s own.","commit_id":"6892eb2c74cd3aad576f76c40346c6c6afd0df59"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7b7a3a5c9c606d75a8616718964e62699b3c6687","unresolved":true,"context_lines":[{"line_number":5315,"context_line":"                              \u0027container.info.infocache.hit\u0027: 1,"},{"line_number":5316,"context_line":"                              \u0027object.shard_updating.cache.miss.200\u0027: 1,"},{"line_number":5317,"context_line":"                              \u0027object.shard_updating.cache.set\u0027: 1,"},{"line_number":5318,"context_line":"                              \u0027token.shard_updating.backend_reqs\u0027: 1},"},{"line_number":5319,"context_line":"                             stats)"},{"line_number":5320,"context_line":"            self.assertEqual([], self.app.logger.log_dict[\u0027set_statsd_prefix\u0027])"},{"line_number":5321,"context_line":"            info_lines \u003d self.logger.get_lines_for_level(\u0027info\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"990a6cbe_26bca30d","line":5318,"updated":"2024-04-23 01:57:19.000000000","message":"afaict the only difference between \"terrible bad worst possible case put everything back on the drawing board\" scenario and \"everything is fine well as expected best implementation ever\" is the *lack* of one \"done_token_reqs\" counter here:\n\nhttps://review.opendev.org/c/openstack/swift/+/908969/16/test/unit/proxy/test_server.py#5101","commit_id":"6892eb2c74cd3aad576f76c40346c6c6afd0df59"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ced1f037169b4ae6f90d1da3b111c734bb4f972d","unresolved":true,"context_lines":[{"line_number":5315,"context_line":"                              \u0027container.info.infocache.hit\u0027: 1,"},{"line_number":5316,"context_line":"                              \u0027object.shard_updating.cache.miss.200\u0027: 1,"},{"line_number":5317,"context_line":"                              \u0027object.shard_updating.cache.set\u0027: 1,"},{"line_number":5318,"context_line":"                              \u0027token.shard_updating.backend_reqs\u0027: 1},"},{"line_number":5319,"context_line":"                             stats)"},{"line_number":5320,"context_line":"            self.assertEqual([], self.app.logger.log_dict[\u0027set_statsd_prefix\u0027])"},{"line_number":5321,"context_line":"            info_lines \u003d self.logger.get_lines_for_level(\u0027info\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"0c30a670_8137c529","line":5318,"in_reply_to":"990a6cbe_26bca30d","updated":"2024-04-30 05:30:24.000000000","message":"will add more metrics to help here too.","commit_id":"6892eb2c74cd3aad576f76c40346c6c6afd0df59"}]}