)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"202ccc1802082b63f96f10e5597a5ed0175ce286","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"bc4eb6f4_b8c89d1f","updated":"2023-10-25 18:54:49.000000000","message":"I expect maybe on the order of 10K reclaimable db messages tops - I don\u0027t think the memory is a concern and once every 24 hours seems sufficiently periodic and it\u0027s configurable.\n\nI expect after a service restart you\u0027d get a storm of \"periodic\" warnings in the first few hours before going silent for the next 24; but if a new db shows up during a subsequent cycle we\u0027ll always log a warning the frist time we see it.\n\nI think this is a minor ergonomic operational improvement that maybe makes the annoying but least problematic part of the design failure only marginally less annoying.  If ops was THAT annoyed by the logs they could probably filter in the log index - the real issue is we\u0027re alerting (loudly!) on the growing pile of reclaimable dbs that we aren\u0027t shrinking, and this spackling maybe makes it easier to kick that can down the road (which I honestly don\u0027t really love).","commit_id":"f14b1de4d56004d92870c99165015f96b4232a06"}],"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e3780c664746741e9700855f954aa0c333cef238","unresolved":true,"context_lines":[{"line_number":50,"context_line":"CLEAVE_FAILED \u003d 1"},{"line_number":51,"context_line":"CLEAVE_EMPTY \u003d 2"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"DEFAULT_PERIODIC_WARNINGS_INTERVAL \u003d 24 * 3600"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"def sharding_enabled(broker):"}],"source_content_type":"text/x-python","patch_set":2,"id":"cdb17fa7_56805556","line":53,"updated":"2023-10-25 00:18:27.000000000","message":"How often do we expect sharders to get restarted? I\u0027m a little worried about this default masking/delaying a (hopefully actionable? 🤞) warning for clusters that *aren\u0027t* swamped.","commit_id":"f14b1de4d56004d92870c99165015f96b4232a06"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24261789d962e65654d4f374b14b9c2ad502dc2b","unresolved":true,"context_lines":[{"line_number":50,"context_line":"CLEAVE_FAILED \u003d 1"},{"line_number":51,"context_line":"CLEAVE_EMPTY \u003d 2"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"DEFAULT_PERIODIC_WARNINGS_INTERVAL \u003d 24 * 3600"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"def sharding_enabled(broker):"}],"source_content_type":"text/x-python","patch_set":2,"id":"cf7eb749_84883741","line":53,"in_reply_to":"cdb17fa7_56805556","updated":"2023-10-25 09:49:35.000000000","message":"The warning will be logged at least once per 24 hours. The first occurrence is logged in each 24 hours of the same sharder process. If the sharder restarts, there may be more than one log in a  24 hour period (one from each sharder process).\n\nPerhaps the name is misleading - it\u0027s not the time *until* a warning is emitted.\n\nDEFAULT_PERIODIC_WARNINGS_RESET_INTERVAL ??","commit_id":"f14b1de4d56004d92870c99165015f96b4232a06"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"202ccc1802082b63f96f10e5597a5ed0175ce286","unresolved":true,"context_lines":[{"line_number":959,"context_line":"        db_file, path \u003d self._get_broker_details(broker)"},{"line_number":960,"context_line":"        key \u003d (db_file, msg)"},{"line_number":961,"context_line":"        if key not in self.periodic_warnings:"},{"line_number":962,"context_line":"            self.periodic_warnings.add(key)"},{"line_number":963,"context_line":"            self._log(logging.WARNING, broker, msg, *args, **kwargs)"},{"line_number":964,"context_line":""},{"line_number":965,"context_line":"    def error(self, broker, msg, *args, **kwargs):"}],"source_content_type":"text/x-python","patch_set":2,"id":"ddcb945e_762233f6","line":962,"updated":"2023-10-25 18:54:49.000000000","message":"so we accumulate these (db_file, msg) keys for a periodic interval and only emit the message once per","commit_id":"f14b1de4d56004d92870c99165015f96b4232a06"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7b6ba82a03254917c1dc0c00e16d92cae7f8a18e","unresolved":true,"context_lines":[{"line_number":959,"context_line":"        db_file, path \u003d self._get_broker_details(broker)"},{"line_number":960,"context_line":"        key \u003d (db_file, msg)"},{"line_number":961,"context_line":"        if key not in self.periodic_warnings:"},{"line_number":962,"context_line":"            self.periodic_warnings.add(key)"},{"line_number":963,"context_line":"            self._log(logging.WARNING, broker, msg, *args, **kwargs)"},{"line_number":964,"context_line":""},{"line_number":965,"context_line":"    def error(self, broker, msg, *args, **kwargs):"}],"source_content_type":"text/x-python","patch_set":2,"id":"4dc8da6c_5f731f7a","line":962,"in_reply_to":"ddcb945e_762233f6","updated":"2023-10-30 07:38:57.000000000","message":"ahh yeah i get it. We get one and if it hasn\u0027t been fired off yet, fire it off. Otherwise it\u0027ll have to wait until self.periodic_warnings os next cleared.","commit_id":"f14b1de4d56004d92870c99165015f96b4232a06"}]}