)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"08fdabca514f3fe621003699a3448be4f9309800","unresolved":true,"context_lines":[{"line_number":9,"context_line":"In the past we have had some async pendings that repeatedly fail for months at"},{"line_number":10,"context_line":"a time. This patch adds an oldest_failed_update statistic to the object"},{"line_number":11,"context_line":"updater\u0027s SweepStats so that the oldest failed async pending update can be surfaced and"},{"line_number":12,"context_line":"logged."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I6d9667d555836cfceda52708a57a1d29ebd1a80b"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"ef061bff_44a1bf8d","line":12,"updated":"2024-10-09 20:28:10.000000000","message":"I wonder if we might actually want two new stats:\n\n- the oldest async that was found and \"processed\" -- i.e., it will not be found on subsequent cycles, and\n- the oldest async that was found and will need to be tried again later.\n\nThat first category seems like it ought to be where quarantines go -- and it\u0027d also be the place to track outdated updates.","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":9,"context_line":"In the past we have had some async pendings that repeatedly fail for months at"},{"line_number":10,"context_line":"a time. This patch adds an oldest_failed_update statistic to the object"},{"line_number":11,"context_line":"updater\u0027s SweepStats so that the oldest failed async pending update can be surfaced and"},{"line_number":12,"context_line":"logged."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I6d9667d555836cfceda52708a57a1d29ebd1a80b"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"a3245083_e659e1a6","line":12,"in_reply_to":"4945dd54_1bc23045","updated":"2024-10-31 17:44:27.000000000","message":"Made a separate patch that can be squashed into this one","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"591e6ef23b304748a33572e8edcd3e5f16ebf979","unresolved":true,"context_lines":[{"line_number":9,"context_line":"In the past we have had some async pendings that repeatedly fail for months at"},{"line_number":10,"context_line":"a time. This patch adds an oldest_failed_update statistic to the object"},{"line_number":11,"context_line":"updater\u0027s SweepStats so that the oldest failed async pending update can be surfaced and"},{"line_number":12,"context_line":"logged."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I6d9667d555836cfceda52708a57a1d29ebd1a80b"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"4945dd54_1bc23045","line":12,"in_reply_to":"ef061bff_44a1bf8d","updated":"2024-10-16 12:34:52.000000000","message":"Also, think this could be a separate patch? Or do you think with the new implementation that the provided stats are sufficient?","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"6abde84d_691bd3ca","updated":"2024-10-17 15:56:28.000000000","message":"I haven\u0027t really gotten into the unittests yet, but I have done some functional testing use a probe test:\n\nhttps://review.opendev.org/c/openstack/swift/+/932625\n\nit seems to work!","commit_id":"a908bf7c6a2b55f13dfc5e91fcc8a063e71fb84c"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3fabc093ccc6ea9b93b8e60f62a8914c9e6bf622","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":16,"id":"a0dfeb0c_c4c43b8c","updated":"2024-10-26 00:10:30.000000000","message":"The age of those async pendings is a mystery in the cluster, we need those info in different situations in order to debug what\u0027s going on, and this patch will sure help! Thanks for working on this patch.\n\nI haven\u0027t looked into test code yet, but the production code is well written. Put ``-1`` here, mainly because the new settings haven\u0027t been added into the sample config file and I wish a lot of names could be simpler.","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"9a4b988f_9957eef2","updated":"2024-11-01 17:57:23.000000000","message":"Perfectly reasonable to carry -- interested in seeing what sort of memory numbers we get out of prod.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"856b29d349492efce0f0c88d46791ca875f27b38","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":33,"id":"5c5bd0ad_574908ff","updated":"2024-11-05 02:57:06.000000000","message":"recheck\n\nAnother probe test hang :-/\n```\n2024-11-04 23:14:57.079649 | centos-9-stream | test/probe/test_container_failures.py::TestContainerFailures::test_two_nodes_fail PASSED [  5%]\n2024-11-04 23:16:07.883432 | centos-9-stream | test/probe/test_container_merge_policy_index.py::TestContainerMergePolicyIndex::test_merge_storage_policy_index PASSED [  6%]\n2024-11-05 01:09:28.312299 | RUN END RESULT_TIMED_OUT: [untrusted : opendev.org/openstack/swift/tools/playbooks/probetests/run.yaml@master]\n2024-11-05 01:09:28.314231 | POST-RUN START: [untrusted : opendev.org/openstack/swift/tools/playbooks/probetests/post.yaml@master]\n```","commit_id":"0a5348eb480e0b9ff0abc9acea183d94a97d700e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"85412472e82b605ec31a49ad2e26dee07056b6f7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":33,"id":"cd79ec99_c35e2b9e","updated":"2024-11-05 05:00:42.000000000","message":"recheck\n\nhttps://bugs.launchpad.net/openstacksdk/+bug/2085654","commit_id":"0a5348eb480e0b9ff0abc9acea183d94a97d700e"}],"swift/obj/updater.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"08fdabca514f3fe621003699a3448be4f9309800","unresolved":true,"context_lines":[{"line_number":538,"context_line":"                        self.stats.unlinks +\u003d 1"},{"line_number":539,"context_line":"                        self.logger.increment(\u0027unlinks\u0027)"},{"line_number":540,"context_line":"                        try:"},{"line_number":541,"context_line":"                            os.unlink(update_path)"},{"line_number":542,"context_line":"                        except OSError as e:"},{"line_number":543,"context_line":"                            if e.errno !\u003d errno.ENOENT:"},{"line_number":544,"context_line":"                                raise"}],"source_content_type":"text/x-python","patch_set":1,"id":"ae0f324b_286c291b","line":541,"updated":"2024-10-09 20:28:10.000000000","message":"I feel like we probably want to be tracking how far behind these are, too.","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":538,"context_line":"                        self.stats.unlinks +\u003d 1"},{"line_number":539,"context_line":"                        self.logger.increment(\u0027unlinks\u0027)"},{"line_number":540,"context_line":"                        try:"},{"line_number":541,"context_line":"                            os.unlink(update_path)"},{"line_number":542,"context_line":"                        except OSError as e:"},{"line_number":543,"context_line":"                            if e.errno !\u003d errno.ENOENT:"},{"line_number":544,"context_line":"                                raise"}],"source_content_type":"text/x-python","patch_set":1,"id":"bbbdf542_4f8bd89c","line":541,"in_reply_to":"60d93b3b_4581a02e","updated":"2024-10-31 17:44:27.000000000","message":"I tracked them in a new patchset here: https://review.opendev.org/c/openstack/swift/+/933880","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"591e6ef23b304748a33572e8edcd3e5f16ebf979","unresolved":true,"context_lines":[{"line_number":538,"context_line":"                        self.stats.unlinks +\u003d 1"},{"line_number":539,"context_line":"                        self.logger.increment(\u0027unlinks\u0027)"},{"line_number":540,"context_line":"                        try:"},{"line_number":541,"context_line":"                            os.unlink(update_path)"},{"line_number":542,"context_line":"                        except OSError as e:"},{"line_number":543,"context_line":"                            if e.errno !\u003d errno.ENOENT:"},{"line_number":544,"context_line":"                                raise"}],"source_content_type":"text/x-python","patch_set":1,"id":"e5497385_6a3cb21c","line":541,"in_reply_to":"ae0f324b_286c291b","updated":"2024-10-16 12:34:52.000000000","message":"Would it make sense to do this in a separate patch as this one is getting large?","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":538,"context_line":"                        self.stats.unlinks +\u003d 1"},{"line_number":539,"context_line":"                        self.logger.increment(\u0027unlinks\u0027)"},{"line_number":540,"context_line":"                        try:"},{"line_number":541,"context_line":"                            os.unlink(update_path)"},{"line_number":542,"context_line":"                        except OSError as e:"},{"line_number":543,"context_line":"                            if e.errno !\u003d errno.ENOENT:"},{"line_number":544,"context_line":"                                raise"}],"source_content_type":"text/x-python","patch_set":1,"id":"60d93b3b_4581a02e","line":541,"in_reply_to":"e5497385_6a3cb21c","updated":"2024-10-17 15:56:28.000000000","message":"large is releative.  If the work is necessary and conceptually related it\u0027s best to have a complete patch.\n\nThere\u0027s two ways to think about the consistency window.  Because we\u0027re going to be looking at recon dumps we\u0027re always going to be a cycle behind, so I think it\u0027s reasonable to try and make as complete a picture as we reasonably can.\n\nThis patch takes the optimistic outlook: when an update fails we know we\u0027re *at least* that far behind.  Tracking stats for \"success\" (i.e. unlink) could allow us to report \"how bad was the last cycle before we got the updates in\" - the delta of which may be able to give us some indication if the situation is *improving*\n\nI think it\u0027s worth considering how this new OldestManager primative would have to adapt if we wanted to report both lists or make some claims about \"progress\" based on their differences/similarities.\n\nHowever, given our experience with how stuck updates happen/behave - I would expect each cycle will unlink a lot of updates that are relatively new, and report a growing consistency window delta of old stuck asyncs.  Then once the down container-server comes back the unlinks will \"spike\" a bunch of old \"success\" and the failures guage will dip back down to normal.  I\u0027m not sure yet I have a convincing picture of how we would use the unlink spikes to inform ops of any corrective action - at that point whatever was wrong is already fixed.","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"08fdabca514f3fe621003699a3448be4f9309800","unresolved":true,"context_lines":[{"line_number":545,"context_line":"                    else:"},{"line_number":546,"context_line":"                        last_obj_hash \u003d obj_hash"},{"line_number":547,"context_line":"                        update \u003d self._load_update(device, update_path)"},{"line_number":548,"context_line":"                        if update is None:"},{"line_number":549,"context_line":"                            if ("},{"line_number":550,"context_line":"                                not self.stats.oldest_failed_update"},{"line_number":551,"context_line":"                                or timestamp \u003c self.stats.oldest_failed_update"}],"source_content_type":"text/x-python","patch_set":1,"id":"04b5aeda_a3a6e97b","line":548,"updated":"2024-10-09 20:28:10.000000000","message":"If it comes back `None`, it\u0027s been quarantined -- it\u0027s not clear to me that we should count it for this stat.","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"9c0ddcbca786e4a78881909dfb67f9f85747afa2","unresolved":true,"context_lines":[{"line_number":545,"context_line":"                    else:"},{"line_number":546,"context_line":"                        last_obj_hash \u003d obj_hash"},{"line_number":547,"context_line":"                        update \u003d self._load_update(device, update_path)"},{"line_number":548,"context_line":"                        if update is None:"},{"line_number":549,"context_line":"                            if ("},{"line_number":550,"context_line":"                                not self.stats.oldest_failed_update"},{"line_number":551,"context_line":"                                or timestamp \u003c self.stats.oldest_failed_update"}],"source_content_type":"text/x-python","patch_set":1,"id":"663efb89_6316e2b7","line":548,"in_reply_to":"04b5aeda_a3a6e97b","updated":"2024-10-10 16:02:46.000000000","message":"Alas, I was confused.","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"4faa3329430dc252ea99604c69c02c93857eb289","unresolved":false,"context_lines":[{"line_number":545,"context_line":"                    else:"},{"line_number":546,"context_line":"                        last_obj_hash \u003d obj_hash"},{"line_number":547,"context_line":"                        update \u003d self._load_update(device, update_path)"},{"line_number":548,"context_line":"                        if update is None:"},{"line_number":549,"context_line":"                            if ("},{"line_number":550,"context_line":"                                not self.stats.oldest_failed_update"},{"line_number":551,"context_line":"                                or timestamp \u003c self.stats.oldest_failed_update"}],"source_content_type":"text/x-python","patch_set":1,"id":"e875d30f_65d2f5f3","line":548,"in_reply_to":"663efb89_6316e2b7","updated":"2024-10-16 18:09:31.000000000","message":"Did my counting upon failure rather than quarantine.","commit_id":"a50c71034bbd4ed9f4ba4db096df80decd23458c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":259,"context_line":"        self.account_container_recon_dump_count \u003d ("},{"line_number":260,"context_line":"            account_container_recon_dump_count"},{"line_number":261,"context_line":"        )"},{"line_number":262,"context_line":"        self.timestamp_to_account_containers \u003d SortedDict()"},{"line_number":263,"context_line":"        self.account_container_to_timestamp \u003d {}"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"    def __eq__(self, other):"}],"source_content_type":"text/x-python","patch_set":10,"id":"308b886c_292452dd","line":262,"updated":"2024-10-17 15:56:28.000000000","message":"this maps timestamps to a set of ac pairs\n\n```\n(Pdb) print(json.dumps({ts: list(ac) for ts, ac in self.stats.oldest_async_pendings.timestamp_to_account_containers.items()}, indent\u003d2))\n{\n  \"1729178000.50682\": [\n    [\n      \"AUTH_user000\",\n      \"cont000\"\n    ]\n  ],\n  \"1729178001.15179\": [\n    [\n      \"AUTH_user000\",\n      \"cont002\"\n    ]\n  ],\n  \"1729178001.32445\": [\n    [\n      \"AUTH_user000\",\n      \"cont003\"\n    ]\n  ],\n  \"1729178001.75655\": [\n    [\n      \"AUTH_user001\",\n      \"cont000\"\n    ]\n  ],\n  \"1729178001.86994\": [\n    [\n      \"AUTH_user001\",\n      \"cont001\"\n    ]\n  ],\n  \"1729178002.20356\": [\n    [\n      \"AUTH_user001\",\n      \"cont002\"\n    ]\n  ],\n  \"1729178002.57021\": [\n    [\n      \"AUTH_user001\",\n      \"cont003\"\n    ]\n  ],\n  \"1729178002.68769\": [\n    [\n      \"AUTH_user002\",\n      \"cont000\"\n    ]\n  ],\n  \"1729178003.06466\": [\n    [\n      \"AUTH_user002\",\n      \"cont001\"\n    ]\n  ],\n  \"1729178003.29279\": [\n    [\n      \"AUTH_user002\",\n      \"cont002\"\n    ]\n  ],\n  \"1729178003.51684\": [\n    [\n      \"AUTH_user002\",\n      \"cont003\"\n    ]\n  ]\n}\n```","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":259,"context_line":"        self.account_container_recon_dump_count \u003d ("},{"line_number":260,"context_line":"            account_container_recon_dump_count"},{"line_number":261,"context_line":"        )"},{"line_number":262,"context_line":"        self.timestamp_to_account_containers \u003d SortedDict()"},{"line_number":263,"context_line":"        self.account_container_to_timestamp \u003d {}"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"    def __eq__(self, other):"}],"source_content_type":"text/x-python","patch_set":10,"id":"9cd90607_c47dc022","line":262,"in_reply_to":"308b886c_292452dd","updated":"2024-10-31 17:44:27.000000000","message":"Acknowledged","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":260,"context_line":"            account_container_recon_dump_count"},{"line_number":261,"context_line":"        )"},{"line_number":262,"context_line":"        self.timestamp_to_account_containers \u003d SortedDict()"},{"line_number":263,"context_line":"        self.account_container_to_timestamp \u003d {}"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"    def __eq__(self, other):"},{"line_number":266,"context_line":"        if isinstance(other, AccountContainerOldestAsyncPendingManager):"}],"source_content_type":"text/x-python","patch_set":10,"id":"c09d5d08_e303dacb","line":263,"updated":"2024-10-17 15:56:28.000000000","message":"this maps ac pairs to timestamps\n\n```\n(Pdb) print(json.dumps({\u0027/\u0027.join(ac): ts for ac, ts in self.stats.oldest_async_pendings.account_container_to_timestamp.items()}, indent\u003d2))\n{\n  \"AUTH_user001/cont003\": \"1729178002.57021\",\n  \"AUTH_user001/cont001\": \"1729178001.86994\",\n  \"AUTH_user002/cont000\": \"1729178002.68769\",\n  \"AUTH_user000/cont003\": \"1729178001.32445\",\n  \"AUTH_user001/cont000\": \"1729178001.75655\",\n  \"AUTH_user002/cont002\": \"1729178003.29279\",\n  \"AUTH_user000/cont002\": \"1729178001.15179\",\n  \"AUTH_user000/cont000\": \"1729178000.50682\",\n  \"AUTH_user001/cont002\": \"1729178002.20356\",\n  \"AUTH_user002/cont003\": \"1729178003.51684\",\n  \"AUTH_user002/cont001\": \"1729178003.06466\"\n}\n```","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":260,"context_line":"            account_container_recon_dump_count"},{"line_number":261,"context_line":"        )"},{"line_number":262,"context_line":"        self.timestamp_to_account_containers \u003d SortedDict()"},{"line_number":263,"context_line":"        self.account_container_to_timestamp \u003d {}"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"    def __eq__(self, other):"},{"line_number":266,"context_line":"        if isinstance(other, AccountContainerOldestAsyncPendingManager):"}],"source_content_type":"text/x-python","patch_set":10,"id":"17748bca_6a48d06d","line":263,"in_reply_to":"c09d5d08_e303dacb","updated":"2024-10-31 17:44:27.000000000","message":"Acknowledged","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":275,"context_line":"                and self.account_container_to_timestamp"},{"line_number":276,"context_line":"                \u003d\u003d other.account_container_to_timestamp"},{"line_number":277,"context_line":"            )"},{"line_number":278,"context_line":"        return False"},{"line_number":279,"context_line":""},{"line_number":280,"context_line":"    def add_update(self, account, container, timestamp):"},{"line_number":281,"context_line":"        account_container \u003d (account, container)"}],"source_content_type":"text/x-python","patch_set":10,"id":"6839e209_0aa2a859","line":278,"updated":"2024-10-17 15:56:28.000000000","message":"is this needed?  it looks like a pita to maintain.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":275,"context_line":"                and self.account_container_to_timestamp"},{"line_number":276,"context_line":"                \u003d\u003d other.account_container_to_timestamp"},{"line_number":277,"context_line":"            )"},{"line_number":278,"context_line":"        return False"},{"line_number":279,"context_line":""},{"line_number":280,"context_line":"    def add_update(self, account, container, timestamp):"},{"line_number":281,"context_line":"        account_container \u003d (account, container)"}],"source_content_type":"text/x-python","patch_set":10,"id":"d1f706c8_7c82ac3a","line":278,"in_reply_to":"6839e209_0aa2a859","updated":"2024-10-31 17:44:27.000000000","message":"Right, it very well might be. It is used only in the testing for when we check for equality.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":292,"context_line":"                    account_container"},{"line_number":293,"context_line":"                )"},{"line_number":294,"context_line":"                if not self.timestamp_to_account_containers[old_timestamp]:"},{"line_number":295,"context_line":"                    del self.timestamp_to_account_containers[old_timestamp]"},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"                # Remove the old_timestamp from"},{"line_number":298,"context_line":"                # account_container_to_timestamp"}],"source_content_type":"text/x-python","patch_set":10,"id":"dcd3064f_14df7faf","line":295,"updated":"2024-10-17 15:56:28.000000000","message":"even if we *don\u0027t* evict a key here, I assume the cache size limit is on `account_container_to_timestamp`; worst case we\u0027re moving the ac from one timestamp set to another one.\n\n```\nsum(len(ac) for ts, ac in self.timestamp_to_acs.items()) \u003d\u003d len(self.ts_to_ac)\n```","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":292,"context_line":"                    account_container"},{"line_number":293,"context_line":"                )"},{"line_number":294,"context_line":"                if not self.timestamp_to_account_containers[old_timestamp]:"},{"line_number":295,"context_line":"                    del self.timestamp_to_account_containers[old_timestamp]"},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"                # Remove the old_timestamp from"},{"line_number":298,"context_line":"                # account_container_to_timestamp"}],"source_content_type":"text/x-python","patch_set":10,"id":"0cbb9112_a8a4540c","line":295,"in_reply_to":"dcd3064f_14df7faf","updated":"2024-10-31 17:44:27.000000000","message":"Acknowledged","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":300,"context_line":""},{"line_number":301,"context_line":"                # Add the new timestamp"},{"line_number":302,"context_line":"                if timestamp not in self.timestamp_to_account_containers:"},{"line_number":303,"context_line":"                    self.timestamp_to_account_containers[timestamp] \u003d set()"},{"line_number":304,"context_line":""},{"line_number":305,"context_line":"                self.timestamp_to_account_containers[timestamp].add("},{"line_number":306,"context_line":"                    account_container"}],"source_content_type":"text/x-python","patch_set":10,"id":"44d9a6cf_9f137855","line":303,"updated":"2024-10-17 15:56:28.000000000","message":"oic, we evict the ac from from the *old* timestamp (and probably delete the key) and then we need to create a *new* key for this older timestamp.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":300,"context_line":""},{"line_number":301,"context_line":"                # Add the new timestamp"},{"line_number":302,"context_line":"                if timestamp not in self.timestamp_to_account_containers:"},{"line_number":303,"context_line":"                    self.timestamp_to_account_containers[timestamp] \u003d set()"},{"line_number":304,"context_line":""},{"line_number":305,"context_line":"                self.timestamp_to_account_containers[timestamp].add("},{"line_number":306,"context_line":"                    account_container"}],"source_content_type":"text/x-python","patch_set":10,"id":"818fda1b_5a159a2c","line":303,"in_reply_to":"44d9a6cf_9f137855","updated":"2024-10-31 17:44:27.000000000","message":"Yes, exactly","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":307,"context_line":"                )"},{"line_number":308,"context_line":"                self.account_container_to_timestamp[account_container] \u003d ("},{"line_number":309,"context_line":"                    timestamp"},{"line_number":310,"context_line":"                )"},{"line_number":311,"context_line":"        else:"},{"line_number":312,"context_line":"            if timestamp not in self.timestamp_to_account_containers:"},{"line_number":313,"context_line":"                self.timestamp_to_account_containers[timestamp] \u003d set()"}],"source_content_type":"text/x-python","patch_set":10,"id":"acfc96eb_a9caa678","line":310,"updated":"2024-10-17 15:56:28.000000000","message":"this is always updating the value of an existing key; I think we could return here - but I\u0027m not sure it\u0027d be significantly clearer.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":307,"context_line":"                )"},{"line_number":308,"context_line":"                self.account_container_to_timestamp[account_container] \u003d ("},{"line_number":309,"context_line":"                    timestamp"},{"line_number":310,"context_line":"                )"},{"line_number":311,"context_line":"        else:"},{"line_number":312,"context_line":"            if timestamp not in self.timestamp_to_account_containers:"},{"line_number":313,"context_line":"                self.timestamp_to_account_containers[timestamp] \u003d set()"}],"source_content_type":"text/x-python","patch_set":10,"id":"ec58f2fd_86bebea5","line":310,"in_reply_to":"acfc96eb_a9caa678","updated":"2024-10-31 17:44:27.000000000","message":"Acknowledged","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":315,"context_line":"            self.timestamp_to_account_containers[timestamp].add("},{"line_number":316,"context_line":"                account_container"},{"line_number":317,"context_line":"            )"},{"line_number":318,"context_line":"            self.account_container_to_timestamp[account_container] \u003d timestamp"},{"line_number":319,"context_line":""},{"line_number":320,"context_line":"        # Check size and evict newest account_container(s) if necessary"},{"line_number":321,"context_line":"        if ("}],"source_content_type":"text/x-python","patch_set":10,"id":"3484b58d_fe2bb7f6","line":318,"updated":"2024-10-17 15:56:28.000000000","message":"for some reason I\u0027m surprised adding new account, container is unconditional - I would assume we\u0027d want to check if the timestamp is actually any more interesting than what we already have.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":315,"context_line":"            self.timestamp_to_account_containers[timestamp].add("},{"line_number":316,"context_line":"                account_container"},{"line_number":317,"context_line":"            )"},{"line_number":318,"context_line":"            self.account_container_to_timestamp[account_container] \u003d timestamp"},{"line_number":319,"context_line":""},{"line_number":320,"context_line":"        # Check size and evict newest account_container(s) if necessary"},{"line_number":321,"context_line":"        if ("}],"source_content_type":"text/x-python","patch_set":10,"id":"d0a0d48f_d1eb6a69","line":318,"in_reply_to":"3484b58d_fe2bb7f6","updated":"2024-10-31 17:44:27.000000000","message":"Acknowledged","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":324,"context_line":"        ):"},{"line_number":325,"context_line":"            newest_account_containers \u003d ("},{"line_number":326,"context_line":"                self.timestamp_to_account_containers.popitem(-1)[1]"},{"line_number":327,"context_line":"            )"},{"line_number":328,"context_line":"            for newest_account_container in newest_account_containers:"},{"line_number":329,"context_line":"                del self.account_container_to_timestamp["},{"line_number":330,"context_line":"                    newest_account_container"}],"source_content_type":"text/x-python","patch_set":10,"id":"b4a0b082_ff32b6fc","line":327,"updated":"2024-10-17 15:56:28.000000000","message":"this *could* be the one we just added right?","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":324,"context_line":"        ):"},{"line_number":325,"context_line":"            newest_account_containers \u003d ("},{"line_number":326,"context_line":"                self.timestamp_to_account_containers.popitem(-1)[1]"},{"line_number":327,"context_line":"            )"},{"line_number":328,"context_line":"            for newest_account_container in newest_account_containers:"},{"line_number":329,"context_line":"                del self.account_container_to_timestamp["},{"line_number":330,"context_line":"                    newest_account_container"}],"source_content_type":"text/x-python","patch_set":10,"id":"ed2076be_a5009105","line":327,"in_reply_to":"b4a0b082_ff32b6fc","updated":"2024-10-31 17:44:27.000000000","message":"Yes it could.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":341,"context_line":"            for account_container in account_containers:"},{"line_number":342,"context_line":"                oldest_timestamp_account_containers.append("},{"line_number":343,"context_line":"                    (timestamp, account_container)"},{"line_number":344,"context_line":"                )"},{"line_number":345,"context_line":"                if len(oldest_timestamp_account_containers) \u003d\u003d n:"},{"line_number":346,"context_line":"                    return oldest_timestamp_account_containers"},{"line_number":347,"context_line":"        return oldest_timestamp_account_containers"}],"source_content_type":"text/x-python","patch_set":10,"id":"3237193b_c1e94ce8","line":344,"updated":"2024-10-17 15:56:28.000000000","message":"this would be better as a descriptive dictionary:\n\n```\nvagrant@saio:~$ cat /var/cache/swift/node1/object.recon | python -m json.tool\n{\n    \"object_updater_stats\": {\n        \"async_pending_account_container_count\": 6,\n        \"async_pending_oldest_age_seconds\": 348.76739263534546,\n        \"async_pending_oldest_timestamp_account_containers\": [\n            [\n                \"1729173359.15710\",\n                [\n                    \"AUTH_user001\",\n                    \"cont000\"\n                ]\n            ],\n            [\n                \"1729173359.38328\",\n                [\n                    \"AUTH_user001\",\n                    \"cont001\"\n                ]\n            ],\n            [\n                \"1729173359.66363\",\n                [\n                    \"AUTH_user001\",\n                    \"cont002\"\n                ]\n            ],\n            [\n                \"1729173359.93594\",\n                [\n                    \"AUTH_user001\",\n                    \"cont003\"\n                ]\n            ],\n            [\n                \"1729173360.34911\",\n                [\n                    \"AUTH_user002\",\n                    \"cont000\"\n                ]\n            ]\n        ]\n    },\n    \"object_updater_sweep\": 0.41168832778930664\n}\n```","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":341,"context_line":"            for account_container in account_containers:"},{"line_number":342,"context_line":"                oldest_timestamp_account_containers.append("},{"line_number":343,"context_line":"                    (timestamp, account_container)"},{"line_number":344,"context_line":"                )"},{"line_number":345,"context_line":"                if len(oldest_timestamp_account_containers) \u003d\u003d n:"},{"line_number":346,"context_line":"                    return oldest_timestamp_account_containers"},{"line_number":347,"context_line":"        return oldest_timestamp_account_containers"}],"source_content_type":"text/x-python","patch_set":10,"id":"e21491ef_1d37907e","line":344,"in_reply_to":"3237193b_c1e94ce8","updated":"2024-10-31 17:44:27.000000000","message":"Done","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":372,"context_line":"                 unlinks\u003d0, redirects\u003d0, skips\u003d0, deferrals\u003d0, drains\u003d0,"},{"line_number":373,"context_line":"                 oldest_async_pendings\u003d("},{"line_number":374,"context_line":"                     AccountContainerOldestAsyncPendingManager()"},{"line_number":375,"context_line":"                 )):"},{"line_number":376,"context_line":"        self.errors \u003d errors"},{"line_number":377,"context_line":"        self.failures \u003d failures"},{"line_number":378,"context_line":"        self.quarantines \u003d quarantines"}],"source_content_type":"text/x-python","patch_set":10,"id":"11dadd09_300f1974","line":375,"updated":"2024-10-17 15:56:28.000000000","message":"meh, mutable object in kwargs is bad.  I guess you need the kwarg for `copy` tho.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":true,"context_lines":[{"line_number":372,"context_line":"                 unlinks\u003d0, redirects\u003d0, skips\u003d0, deferrals\u003d0, drains\u003d0,"},{"line_number":373,"context_line":"                 oldest_async_pendings\u003d("},{"line_number":374,"context_line":"                     AccountContainerOldestAsyncPendingManager()"},{"line_number":375,"context_line":"                 )):"},{"line_number":376,"context_line":"        self.errors \u003d errors"},{"line_number":377,"context_line":"        self.failures \u003d failures"},{"line_number":378,"context_line":"        self.quarantines \u003d quarantines"}],"source_content_type":"text/x-python","patch_set":10,"id":"fadf4010_fb90329e","line":375,"in_reply_to":"11dadd09_300f1974","updated":"2024-10-31 17:44:27.000000000","message":"Right, let me know if you think this must change.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"fbb3ee08c6e52c352bf300992335c8d10a6571c2","unresolved":false,"context_lines":[{"line_number":372,"context_line":"                 unlinks\u003d0, redirects\u003d0, skips\u003d0, deferrals\u003d0, drains\u003d0,"},{"line_number":373,"context_line":"                 oldest_async_pendings\u003d("},{"line_number":374,"context_line":"                     AccountContainerOldestAsyncPendingManager()"},{"line_number":375,"context_line":"                 )):"},{"line_number":376,"context_line":"        self.errors \u003d errors"},{"line_number":377,"context_line":"        self.failures \u003d failures"},{"line_number":378,"context_line":"        self.quarantines \u003d quarantines"}],"source_content_type":"text/x-python","patch_set":10,"id":"a699c1fa_7f87ddd5","line":375,"in_reply_to":"fadf4010_fb90329e","updated":"2024-11-01 16:15:01.000000000","message":"Done","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"d4ec02b52b4ae8bb202648fa13c1283c1d213b91","unresolved":false,"context_lines":[{"line_number":372,"context_line":"                 unlinks\u003d0, redirects\u003d0, skips\u003d0, deferrals\u003d0, drains\u003d0,"},{"line_number":373,"context_line":"                 oldest_async_pendings\u003d("},{"line_number":374,"context_line":"                     AccountContainerOldestAsyncPendingManager()"},{"line_number":375,"context_line":"                 )):"},{"line_number":376,"context_line":"        self.errors \u003d errors"},{"line_number":377,"context_line":"        self.failures \u003d failures"},{"line_number":378,"context_line":"        self.quarantines \u003d quarantines"}],"source_content_type":"text/x-python","patch_set":10,"id":"a4ba7e36_21d2b8c4","line":375,"in_reply_to":"fadf4010_fb90329e","updated":"2024-11-01 16:14:51.000000000","message":"Updated.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":401,"context_line":"                          self.deferrals - other.deferrals,"},{"line_number":402,"context_line":"                          self.drains - other.drains,"},{"line_number":403,"context_line":"                          self.oldest_async_pendings"},{"line_number":404,"context_line":"                          )"},{"line_number":405,"context_line":""},{"line_number":406,"context_line":"    def reset(self):"},{"line_number":407,"context_line":"        self.errors \u003d 0"}],"source_content_type":"text/x-python","patch_set":10,"id":"f215e2a4_980fff46","line":404,"updated":"2024-10-17 15:56:28.000000000","message":"wtf is \"since\" - does the oldest-async manager even make sense with `since`?  Maybe the new object should get a fresh one?","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":401,"context_line":"                          self.deferrals - other.deferrals,"},{"line_number":402,"context_line":"                          self.drains - other.drains,"},{"line_number":403,"context_line":"                          self.oldest_async_pendings"},{"line_number":404,"context_line":"                          )"},{"line_number":405,"context_line":""},{"line_number":406,"context_line":"    def reset(self):"},{"line_number":407,"context_line":"        self.errors \u003d 0"}],"source_content_type":"text/x-python","patch_set":10,"id":"d83f8cf7_56ce749b","line":404,"in_reply_to":"f215e2a4_980fff46","updated":"2024-10-31 17:44:27.000000000","message":"Right, I believe right now it is getting the fresh one.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":415,"context_line":"        self.drains \u003d 0"},{"line_number":416,"context_line":"        self.oldest_async_pendings \u003d ("},{"line_number":417,"context_line":"            AccountContainerOldestAsyncPendingManager()"},{"line_number":418,"context_line":"        )"},{"line_number":419,"context_line":""},{"line_number":420,"context_line":"    def __str__(self):"},{"line_number":421,"context_line":"        keys \u003d ("}],"source_content_type":"text/x-python","patch_set":10,"id":"c3074253_f7b9384f","line":418,"updated":"2024-10-17 15:56:28.000000000","message":"I think I\u0027d prefer, `\\` over parens - the parens tend to make me think of tuple literals.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":415,"context_line":"        self.drains \u003d 0"},{"line_number":416,"context_line":"        self.oldest_async_pendings \u003d ("},{"line_number":417,"context_line":"            AccountContainerOldestAsyncPendingManager()"},{"line_number":418,"context_line":"        )"},{"line_number":419,"context_line":""},{"line_number":420,"context_line":"    def __str__(self):"},{"line_number":421,"context_line":"        keys \u003d ("}],"source_content_type":"text/x-python","patch_set":10,"id":"fcf09307_9e8a3923","line":418,"in_reply_to":"c3074253_f7b9384f","updated":"2024-10-31 17:44:27.000000000","message":"I try to avoid \\ since I don\u0027t love how it looks and since it can be slightly dangerous at times (https://stackoverflow.com/questions/48830681/is-line-continuation-with-backslash-dangerous-in-python) but I understand where you\u0027re coming from with the tuple literal comment, will modify.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":578,"context_line":"            object_updater_stats \u003d {"},{"line_number":579,"context_line":"                \u0027async_pending_oldest_age_seconds\u0027: ("},{"line_number":580,"context_line":"                    self.stats.oldest_async_pendings.get_oldest_timestamp_age()"},{"line_number":581,"context_line":"                ),"},{"line_number":582,"context_line":"                \u0027async_pending_account_container_count\u0027: ("},{"line_number":583,"context_line":"                    len("},{"line_number":584,"context_line":"                        self.stats.oldest_async_pendings"}],"source_content_type":"text/x-python","patch_set":10,"id":"9e2e639e_6cdad8fa","line":581,"updated":"2024-10-17 15:56:28.000000000","message":"it might be useful to include the time offset from which this delta is calculated; honestly it might be better to do the translation at collection time.\n\nI think we could add translation in /recon - but our SRE just reads the json files directly.  We should ask them what would be most convienent.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":578,"context_line":"            object_updater_stats \u003d {"},{"line_number":579,"context_line":"                \u0027async_pending_oldest_age_seconds\u0027: ("},{"line_number":580,"context_line":"                    self.stats.oldest_async_pendings.get_oldest_timestamp_age()"},{"line_number":581,"context_line":"                ),"},{"line_number":582,"context_line":"                \u0027async_pending_account_container_count\u0027: ("},{"line_number":583,"context_line":"                    len("},{"line_number":584,"context_line":"                        self.stats.oldest_async_pendings"}],"source_content_type":"text/x-python","patch_set":10,"id":"ccb2b5e0_8810439c","line":581,"in_reply_to":"9e2e639e_6cdad8fa","updated":"2024-10-31 17:44:27.000000000","message":"Added both the time offset and the collection time.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":582,"context_line":"                \u0027async_pending_account_container_count\u0027: ("},{"line_number":583,"context_line":"                    len("},{"line_number":584,"context_line":"                        self.stats.oldest_async_pendings"},{"line_number":585,"context_line":"                            .account_container_to_timestamp"},{"line_number":586,"context_line":"                    )"},{"line_number":587,"context_line":"                ),"},{"line_number":588,"context_line":"                \u0027async_pending_oldest_timestamp_account_containers\u0027: ("}],"source_content_type":"text/x-python","patch_set":10,"id":"265913a7_fddb9bca","line":585,"updated":"2024-10-17 15:56:28.000000000","message":"```\n(Pdb) recon[\u0027object_updater_stats\u0027][\u0027async_pending_account_container_count\u0027]\n11\n(Pdb) len(recon[\u0027object_updater_stats\u0027][\u0027async_pending_oldest_timestamp_account_containers\u0027])\n5\n```\n\noic, 100 is the default stored (this node only had 11) - 5 is the number dumped (so the newest stored are not reported)","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":582,"context_line":"                \u0027async_pending_account_container_count\u0027: ("},{"line_number":583,"context_line":"                    len("},{"line_number":584,"context_line":"                        self.stats.oldest_async_pendings"},{"line_number":585,"context_line":"                            .account_container_to_timestamp"},{"line_number":586,"context_line":"                    )"},{"line_number":587,"context_line":"                ),"},{"line_number":588,"context_line":"                \u0027async_pending_oldest_timestamp_account_containers\u0027: ("}],"source_content_type":"text/x-python","patch_set":10,"id":"fc7083f7_00f866b0","line":585,"in_reply_to":"265913a7_fddb9bca","updated":"2024-10-31 17:44:27.000000000","message":"Acknowledged","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"67726534bf6b4a2930318d113960b92826c5b5be","unresolved":true,"context_lines":[{"line_number":643,"context_line":"            },"},{"line_number":644,"context_line":"            self.rcache,"},{"line_number":645,"context_line":"            self.logger,"},{"line_number":646,"context_line":"        )"},{"line_number":647,"context_line":""},{"line_number":648,"context_line":"    def _load_update(self, device, update_path):"},{"line_number":649,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":10,"id":"cbbe4aa8_f1cf5686","line":646,"updated":"2024-10-17 15:56:28.000000000","message":"please refactor this so there\u0027s a single code path for dump_recon between run_forever and run_once","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":643,"context_line":"            },"},{"line_number":644,"context_line":"            self.rcache,"},{"line_number":645,"context_line":"            self.logger,"},{"line_number":646,"context_line":"        )"},{"line_number":647,"context_line":""},{"line_number":648,"context_line":"    def _load_update(self, device, update_path):"},{"line_number":649,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":10,"id":"cd73bb1b_8c46345e","line":646,"in_reply_to":"cbbe4aa8_f1cf5686","updated":"2024-10-31 17:44:27.000000000","message":"Done.","commit_id":"0cb40eb5b4c62648bfc3583c9c2539ac2261cc50"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3fabc093ccc6ea9b93b8e60f62a8914c9e6bf622","unresolved":true,"context_lines":[{"line_number":216,"context_line":"    __next__ \u003d next"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"class AccountContainerOldestAsyncPendingManager:"},{"line_number":220,"context_line":"    \"\"\""},{"line_number":221,"context_line":"    Manages the tracking of the oldest async pending updates for each"},{"line_number":222,"context_line":"    account-container pair using a sorted list for timestamps."}],"source_content_type":"text/x-python","patch_set":16,"id":"5fd63901_2032b598","line":219,"updated":"2024-10-26 00:10:30.000000000","message":"how about naming it as ``OldestAsyncPendingTracker``?","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1d5730f250615cbbc6bc603daa598f1affc76fe1","unresolved":false,"context_lines":[{"line_number":216,"context_line":"    __next__ \u003d next"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"class AccountContainerOldestAsyncPendingManager:"},{"line_number":220,"context_line":"    \"\"\""},{"line_number":221,"context_line":"    Manages the tracking of the oldest async pending updates for each"},{"line_number":222,"context_line":"    account-container pair using a sorted list for timestamps."}],"source_content_type":"text/x-python","patch_set":16,"id":"edd9faea_ba3a9760","line":219,"in_reply_to":"5fd63901_2032b598","updated":"2024-11-01 16:11:41.000000000","message":"Done","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":216,"context_line":"    __next__ \u003d next"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"class AccountContainerOldestAsyncPendingManager:"},{"line_number":220,"context_line":"    \"\"\""},{"line_number":221,"context_line":"    Manages the tracking of the oldest async pending updates for each"},{"line_number":222,"context_line":"    account-container pair using a sorted list for timestamps."}],"source_content_type":"text/x-python","patch_set":16,"id":"fa621b8b_98d00bdd","line":219,"in_reply_to":"5fd63901_2032b598","updated":"2024-10-31 17:44:27.000000000","message":"Sure!","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3fabc093ccc6ea9b93b8e60f62a8914c9e6bf622","unresolved":true,"context_lines":[{"line_number":228,"context_line":"    \"\"\""},{"line_number":229,"context_line":"    def __init__("},{"line_number":230,"context_line":"        self,"},{"line_number":231,"context_line":"        max_account_container_count\u003d100,"},{"line_number":232,"context_line":"        account_container_recon_dump_count\u003d5,"},{"line_number":233,"context_line":"    ):"},{"line_number":234,"context_line":"        self.max_account_container_count \u003d max_account_container_count"}],"source_content_type":"text/x-python","patch_set":16,"id":"9d60d2d4_21f8b3ed","line":231,"updated":"2024-10-26 00:10:30.000000000","message":"how about renaming those variables to be simpler, and explain what an entry is in the docstring?\n\n```\nmax_account_container_count -\u003e max_entries\naccount_container_recon_dump_count -\u003e dump_count\nsorted_timestamp_account_containers -\u003e sorted_entries\naccount_container_to_timestamp -\u003e ac_to_timestamp\n```","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1d5730f250615cbbc6bc603daa598f1affc76fe1","unresolved":false,"context_lines":[{"line_number":228,"context_line":"    \"\"\""},{"line_number":229,"context_line":"    def __init__("},{"line_number":230,"context_line":"        self,"},{"line_number":231,"context_line":"        max_account_container_count\u003d100,"},{"line_number":232,"context_line":"        account_container_recon_dump_count\u003d5,"},{"line_number":233,"context_line":"    ):"},{"line_number":234,"context_line":"        self.max_account_container_count \u003d max_account_container_count"}],"source_content_type":"text/x-python","patch_set":16,"id":"bccbd067_bf21d6d8","line":231,"in_reply_to":"9d60d2d4_21f8b3ed","updated":"2024-11-01 16:11:41.000000000","message":"Done","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":228,"context_line":"    \"\"\""},{"line_number":229,"context_line":"    def __init__("},{"line_number":230,"context_line":"        self,"},{"line_number":231,"context_line":"        max_account_container_count\u003d100,"},{"line_number":232,"context_line":"        account_container_recon_dump_count\u003d5,"},{"line_number":233,"context_line":"    ):"},{"line_number":234,"context_line":"        self.max_account_container_count \u003d max_account_container_count"}],"source_content_type":"text/x-python","patch_set":16,"id":"194eaa92_b9827990","line":231,"in_reply_to":"9d60d2d4_21f8b3ed","updated":"2024-10-31 17:44:27.000000000","message":"Sure!","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3fabc093ccc6ea9b93b8e60f62a8914c9e6bf622","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        self.max_account_container_count \u003d max_account_container_count"},{"line_number":235,"context_line":"        self.account_container_recon_dump_count \u003d \\"},{"line_number":236,"context_line":"            account_container_recon_dump_count"},{"line_number":237,"context_line":"        self.sorted_timestamp_account_containers \u003d []"},{"line_number":238,"context_line":"        self.account_container_to_timestamp \u003d {}"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    def __eq__(self, other):"}],"source_content_type":"text/x-python","patch_set":16,"id":"eaa94231_ba47b383","line":237,"updated":"2024-10-26 00:10:30.000000000","message":"this is the main data structure, and insort() on a list is O(n) for insertion and deletion. But it\u0027s not a big deal, since by default this list only has 100 elements in max. And SortedList from sortedcontainers (which has O(lgn)) will introduce a new library to install.","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1d5730f250615cbbc6bc603daa598f1affc76fe1","unresolved":false,"context_lines":[{"line_number":234,"context_line":"        self.max_account_container_count \u003d max_account_container_count"},{"line_number":235,"context_line":"        self.account_container_recon_dump_count \u003d \\"},{"line_number":236,"context_line":"            account_container_recon_dump_count"},{"line_number":237,"context_line":"        self.sorted_timestamp_account_containers \u003d []"},{"line_number":238,"context_line":"        self.account_container_to_timestamp \u003d {}"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    def __eq__(self, other):"}],"source_content_type":"text/x-python","patch_set":16,"id":"93d80a72_9afabd9e","line":237,"in_reply_to":"eaa94231_ba47b383","updated":"2024-11-01 16:11:41.000000000","message":"Acknowledged","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":234,"context_line":"        self.max_account_container_count \u003d max_account_container_count"},{"line_number":235,"context_line":"        self.account_container_recon_dump_count \u003d \\"},{"line_number":236,"context_line":"            account_container_recon_dump_count"},{"line_number":237,"context_line":"        self.sorted_timestamp_account_containers \u003d []"},{"line_number":238,"context_line":"        self.account_container_to_timestamp \u003d {}"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    def __eq__(self, other):"}],"source_content_type":"text/x-python","patch_set":16,"id":"5132e63d_ebdb7da0","line":237,"in_reply_to":"eaa94231_ba47b383","updated":"2024-10-31 17:44:27.000000000","message":"Right :)","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3fabc093ccc6ea9b93b8e60f62a8914c9e6bf622","unresolved":true,"context_lines":[{"line_number":465,"context_line":"                                         DEFAULT_RECON_CACHE_PATH)"},{"line_number":466,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path, RECON_OBJECT_FILE)"},{"line_number":467,"context_line":"        default_manager \u003d AccountContainerOldestAsyncPendingManager()"},{"line_number":468,"context_line":"        max_account_container_count \u003d config_positive_int_value("},{"line_number":469,"context_line":"            conf.get("},{"line_number":470,"context_line":"                \u0027max_account_container_count\u0027,"},{"line_number":471,"context_line":"                default_manager.max_account_container_count,"}],"source_content_type":"text/x-python","patch_set":16,"id":"c3aa1fea_0caf37cb","line":468,"updated":"2024-10-26 00:10:30.000000000","message":"please add the description of those new settings in the sample config file: https://github.com/NVIDIA/swift/blob/master/etc/object-server.conf-sample#L517","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":465,"context_line":"                                         DEFAULT_RECON_CACHE_PATH)"},{"line_number":466,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path, RECON_OBJECT_FILE)"},{"line_number":467,"context_line":"        default_manager \u003d AccountContainerOldestAsyncPendingManager()"},{"line_number":468,"context_line":"        max_account_container_count \u003d config_positive_int_value("},{"line_number":469,"context_line":"            conf.get("},{"line_number":470,"context_line":"                \u0027max_account_container_count\u0027,"},{"line_number":471,"context_line":"                default_manager.max_account_container_count,"}],"source_content_type":"text/x-python","patch_set":16,"id":"1b77a11e_e65e1605","line":468,"in_reply_to":"c3aa1fea_0caf37cb","updated":"2024-10-31 17:44:27.000000000","message":"Done","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1d5730f250615cbbc6bc603daa598f1affc76fe1","unresolved":false,"context_lines":[{"line_number":465,"context_line":"                                         DEFAULT_RECON_CACHE_PATH)"},{"line_number":466,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path, RECON_OBJECT_FILE)"},{"line_number":467,"context_line":"        default_manager \u003d AccountContainerOldestAsyncPendingManager()"},{"line_number":468,"context_line":"        max_account_container_count \u003d config_positive_int_value("},{"line_number":469,"context_line":"            conf.get("},{"line_number":470,"context_line":"                \u0027max_account_container_count\u0027,"},{"line_number":471,"context_line":"                default_manager.max_account_container_count,"}],"source_content_type":"text/x-python","patch_set":16,"id":"954a65d0_1f74173f","line":468,"in_reply_to":"c3aa1fea_0caf37cb","updated":"2024-11-01 16:11:41.000000000","message":"Done","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"3fabc093ccc6ea9b93b8e60f62a8914c9e6bf622","unresolved":true,"context_lines":[{"line_number":467,"context_line":"        default_manager \u003d AccountContainerOldestAsyncPendingManager()"},{"line_number":468,"context_line":"        max_account_container_count \u003d config_positive_int_value("},{"line_number":469,"context_line":"            conf.get("},{"line_number":470,"context_line":"                \u0027max_account_container_count\u0027,"},{"line_number":471,"context_line":"                default_manager.max_account_container_count,"},{"line_number":472,"context_line":"            )"},{"line_number":473,"context_line":"        )"}],"source_content_type":"text/x-python","patch_set":16,"id":"0daee583_71300cad","line":470,"updated":"2024-10-26 00:10:30.000000000","message":"the name of those new settings are not that descriptive. how about?\n\n```\nmax_account_container_count -\u003e async_tracker_max_entries\naccount_container_recon_dump_count -\u003e async_tracker_dump_count\n```","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"1d5730f250615cbbc6bc603daa598f1affc76fe1","unresolved":false,"context_lines":[{"line_number":467,"context_line":"        default_manager \u003d AccountContainerOldestAsyncPendingManager()"},{"line_number":468,"context_line":"        max_account_container_count \u003d config_positive_int_value("},{"line_number":469,"context_line":"            conf.get("},{"line_number":470,"context_line":"                \u0027max_account_container_count\u0027,"},{"line_number":471,"context_line":"                default_manager.max_account_container_count,"},{"line_number":472,"context_line":"            )"},{"line_number":473,"context_line":"        )"}],"source_content_type":"text/x-python","patch_set":16,"id":"168a7b1c_7972c81a","line":470,"in_reply_to":"0daee583_71300cad","updated":"2024-11-01 16:11:41.000000000","message":"Done","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"406cc69efce930ff3b91d87baf1bb7e2e85e0aea","unresolved":false,"context_lines":[{"line_number":467,"context_line":"        default_manager \u003d AccountContainerOldestAsyncPendingManager()"},{"line_number":468,"context_line":"        max_account_container_count \u003d config_positive_int_value("},{"line_number":469,"context_line":"            conf.get("},{"line_number":470,"context_line":"                \u0027max_account_container_count\u0027,"},{"line_number":471,"context_line":"                default_manager.max_account_container_count,"},{"line_number":472,"context_line":"            )"},{"line_number":473,"context_line":"        )"}],"source_content_type":"text/x-python","patch_set":16,"id":"da9b82a5_46658585","line":470,"in_reply_to":"0daee583_71300cad","updated":"2024-10-31 17:44:27.000000000","message":"Sure!","commit_id":"ad6f21325c85fd93b4adfdccb35fe94a14ca7487"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":22,"context_line":"import time"},{"line_number":23,"context_line":"import uuid"},{"line_number":24,"context_line":"from random import random, shuffle"},{"line_number":25,"context_line":"from bisect import insort"},{"line_number":26,"context_line":"from collections import deque"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"from eventlet import spawn, Timeout"}],"source_content_type":"text/x-python","patch_set":29,"id":"ecc3b2de_db89e49d","line":25,"range":{"start_line":25,"start_character":19,"end_line":25,"end_character":25},"updated":"2024-11-01 17:57:23.000000000","message":"I\u0027d completely forgotten that `bisect` has this method! Maybe because that\n\n\u003e Keep in mind that the _O(log n)_ search is dominated by the slow _O(n)_ insertion step.\n\nnote scared me off from using it. Maybe [a priority queue based on `heapq`](https://docs.python.org/3/library/heapq.html#priority-queue-implementation-notes) would be a better choice of structure?\n\nThough it probably doesn\u0027t matter much as long as we keep the max entries small enough -- I bet we could go two, even three orders of magnitude higher than the default without too much trouble.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":22,"context_line":"import time"},{"line_number":23,"context_line":"import uuid"},{"line_number":24,"context_line":"from random import random, shuffle"},{"line_number":25,"context_line":"from bisect import insort"},{"line_number":26,"context_line":"from collections import deque"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"from eventlet import spawn, Timeout"}],"source_content_type":"text/x-python","patch_set":29,"id":"d36ba0f0_e0d751df","line":25,"range":{"start_line":25,"start_character":19,"end_line":25,"end_character":25},"in_reply_to":"ecc3b2de_db89e49d","updated":"2024-11-01 21:01:21.000000000","message":"Yes, I used a priority queue was in my very first implementation. There are 2 main issues with this implementation. First, you would need 2 priority queues. The first priority queue is needed if we wanted to support the ability for __oldest__ timestamps (smallest float values) to be retrieved in O(log n) time. This is needed when we call the get_oldest_timestamp `O(log(n))` or when calling get_\"k\"_oldest_timestamp_acs `O(k*log n)`. The other priority queue is needed to evict the __newest__ (largest float values) when the maximum has been reached. Since we can\u0027t have a heap that is both a min and a max heap, we could have both a heap and max heap and perform lazy deletion with our heap values upon eviction. This could work by using our ac_to_timestamp dict as our source of truth and using while loops to remove values in the heaps that are no longer valid (still amortized `O(log(n))` removals). This complexity of data structure seemed overly complex, but it would allow us to be nearly as asymptotically performant as the SortedDict / SortedList solutions while only using imports from the standard library. However, since I don\u0027t anticipate our max_entries being large, I don\u0027t think this is currently worth worrying about.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        dump_count\u003dASYNC_TRACKER_DUMP_COUNT,"},{"line_number":235,"context_line":"    ):"},{"line_number":236,"context_line":"        self.max_entries \u003d max_entries"},{"line_number":237,"context_line":"        self.dump_count \u003d dump_count"},{"line_number":238,"context_line":"        self.sorted_entries \u003d []"},{"line_number":239,"context_line":"        self.ac_to_timestamp \u003d {}"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":29,"id":"fe6aac19_50cf619e","line":237,"updated":"2024-11-01 17:57:23.000000000","message":"We only use this to provide a default to `get_n_oldest_timestamp_acs` -- do we actually want/need this on this data structure? Or should we just hang `dump_count` off the `ObjectUpdater` as we\u0027re reading it out of the config?","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":234,"context_line":"        dump_count\u003dASYNC_TRACKER_DUMP_COUNT,"},{"line_number":235,"context_line":"    ):"},{"line_number":236,"context_line":"        self.max_entries \u003d max_entries"},{"line_number":237,"context_line":"        self.dump_count \u003d dump_count"},{"line_number":238,"context_line":"        self.sorted_entries \u003d []"},{"line_number":239,"context_line":"        self.ac_to_timestamp \u003d {}"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":29,"id":"1f007c92_5e1e3d8d","line":237,"in_reply_to":"fe6aac19_50cf619e","updated":"2024-11-01 21:01:21.000000000","message":"No, we don\u0027t need it on the data structure. I just thought it made some sense to have a reasonable default instead of needing to specify the dump_count each time we call get_n_oldest_timestamp_acs. I changed it so that `n` is a positional argument.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":288,"context_line":""},{"line_number":289,"context_line":"    def get_oldest_timestamp_age(self, current_time\u003dNone):"},{"line_number":290,"context_line":"        if current_time is None:"},{"line_number":291,"context_line":"            current_time \u003d time.time()"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"        oldest_timestamp \u003d self.get_oldest_timestamp()"},{"line_number":294,"context_line":"        if oldest_timestamp is not None:"}],"source_content_type":"text/x-python","patch_set":29,"id":"cc1f3f48_3af15459","line":291,"updated":"2024-11-01 17:57:23.000000000","message":"We always take this branch, yeah? Should we just drop the `current_time` arg?\n\nAlternatively, I could see it being nice to get some consistency with the cycle-end time -- maybe `dump_recon` should take start and end times instead of `elapsed`, and pass that end time to this call...","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":288,"context_line":""},{"line_number":289,"context_line":"    def get_oldest_timestamp_age(self, current_time\u003dNone):"},{"line_number":290,"context_line":"        if current_time is None:"},{"line_number":291,"context_line":"            current_time \u003d time.time()"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"        oldest_timestamp \u003d self.get_oldest_timestamp()"},{"line_number":294,"context_line":"        if oldest_timestamp is not None:"}],"source_content_type":"text/x-python","patch_set":29,"id":"d2c70a3d_116c9e99","line":291,"in_reply_to":"cc1f3f48_3af15459","updated":"2024-11-01 21:01:21.000000000","message":"Sure, I did the former and just dropped the `current_time` arg.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":291,"context_line":"            current_time \u003d time.time()"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"        oldest_timestamp \u003d self.get_oldest_timestamp()"},{"line_number":294,"context_line":"        if oldest_timestamp is not None:"},{"line_number":295,"context_line":"            return current_time - oldest_timestamp"},{"line_number":296,"context_line":"        return float(\u0027inf\u0027)"},{"line_number":297,"context_line":""}],"source_content_type":"text/x-python","patch_set":29,"id":"72bd1f86_cf998fb0","line":294,"updated":"2024-11-01 17:57:23.000000000","message":"This is currently never `None`, though, is it?","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":291,"context_line":"            current_time \u003d time.time()"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"        oldest_timestamp \u003d self.get_oldest_timestamp()"},{"line_number":294,"context_line":"        if oldest_timestamp is not None:"},{"line_number":295,"context_line":"            return current_time - oldest_timestamp"},{"line_number":296,"context_line":"        return float(\u0027inf\u0027)"},{"line_number":297,"context_line":""}],"source_content_type":"text/x-python","patch_set":29,"id":"b26131dd_2f916dfd","line":294,"in_reply_to":"72bd1f86_cf998fb0","updated":"2024-11-01 21:01:21.000000000","message":"Right, we always return some float. However, with the newest version, `oldest_timestamp` can be `None`, so the check will remain.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":293,"context_line":"        oldest_timestamp \u003d self.get_oldest_timestamp()"},{"line_number":294,"context_line":"        if oldest_timestamp is not None:"},{"line_number":295,"context_line":"            return current_time - oldest_timestamp"},{"line_number":296,"context_line":"        return float(\u0027inf\u0027)"},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"    def reset(self):"},{"line_number":299,"context_line":"        self.sorted_entries \u003d []"}],"source_content_type":"text/x-python","patch_set":29,"id":"f80cc6bd_5d16aef5","line":296,"updated":"2024-11-01 17:57:23.000000000","message":"Do we know how our metrics pipelines will react to these `inf`/`-inf` values? Are they going to blow out our axes, or just get dropped? If they\u0027re getting dropped anyway, I wonder if `None` might be a more-obvious sentinel for \"no data\".","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":293,"context_line":"        oldest_timestamp \u003d self.get_oldest_timestamp()"},{"line_number":294,"context_line":"        if oldest_timestamp is not None:"},{"line_number":295,"context_line":"            return current_time - oldest_timestamp"},{"line_number":296,"context_line":"        return float(\u0027inf\u0027)"},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"    def reset(self):"},{"line_number":299,"context_line":"        self.sorted_entries \u003d []"}],"source_content_type":"text/x-python","patch_set":29,"id":"2f0b2150_355fc77d","line":296,"in_reply_to":"f80cc6bd_5d16aef5","updated":"2024-11-01 21:01:21.000000000","message":"I do not know. I have changed the sentinel to `None`.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":559,"context_line":"    def dump_recon(self, elapsed):"},{"line_number":560,"context_line":"        \"\"\"Gathers stats and dumps recon cache.\"\"\""},{"line_number":561,"context_line":"        object_updater_stats \u003d {"},{"line_number":562,"context_line":"            \u0027async_pending_failures_oldest_timestamp\u0027: ("},{"line_number":563,"context_line":"                self.oldest_async_pendings.get_oldest_timestamp()"},{"line_number":564,"context_line":"            ),"},{"line_number":565,"context_line":"            \u0027async_pending_failures_oldest_timestamp_age\u0027: ("}],"source_content_type":"text/x-python","patch_set":29,"id":"0e3ac63c_1b052a29","line":562,"range":{"start_line":562,"start_character":13,"end_line":562,"end_character":27},"updated":"2024-11-01 17:57:23.000000000","message":"I\u0027m not sure how much value we get out of this prefix -- we already know we\u0027re dealing with the object-updater (from the `object_updater_stats` key), so it\u0027s processing async pendings.\n\nThe `failures_` prefix seems smart, though, as I peek further down the chain.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":559,"context_line":"    def dump_recon(self, elapsed):"},{"line_number":560,"context_line":"        \"\"\"Gathers stats and dumps recon cache.\"\"\""},{"line_number":561,"context_line":"        object_updater_stats \u003d {"},{"line_number":562,"context_line":"            \u0027async_pending_failures_oldest_timestamp\u0027: ("},{"line_number":563,"context_line":"                self.oldest_async_pendings.get_oldest_timestamp()"},{"line_number":564,"context_line":"            ),"},{"line_number":565,"context_line":"            \u0027async_pending_failures_oldest_timestamp_age\u0027: ("}],"source_content_type":"text/x-python","patch_set":29,"id":"c7d913cf_167991e1","line":562,"range":{"start_line":562,"start_character":13,"end_line":562,"end_character":27},"in_reply_to":"0e3ac63c_1b052a29","updated":"2024-11-01 21:01:21.000000000","message":"Since it\u0027s implied that we\u0027re processing async pendings, I will drop the `async_pending` prefix","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":572,"context_line":"                self.oldest_async_pendings.get_n_oldest_timestamp_acs()"},{"line_number":573,"context_line":"            ),"},{"line_number":574,"context_line":"            \u0027async_pending_manager_memory_usage\u0027: ("},{"line_number":575,"context_line":"                self.oldest_async_pendings.get_memory_usage()"},{"line_number":576,"context_line":"            ),"},{"line_number":577,"context_line":"        }"},{"line_number":578,"context_line":"        dump_recon_cache("}],"source_content_type":"text/x-python","patch_set":29,"id":"4933af8b_80a7bf1e","line":575,"updated":"2024-11-01 17:57:23.000000000","message":"How large of a test have you done so far? What sort of values have you seen coming out of this?\n\nDo we want this key when it\u0027s all merged to master, or is this mainly for now, while we\u0027re trying it out, so we can offer guidance on a recommended upper-bound for max entries?","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":572,"context_line":"                self.oldest_async_pendings.get_n_oldest_timestamp_acs()"},{"line_number":573,"context_line":"            ),"},{"line_number":574,"context_line":"            \u0027async_pending_manager_memory_usage\u0027: ("},{"line_number":575,"context_line":"                self.oldest_async_pendings.get_memory_usage()"},{"line_number":576,"context_line":"            ),"},{"line_number":577,"context_line":"        }"},{"line_number":578,"context_line":"        dump_recon_cache("}],"source_content_type":"text/x-python","patch_set":29,"id":"53b44a5c_1679f2de","line":575,"in_reply_to":"4933af8b_80a7bf1e","updated":"2024-11-01 21:01:21.000000000","message":"Our class size is ~45KB with the following details:\n- Cap of 100 account containers for our async tracker\n- 100 accounts, 4 containers per account, and 5 objects per account container\n- acct, container, and obj strs of ~6 characters each\n\n- As expected the memory usage does not change when more accounts are added unless the cap is increased.\n\nWhen there is no cap and the same number of contains per account, objects per account container and number of characters per act container, and object str.\n\n10 accounts:\n0.01 MB\n\n100 accounts:\n0.13 MB\n\n1000 accounts:\n1.44 MB\n\n10000 accounts:\n14.53 MB\n\nI think the key may be useful in master as a warning sign in case the structure is consuming far more memory than expected.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"}],"test/probe/test_object_async_update.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":405,"context_line":"                self.assertIsNotNone(timestamp)"},{"line_number":406,"context_line":"                ac_set.add((account, container))"},{"line_number":407,"context_line":""},{"line_number":408,"context_line":"            # All reported asyncs came from our uploads"},{"line_number":409,"context_line":"            for ac in ac_set:"},{"line_number":410,"context_line":"                self.assertIn(ac, set(ac_pairs))"},{"line_number":411,"context_line":""},{"line_number":412,"context_line":"            # Each node had at most one of each ac"},{"line_number":413,"context_line":"            for found_count in found_counts:"},{"line_number":414,"context_line":"                self.assertLessEqual(found_count, len(ac_pairs))"},{"line_number":415,"context_line":""},{"line_number":416,"context_line":""},{"line_number":417,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":29,"id":"bb56fd93_5059dfdb","line":414,"range":{"start_line":408,"start_character":0,"end_line":414,"end_character":64},"updated":"2024-11-01 17:57:23.000000000","message":"Are these supposed to be in the `for recon in recons:` loop? I feel like these should come just once, after it.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":405,"context_line":"                self.assertIsNotNone(timestamp)"},{"line_number":406,"context_line":"                ac_set.add((account, container))"},{"line_number":407,"context_line":""},{"line_number":408,"context_line":"            # All reported asyncs came from our uploads"},{"line_number":409,"context_line":"            for ac in ac_set:"},{"line_number":410,"context_line":"                self.assertIn(ac, set(ac_pairs))"},{"line_number":411,"context_line":""},{"line_number":412,"context_line":"            # Each node had at most one of each ac"},{"line_number":413,"context_line":"            for found_count in found_counts:"},{"line_number":414,"context_line":"                self.assertLessEqual(found_count, len(ac_pairs))"},{"line_number":415,"context_line":""},{"line_number":416,"context_line":""},{"line_number":417,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":29,"id":"4b986ecd_5339f2fa","line":414,"range":{"start_line":408,"start_character":0,"end_line":414,"end_character":64},"in_reply_to":"bb56fd93_5059dfdb","updated":"2024-11-01 21:01:21.000000000","message":"Yes, that makes more sense. Updated.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"}],"test/unit/obj/test_updater.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":2181,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)"},{"line_number":2182,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 900.0)"},{"line_number":2183,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"},{"line_number":2184,"context_line":"        self.assertNotIn(1000.0, self.manager.sorted_entries)"},{"line_number":2185,"context_line":""},{"line_number":2186,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1100.0)"},{"line_number":2187,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"}],"source_content_type":"text/x-python","patch_set":29,"id":"4d607f0a_74738fbd","line":2184,"range":{"start_line":2184,"start_character":25,"end_line":2184,"end_character":31},"updated":"2024-11-01 17:57:23.000000000","message":"`(1000.0, (\u0027a1\u0027, \u0027c1\u0027))`, surely?","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":true,"context_lines":[{"line_number":2181,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)"},{"line_number":2182,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 900.0)"},{"line_number":2183,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"},{"line_number":2184,"context_line":"        self.assertNotIn(1000.0, self.manager.sorted_entries)"},{"line_number":2185,"context_line":""},{"line_number":2186,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1100.0)"},{"line_number":2187,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"}],"source_content_type":"text/x-python","patch_set":29,"id":"91b6cc1e_c2697943","line":2184,"range":{"start_line":2184,"start_character":25,"end_line":2184,"end_character":31},"in_reply_to":"4d607f0a_74738fbd","updated":"2024-11-01 21:01:21.000000000","message":"I don\u0027t understand. The current signature for `add_update` is \n```python\n(method) def add_update(\n    self: Self@OldestAsyncPendingTracker,\n    account: Any,\n    container: Any,\n    timestamp: Any\n) -\u003e None\n```\nSo I think `self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)` is correct. Please let me know if I am misunderstanding your comment.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6ccc5b13387eb988ef4f742eeb4bb2e18aa5be84","unresolved":false,"context_lines":[{"line_number":2181,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)"},{"line_number":2182,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 900.0)"},{"line_number":2183,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"},{"line_number":2184,"context_line":"        self.assertNotIn(1000.0, self.manager.sorted_entries)"},{"line_number":2185,"context_line":""},{"line_number":2186,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1100.0)"},{"line_number":2187,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"}],"source_content_type":"text/x-python","patch_set":29,"id":"d8052cbe_5f254a14","line":2184,"range":{"start_line":2184,"start_character":25,"end_line":2184,"end_character":31},"in_reply_to":"91b6cc1e_c2697943","updated":"2024-11-01 21:14:07.000000000","message":"The `add_update` call looks right, but I don\u0027t think there should ever be a `1000` in `sorted_entries` -- but you\u0027ve got that covered now!","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":2184,"context_line":"        self.assertNotIn(1000.0, self.manager.sorted_entries)"},{"line_number":2185,"context_line":""},{"line_number":2186,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1100.0)"},{"line_number":2187,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"},{"line_number":2188,"context_line":""},{"line_number":2189,"context_line":"    def test_evict_newest_pairs_when_limit_exceeded(self):"},{"line_number":2190,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)"}],"source_content_type":"text/x-python","patch_set":29,"id":"b5a4b257_45cc0495","line":2187,"updated":"2024-11-01 17:57:23.000000000","message":"Should we also check `sorted_entries`?","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":2184,"context_line":"        self.assertNotIn(1000.0, self.manager.sorted_entries)"},{"line_number":2185,"context_line":""},{"line_number":2186,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1100.0)"},{"line_number":2187,"context_line":"        self.assertEqual(self.manager.ac_to_timestamp[(\u0027a1\u0027, \u0027c1\u0027)], 900)"},{"line_number":2188,"context_line":""},{"line_number":2189,"context_line":"    def test_evict_newest_pairs_when_limit_exceeded(self):"},{"line_number":2190,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)"}],"source_content_type":"text/x-python","patch_set":29,"id":"50a369ac_a9f5454e","line":2187,"in_reply_to":"b5a4b257_45cc0495","updated":"2024-11-01 21:01:21.000000000","message":"Sure! Added some `assertIn`s for `sorted_entries`","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67b677e8ed10650018c3025b0257c684053a1e2","unresolved":true,"context_lines":[{"line_number":2192,"context_line":"        self.manager.add_update(\u0027a3\u0027, \u0027c3\u0027, 3000.0)"},{"line_number":2193,"context_line":""},{"line_number":2194,"context_line":"        self.manager.add_update(\u0027a4\u0027, \u0027c4\u0027, 4000.0)"},{"line_number":2195,"context_line":"        self.assertNotIn((\u0027a4\u0027, \u0027c4\u0027), self.manager.ac_to_timestamp)"},{"line_number":2196,"context_line":"        self.assertIn((\u0027a3\u0027, \u0027c3\u0027), self.manager.ac_to_timestamp)"},{"line_number":2197,"context_line":""},{"line_number":2198,"context_line":"    def test_get_n_oldest_timestamp_acs(self):"}],"source_content_type":"text/x-python","patch_set":29,"id":"a3a0b4f7_ad8037ff","line":2195,"updated":"2024-11-01 17:57:23.000000000","message":"I\u0027m not sure I\u0027d call this an _eviction_, exactly -- it\u0027s more that we skip recording it at all. Still an important case to cover, but I think I\u0027d like to see something at 2500, say, pushing out the 3000 entry.","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"ec52935f460ea929d4ab528ecee8e6432e1d777a","unresolved":false,"context_lines":[{"line_number":2192,"context_line":"        self.manager.add_update(\u0027a3\u0027, \u0027c3\u0027, 3000.0)"},{"line_number":2193,"context_line":""},{"line_number":2194,"context_line":"        self.manager.add_update(\u0027a4\u0027, \u0027c4\u0027, 4000.0)"},{"line_number":2195,"context_line":"        self.assertNotIn((\u0027a4\u0027, \u0027c4\u0027), self.manager.ac_to_timestamp)"},{"line_number":2196,"context_line":"        self.assertIn((\u0027a3\u0027, \u0027c3\u0027), self.manager.ac_to_timestamp)"},{"line_number":2197,"context_line":""},{"line_number":2198,"context_line":"    def test_get_n_oldest_timestamp_acs(self):"}],"source_content_type":"text/x-python","patch_set":29,"id":"fdb89754_1d4ea43e","line":2195,"in_reply_to":"a3a0b4f7_ad8037ff","updated":"2024-11-01 21:01:21.000000000","message":"Modified to `test_newest_pairs_not_added_when_limit_exceeded` and added another test case which does what you\u0027re describing.\n```python\n    def test_eviction_when_limit_exceeded(self):\n        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)\n        self.manager.add_update(\u0027a2\u0027, \u0027c2\u0027, 2000.0)\n        self.manager.add_update(\u0027a3\u0027, \u0027c3\u0027, 3000.0)\n\n        self.manager.add_update(\u0027a4\u0027, \u0027c4\u0027, 2500.0)\n        self.assertIn((\u0027a4\u0027, \u0027c4\u0027), self.manager.ac_to_timestamp)\n        self.assertNotIn((\u0027a3\u0027, \u0027c3\u0027), self.manager.ac_to_timestamp)\n```","commit_id":"87096f0155c66e2f7d88737129e5806bb80b3b06"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6ccc5b13387eb988ef4f742eeb4bb2e18aa5be84","unresolved":true,"context_lines":[{"line_number":2242,"context_line":"        age \u003d self.manager.get_oldest_timestamp_age()"},{"line_number":2243,"context_line":"        self.assertAlmostEqual(age, 200.0, delta\u003d1)"},{"line_number":2244,"context_line":""},{"line_number":2245,"context_line":"    def test_get_oldest_timestamp_age_no_updates(self):"},{"line_number":2246,"context_line":"        age \u003d self.manager.get_oldest_timestamp_age()"},{"line_number":2247,"context_line":"        self.assertEqual(age, None)"},{"line_number":2248,"context_line":""},{"line_number":2249,"context_line":"    def test_empty_oldest_timestamp_age(self):"},{"line_number":2250,"context_line":"        self.assertEqual(self.manager.get_oldest_timestamp_age(), None)"},{"line_number":2251,"context_line":""},{"line_number":2252,"context_line":"    def test_eviction_when_multiple_same_timestamps(self):"},{"line_number":2253,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)"}],"source_content_type":"text/x-python","patch_set":32,"id":"c266ac53_e168e695","line":2250,"range":{"start_line":2245,"start_character":4,"end_line":2250,"end_character":71},"updated":"2024-11-01 21:14:07.000000000","message":"These are the same test.","commit_id":"c7bb71bdc50bef0f2a95025ece2d4ebfe3e10719"},{"author":{"_account_id":37271,"name":"Chinemerem Chigbo","display_name":"Chinemerem","email":"cchigbo@nvidia.com","username":"chinemerem"},"change_message_id":"75e93d2fa0eae5268b59ace17ae87e8d7567fc9b","unresolved":false,"context_lines":[{"line_number":2242,"context_line":"        age \u003d self.manager.get_oldest_timestamp_age()"},{"line_number":2243,"context_line":"        self.assertAlmostEqual(age, 200.0, delta\u003d1)"},{"line_number":2244,"context_line":""},{"line_number":2245,"context_line":"    def test_get_oldest_timestamp_age_no_updates(self):"},{"line_number":2246,"context_line":"        age \u003d self.manager.get_oldest_timestamp_age()"},{"line_number":2247,"context_line":"        self.assertEqual(age, None)"},{"line_number":2248,"context_line":""},{"line_number":2249,"context_line":"    def test_empty_oldest_timestamp_age(self):"},{"line_number":2250,"context_line":"        self.assertEqual(self.manager.get_oldest_timestamp_age(), None)"},{"line_number":2251,"context_line":""},{"line_number":2252,"context_line":"    def test_eviction_when_multiple_same_timestamps(self):"},{"line_number":2253,"context_line":"        self.manager.add_update(\u0027a1\u0027, \u0027c1\u0027, 1000.0)"}],"source_content_type":"text/x-python","patch_set":32,"id":"923b1d96_801447e2","line":2250,"range":{"start_line":2245,"start_character":4,"end_line":2250,"end_character":71},"in_reply_to":"c266ac53_e168e695","updated":"2024-11-01 22:51:18.000000000","message":"Removed.","commit_id":"c7bb71bdc50bef0f2a95025ece2d4ebfe3e10719"}]}