)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"bbd3726826f96bf1f0d1553ce4077d1a152604d3","unresolved":false,"context_lines":[{"line_number":9,"context_line":"In race condition, when the cleanup thread and the metastatic"},{"line_number":10,"context_line":"statemachine runner try to delete the node at the same time,"},{"line_number":11,"context_line":"exception can be thrown in the statemachine runner, and the backing"},{"line_number":12,"context_line":"node would not be removed from the \u0027backing_node_records\"."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Fixed by removing the bnr before zk operations to make sure the"},{"line_number":15,"context_line":"bnr is always removed when the backing node is released."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"38da853c_37a1788d","line":12,"updated":"2024-08-12 17:30:51.000000000","message":"I\u0027m not sure I understand the situation you\u0027re describing.\n\nWhy would the statemachine runner delete the backing node?  It should still be locked by the metastatic driver, which means it\u0027s in use.  The statemachine runner shouldn\u0027t decide to delete it until after the listResources decides it should be allowed to be deleted.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"460d790ffccc333f6397715f474ce7c4c9bed21b","unresolved":true,"context_lines":[{"line_number":9,"context_line":"In race condition, when the cleanup thread and the metastatic"},{"line_number":10,"context_line":"statemachine runner try to delete the node at the same time,"},{"line_number":11,"context_line":"exception can be thrown in the statemachine runner, and the backing"},{"line_number":12,"context_line":"node would not be removed from the \u0027backing_node_records\"."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Fixed by removing the bnr before zk operations to make sure the"},{"line_number":15,"context_line":"bnr is always removed when the backing node is released."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"cf1afa46_1aa38ddc","line":12,"in_reply_to":"29b11eeb_507c0694","updated":"2024-08-14 11:55:15.000000000","message":"I agree, and I think my statement does not contradict with yours. \nWhat I am trying to says is when the state machine runner tries to lock the node for deleting, it creates the lock contender under that node. i.e. in your example of your change description:\nA] call forceUnlockNode\nA] delete node/lock/contender0\nB] create node/lock/contender1\nA] delete node/lock\nthe \"B\" here is probably the state machine runner.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"386f3204e684193b1c0b3e6146aee17a7ee0537a","unresolved":true,"context_lines":[{"line_number":9,"context_line":"In race condition, when the cleanup thread and the metastatic"},{"line_number":10,"context_line":"statemachine runner try to delete the node at the same time,"},{"line_number":11,"context_line":"exception can be thrown in the statemachine runner, and the backing"},{"line_number":12,"context_line":"node would not be removed from the \u0027backing_node_records\"."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Fixed by removing the bnr before zk operations to make sure the"},{"line_number":15,"context_line":"bnr is always removed when the backing node is released."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"d3fcd1be_15c60690","line":12,"in_reply_to":"38da853c_37a1788d","updated":"2024-08-13 06:53:24.000000000","message":"I think as soon as the following two lines are executed, the node is allowed to be deleted:\n```\nnode.state \u003d zk.USED\nself.zk.storeNode(node)\n```\nAnd it could happen the next line `self.zk.forceUnlockNode(node)` and the statemachine runner tries to delete the node at the same time.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"f79eb922e7a0d5e075b6c59df88139c188715401","unresolved":true,"context_lines":[{"line_number":9,"context_line":"In race condition, when the cleanup thread and the metastatic"},{"line_number":10,"context_line":"statemachine runner try to delete the node at the same time,"},{"line_number":11,"context_line":"exception can be thrown in the statemachine runner, and the backing"},{"line_number":12,"context_line":"node would not be removed from the \u0027backing_node_records\"."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Fixed by removing the bnr before zk operations to make sure the"},{"line_number":15,"context_line":"bnr is always removed when the backing node is released."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"29b11eeb_507c0694","line":12,"in_reply_to":"38da853c_37a1788d","updated":"2024-08-13 16:10:08.000000000","message":"The launcher should not attempt to delete a node that it can\u0027t lock, and it should not be able to lock the node until we unlock it.\n\nI believe the problem is described here: https://review.opendev.org/c/zuul/nodepool/+/926221","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d21a8d2df9e83812bacc78e884d0b4bfefc1bf18","unresolved":false,"context_lines":[{"line_number":9,"context_line":"In race condition, when the cleanup thread and the metastatic"},{"line_number":10,"context_line":"statemachine runner try to delete the node at the same time,"},{"line_number":11,"context_line":"exception can be thrown in the statemachine runner, and the backing"},{"line_number":12,"context_line":"node would not be removed from the \u0027backing_node_records\"."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Fixed by removing the bnr before zk operations to make sure the"},{"line_number":15,"context_line":"bnr is always removed when the backing node is released."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"a9e88e55_506fa348","line":12,"in_reply_to":"cf1afa46_1aa38ddc","updated":"2024-08-14 23:18:34.000000000","message":"You said:\n\u003e I think as soon as the following two lines are executed, the node is allowed to be deleted:\n\u003e\n\u003e\u003e node.state \u003d zk.USED\n\u003e\u003e self.zk.storeNode(node)\n\nBut that\u0027s not true, only after the lock is deleted (the next line after those) will the launcher try to delete the node.\n\nAnyway, your last comment is correct, so regardless of how we got here, we both understand the issue now.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"fbec88bfe6f96936f8455a53cd15fa7b0bf1fb1b","unresolved":true,"context_lines":[{"line_number":12,"context_line":"node would not be removed from the \u0027backing_node_records\"."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Fixed by removing the bnr before zk operations to make sure the"},{"line_number":15,"context_line":"bnr is always removed when the backing node is released."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Related exception:"},{"line_number":18,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":14,"id":"a05baed3_c1a06fa7","line":15,"updated":"2024-08-15 20:39:44.000000000","message":"We need a new commit message.  Something like\n\nAdd a test for backing node deletion errors\n\nThis adds a test to confirm that if a backing node is externally\ndeleted, that the metastatic driver will eventually remove its\ninternal backing node record.","commit_id":"495e963dfd35a7d35705e465b56be2defe0f8d64"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"effd1786b20399997ae3674b7682a73e027c953e","unresolved":false,"context_lines":[{"line_number":12,"context_line":"node would not be removed from the \u0027backing_node_records\"."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Fixed by removing the bnr before zk operations to make sure the"},{"line_number":15,"context_line":"bnr is always removed when the backing node is released."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Related exception:"},{"line_number":18,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":14,"id":"85762234_47b5880c","line":15,"in_reply_to":"a05baed3_c1a06fa7","updated":"2024-08-20 16:16:46.000000000","message":"Done","commit_id":"495e963dfd35a7d35705e465b56be2defe0f8d64"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"460d790ffccc333f6397715f474ce7c4c9bed21b","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"7826c90b_5eb97969","updated":"2024-08-14 11:55:15.000000000","message":"Do you think it make sense to also merge my change in addition to yours?\n\nBecause your changes fixed the underlying issue that caused the exception. \nAnd my change:\n1. Avoid bnr not being deleted when other exceptions that could possibly happen again in the future.\n2. In case the node enters a invalid status for some reason, it checks if the corresponding node of the bnr is in valid status before trying to use it.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":35923,"name":"Andrei Dmitriev","email":"andrei.dmitriev@nokia.com","username":"andrei-dmitriev-nokia","status":"Nokia"},"change_message_id":"32ac3c39d8b5180cef8022750384b81a4904fc6c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"c5e099c3_24a13f29","updated":"2024-08-02 08:33:31.000000000","message":"Seeing the same exception and not released  nodes until launcher restart in our environment. Thanks for fix, will also check on our side later.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d21a8d2df9e83812bacc78e884d0b4bfefc1bf18","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"0a628279_e147db6f","in_reply_to":"7826c90b_5eb97969","updated":"2024-08-14 23:18:34.000000000","message":"We could add some safety, yes.  It will need some changes; I\u0027ll leave comments to that effect.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"de65078cdd665f711bee8492cc640e641af5a20d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"6948be80_70e9dcac","updated":"2024-08-15 12:45:03.000000000","message":"I Updated the code accordingly in the latest patch set.","commit_id":"01a5301b150719c25c9d3efd705c3a4dc61d1c4b"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"4c87243df17eaa8a1b17cfccc502e41acbb2bb04","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":13,"id":"ff8b4908_c73d1718","updated":"2024-08-15 12:57:36.000000000","message":"I think it is more proper to rebase my change to dependent on your fix. My change just adds a test.","commit_id":"ea9b4f377a540bf901d95812dbfc216b094f7157"}],"nodepool/driver/metastatic/adapter.py":[{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d21a8d2df9e83812bacc78e884d0b4bfefc1bf18","unresolved":true,"context_lines":[{"line_number":321,"context_line":"                        self.log.info(\"Backing node %s has been idle for \""},{"line_number":322,"context_line":"                                      \"%s seconds, releasing\","},{"line_number":323,"context_line":"                                      bnr.node_id, now - bnr.last_used)"},{"line_number":324,"context_line":"                        backing_node_records.remove(bnr)"},{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"}],"source_content_type":"text/x-python","patch_set":9,"id":"3d53220e_d899e79f","line":324,"range":{"start_line":324,"start_character":0,"end_line":324,"end_character":2},"updated":"2024-08-14 23:18:34.000000000","message":"Now that the underlying issue is fixed in the other change, we should not move this line.  It is safer this way, so that we don\u0027t leak records.  If we move it and encounter an error after this line, we may never allow the backing node to be deleted.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"de65078cdd665f711bee8492cc640e641af5a20d","unresolved":false,"context_lines":[{"line_number":321,"context_line":"                        self.log.info(\"Backing node %s has been idle for \""},{"line_number":322,"context_line":"                                      \"%s seconds, releasing\","},{"line_number":323,"context_line":"                                      bnr.node_id, now - bnr.last_used)"},{"line_number":324,"context_line":"                        backing_node_records.remove(bnr)"},{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"}],"source_content_type":"text/x-python","patch_set":9,"id":"d11023e2_e122f73a","line":324,"range":{"start_line":324,"start_character":0,"end_line":324,"end_character":2},"in_reply_to":"3d53220e_d899e79f","updated":"2024-08-15 12:45:03.000000000","message":"Acknowledged","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"db8faeaff900dffc893a3201be821425f5d87436","unresolved":true,"context_lines":[{"line_number":324,"context_line":"                        backing_node_records.remove(bnr)"},{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"},{"line_number":328,"context_line":"                        self.zk.forceUnlockNode(node)"},{"line_number":329,"context_line":"        return []"},{"line_number":330,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"fe2d9263_924e368e","line":327,"updated":"2024-08-13 10:40:25.000000000","message":"After this line is executed, the node would be allowed to be removed by the statemachine runner. So it could happen that the statemachine runner and the line below `self.zk.forceUnlockNode(node)` could be trying to delete the node at the same time, which leads to the exception mentioned in the description of this change.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d21a8d2df9e83812bacc78e884d0b4bfefc1bf18","unresolved":true,"context_lines":[{"line_number":324,"context_line":"                        backing_node_records.remove(bnr)"},{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"},{"line_number":328,"context_line":"                        self.zk.forceUnlockNode(node)"},{"line_number":329,"context_line":"        return []"},{"line_number":330,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"541efeff_c6cf430f","line":327,"updated":"2024-08-14 23:18:34.000000000","message":"This is one of two lines that can possibly cause our internal state to get out of sync with ZK.\n\nIf it succeeds in saving the state to ZK, but then raises an exception, the internal and zk states will still have the same values.  And we will still hold the lock, and we will not have removed the backing node.  So we\u0027ll try it again later.\n\nIf it fails before saving the state to ZK, then the node will have the \"IN_USE\" state in zk, but \"USED\" in memory.  They mean the same thing to nodepool, so it doesn\u0027t really matter.  The part about holding the lock and keeping the backing node applies to this too.\n\nSo this should be harmless.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"de65078cdd665f711bee8492cc640e641af5a20d","unresolved":false,"context_lines":[{"line_number":324,"context_line":"                        backing_node_records.remove(bnr)"},{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"},{"line_number":328,"context_line":"                        self.zk.forceUnlockNode(node)"},{"line_number":329,"context_line":"        return []"},{"line_number":330,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"ed356517_5fca666b","line":327,"in_reply_to":"541efeff_c6cf430f","updated":"2024-08-15 12:45:03.000000000","message":"Acknowledged","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d21a8d2df9e83812bacc78e884d0b4bfefc1bf18","unresolved":false,"context_lines":[{"line_number":324,"context_line":"                        backing_node_records.remove(bnr)"},{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"},{"line_number":328,"context_line":"                        self.zk.forceUnlockNode(node)"},{"line_number":329,"context_line":"        return []"},{"line_number":330,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9dbf544a_64977f36","line":327,"in_reply_to":"fe2d9263_924e368e","updated":"2024-08-14 23:18:34.000000000","message":"As discussed elsewhere, it\u0027s only after line 328 that the launcher will remove it.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d21a8d2df9e83812bacc78e884d0b4bfefc1bf18","unresolved":true,"context_lines":[{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"},{"line_number":328,"context_line":"                        self.zk.forceUnlockNode(node)"},{"line_number":329,"context_line":"        return []"},{"line_number":330,"context_line":""},{"line_number":331,"context_line":"    def deleteResource(self, resource):"}],"source_content_type":"text/x-python","patch_set":9,"id":"1f8f5cc8_de78273d","line":328,"updated":"2024-08-14 23:18:34.000000000","message":"If this fails in such a way that the node is deleted from ZK but raises an exception after that, then we may end up with a similar situation to the original error.\n\nInstead of checking the node state below (which is resource intensive), why don\u0027t we make this section robust against a nonexistent node.\n\nSo this whole section should read:\n\n```\n  node \u003d self._getNode(bnr.node_id)\n  if node:\n    node.state\u003dUSED\n    storeNode(node)\n    self.zk.forceUnlockNode(node) \n  backing_node_records.remove(bnr)\n```","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"de65078cdd665f711bee8492cc640e641af5a20d","unresolved":true,"context_lines":[{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"},{"line_number":328,"context_line":"                        self.zk.forceUnlockNode(node)"},{"line_number":329,"context_line":"        return []"},{"line_number":330,"context_line":""},{"line_number":331,"context_line":"    def deleteResource(self, resource):"}],"source_content_type":"text/x-python","patch_set":9,"id":"68d9cc02_20997848","line":328,"in_reply_to":"1f8f5cc8_de78273d","updated":"2024-08-15 12:45:03.000000000","message":"I agree, this is a better approach. I will update the code.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"effd1786b20399997ae3674b7682a73e027c953e","unresolved":false,"context_lines":[{"line_number":325,"context_line":"                        node \u003d self._getNode(bnr.node_id)"},{"line_number":326,"context_line":"                        node.state \u003d zk.USED"},{"line_number":327,"context_line":"                        self.zk.storeNode(node)"},{"line_number":328,"context_line":"                        self.zk.forceUnlockNode(node)"},{"line_number":329,"context_line":"        return []"},{"line_number":330,"context_line":""},{"line_number":331,"context_line":"    def deleteResource(self, resource):"}],"source_content_type":"text/x-python","patch_set":9,"id":"6b4cf069_088c0a69","line":328,"in_reply_to":"68d9cc02_20997848","updated":"2024-08-20 16:16:46.000000000","message":"Acknowledged","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d21a8d2df9e83812bacc78e884d0b4bfefc1bf18","unresolved":true,"context_lines":[{"line_number":429,"context_line":"            # double check if the bnr is valid"},{"line_number":430,"context_line":"            if backing_node_record and backing_node_record.node_id:"},{"line_number":431,"context_line":"                backing_node \u003d self._getNode("},{"line_number":432,"context_line":"                    backing_node_record.node_id)"},{"line_number":433,"context_line":"                if not backing_node or backing_node.state \u003d\u003d zk.USED:"},{"line_number":434,"context_line":"                    # this should not happen, we log an error and remove it."},{"line_number":435,"context_line":"                    self.log.error(\"Backing node %s is not valid!\","}],"source_content_type":"text/x-python","patch_set":9,"id":"51fb601e_01b111fa","line":432,"updated":"2024-08-14 23:18:34.000000000","message":"Rather than extra traffic to ZK that shouldn\u0027t be necessary, let\u0027s make the section above safe for missing nodes.","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"},{"author":{"_account_id":33134,"name":"Dong Zhang","email":"dong.zhang@bmw.de","username":"dongzhang"},"change_message_id":"de65078cdd665f711bee8492cc640e641af5a20d","unresolved":false,"context_lines":[{"line_number":429,"context_line":"            # double check if the bnr is valid"},{"line_number":430,"context_line":"            if backing_node_record and backing_node_record.node_id:"},{"line_number":431,"context_line":"                backing_node \u003d self._getNode("},{"line_number":432,"context_line":"                    backing_node_record.node_id)"},{"line_number":433,"context_line":"                if not backing_node or backing_node.state \u003d\u003d zk.USED:"},{"line_number":434,"context_line":"                    # this should not happen, we log an error and remove it."},{"line_number":435,"context_line":"                    self.log.error(\"Backing node %s is not valid!\","}],"source_content_type":"text/x-python","patch_set":9,"id":"2ad74e93_d7b4b7f8","line":432,"in_reply_to":"51fb601e_01b111fa","updated":"2024-08-15 12:45:03.000000000","message":"Acknowledged","commit_id":"9336a76f6348b7e07d7d98b8b6996d81ee8a478c"}],"nodepool/tests/__init__.py":[{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"fbec88bfe6f96936f8455a53cd15fa7b0bf1fb1b","unresolved":true,"context_lines":[{"line_number":603,"context_line":"            if not exists:"},{"line_number":604,"context_line":"                break"},{"line_number":605,"context_line":""},{"line_number":606,"context_line":"    def waitForBnrDeletion(self, bnrs, bnr):"},{"line_number":607,"context_line":"        for _ in iterate_timeout(ONE_MINUTE, Exception,"},{"line_number":608,"context_line":"                                 \"Backing node record deletion\","},{"line_number":609,"context_line":"                                 interval\u003d1):"}],"source_content_type":"text/x-python","patch_set":14,"id":"32cca8d2_209a12e3","line":606,"updated":"2024-08-15 20:39:44.000000000","message":"This should be in the metastatic driver since it\u0027s the only thing that has a bnr.  I think you should just inline the contents of this method into that one test method.","commit_id":"495e963dfd35a7d35705e465b56be2defe0f8d64"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"effd1786b20399997ae3674b7682a73e027c953e","unresolved":false,"context_lines":[{"line_number":603,"context_line":"            if not exists:"},{"line_number":604,"context_line":"                break"},{"line_number":605,"context_line":""},{"line_number":606,"context_line":"    def waitForBnrDeletion(self, bnrs, bnr):"},{"line_number":607,"context_line":"        for _ in iterate_timeout(ONE_MINUTE, Exception,"},{"line_number":608,"context_line":"                                 \"Backing node record deletion\","},{"line_number":609,"context_line":"                                 interval\u003d1):"}],"source_content_type":"text/x-python","patch_set":14,"id":"3cb0b46b_524baa16","line":606,"in_reply_to":"32cca8d2_209a12e3","updated":"2024-08-20 16:16:46.000000000","message":"Done","commit_id":"495e963dfd35a7d35705e465b56be2defe0f8d64"}]}