)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"e525aab0868900209a674bd187cd7c46be25e4a8","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     Julia Kreger \u003cjuliaashleykreger@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2023-07-16 13:36:27 -0700"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"WIP: Retry node update failures due to locks"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Adds a database retry decorator to capture and retry exceptions"},{"line_number":10,"context_line":"rooted in SQLite locking. These locking errors are rooted in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"736266ae_015f8f4d","line":7,"range":{"start_line":7,"start_character":11,"end_line":7,"end_character":15},"updated":"2023-07-17 13:29:25.000000000","message":"Not just node. db writes.","commit_id":"1b067692a46abfdbeb7357815e098d9674f31286"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"4b4e60a9397ea1d094f49b89c03d24eaeb1fde72","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Julia Kreger \u003cjuliaashleykreger@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2023-07-16 13:36:27 -0700"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"WIP: Retry node update failures due to locks"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Adds a database retry decorator to capture and retry exceptions"},{"line_number":10,"context_line":"rooted in SQLite locking. These locking errors are rooted in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"670cdd63_fa0e3fa1","line":7,"range":{"start_line":7,"start_character":11,"end_line":7,"end_character":15},"in_reply_to":"736266ae_015f8f4d","updated":"2023-07-17 14:40:57.000000000","message":"Done","commit_id":"1b067692a46abfdbeb7357815e098d9674f31286"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"8de425a28b6cf94d94a6df4d991c050e4b972c94","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"c32faa34_70fc5398","updated":"2023-07-17 13:04:57.000000000","message":"recheck need a loaded CI to try and reproduce the failure.","commit_id":"1b067692a46abfdbeb7357815e098d9674f31286"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"aafa31bc4b3454b8774c7b15ec496689b9007b2e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"feeb4ef9_176ce6ea","updated":"2023-07-17 04:53:28.000000000","message":"recheck need a loaded ci system to reproduce","commit_id":"1b067692a46abfdbeb7357815e098d9674f31286"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e16210c6b35171f28190cb52db9408fa24aef4e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"ff0300bc_45cd1c1c","updated":"2023-07-17 17:05:17.000000000","message":"recheck lets see if we can get some locking errors!","commit_id":"187f87ef6568b410f2539ac654d3f8939481b082"}],"ironic/db/sqlalchemy/api.py":[{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"b51d49147c7d118e47871ea38d7a44e2d603b19d","unresolved":true,"context_lines":[{"line_number":930,"context_line":"                    node\u003dnode_id)"},{"line_number":931,"context_line":"            else:"},{"line_number":932,"context_line":"                raise"},{"line_number":933,"context_line":"        except Exception as e:"},{"line_number":934,"context_line":"            if \u0027database is locked\u0027 in str(e):"},{"line_number":935,"context_line":"                # NOTE(TheJulia): This is the most sensitive place"},{"line_number":936,"context_line":"                # where a sqlite db lock can derail processes."}],"source_content_type":"text/x-python","patch_set":1,"id":"769196a7_b18e8e4b","line":933,"updated":"2023-07-14 08:06:23.000000000","message":"I think this is the actual solution to many problems. Implementation wise, I suggest we turn this to decorator (probably using tenacity) and apply it pretty generously to all/most calls that write anything.","commit_id":"13b3945a7c49dccfbab61e261384a92127f3f1dc"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"bf0d20faf79b9826a91b2686af7fb38e946536d7","unresolved":true,"context_lines":[{"line_number":930,"context_line":"                    node\u003dnode_id)"},{"line_number":931,"context_line":"            else:"},{"line_number":932,"context_line":"                raise"},{"line_number":933,"context_line":"        except Exception as e:"},{"line_number":934,"context_line":"            if \u0027database is locked\u0027 in str(e):"},{"line_number":935,"context_line":"                # NOTE(TheJulia): This is the most sensitive place"},{"line_number":936,"context_line":"                # where a sqlite db lock can derail processes."}],"source_content_type":"text/x-python","patch_set":1,"id":"f7e4973e_4a8bf65a","line":933,"in_reply_to":"769196a7_b18e8e4b","updated":"2023-07-14 13:34:24.000000000","message":"I was thinking the same. I tried that originally in the early days of these lock issues, so we will just have to see. I guess my concern at the moment is doing so too aggressively, since it seems the nature of the conductor status heartbeat could *still* completely block out most write activity, then again it got to run in it\u0027s own greenthread on it\u0027s own connection basically.","commit_id":"13b3945a7c49dccfbab61e261384a92127f3f1dc"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"518fb4e85950e95509940e9bc7e04f31452eb528","unresolved":false,"context_lines":[{"line_number":930,"context_line":"                    node\u003dnode_id)"},{"line_number":931,"context_line":"            else:"},{"line_number":932,"context_line":"                raise"},{"line_number":933,"context_line":"        except Exception as e:"},{"line_number":934,"context_line":"            if \u0027database is locked\u0027 in str(e):"},{"line_number":935,"context_line":"                # NOTE(TheJulia): This is the most sensitive place"},{"line_number":936,"context_line":"                # where a sqlite db lock can derail processes."}],"source_content_type":"text/x-python","patch_set":1,"id":"b061241d_59a7085f","line":933,"in_reply_to":"f7e4973e_4a8bf65a","updated":"2023-07-16 20:22:56.000000000","message":"Done","commit_id":"13b3945a7c49dccfbab61e261384a92127f3f1dc"}]}