)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"9d1b5855371897afb7320fca473087a3de2943ba","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"7d984f04_e5162a96","updated":"2025-06-17 15:47:18.000000000","message":"A general remark to get started.\n\ncd specs/not-implemented\nln -s ../approved/emergency-bulk-operations.rst emergency-bulk-operations.rst\ngit add emergency-bulk-operations.rst\ngit commit --amend\n\n;)","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"55a7e5fd23be24d7501f1b091a75c68fb41a405f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"ed7a8ddd_0646cc36","updated":"2025-06-17 18:48:14.000000000","message":"Probably best for this to not have ironic-week-prio until it\u0027s closer to landable","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"0f5ed4b53bee73f4fb90aabf71828f9432887440","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"5469ff71_bad47f4e","updated":"2025-06-13 22:49:09.000000000","message":"Some coarse comments. Haven\u0027t reviewed with an eye for detail yet.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"ace324d4abebb19058d0e1bf451a1a8d63cf7644","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"dcef1f25_eadb5602","updated":"2025-07-10 15:06:45.000000000","message":"Handling emergencies, in particular power cuts and their recovery, is a very relevant topic, so thanks for the spec! I have left a couple of thoughts inline.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36770,"name":"cid","display_name":"cid","email":"cid@gr-oss.io","username":"cidelight","status":"@gr-oss upstream: Doing good IRONIC things..."},"change_message_id":"84318c376521e11d87eb914ca9ace7b9d9aa0cb6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"a1866ba9_7685ee5d","updated":"2025-07-06 18:57:43.000000000","message":"I don\u0027t have the Operator perspective, but overall looking good. \u0027ll likely give another review pass later on.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":8,"id":"aca27ab9_b7bdd29d","updated":"2025-09-25 21:20:11.000000000","message":"I also think, generally, this could use an edit of some of the bonus information throughout that reads sorta like AI generated text that is more verbose than necessary.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"3b0f1eb669ba64540a830602a33cc672bbcb92e9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"e1ffe1e3_358b4d3f","updated":"2025-09-23 16:17:09.000000000","message":"recheck updated results needed","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"ad606991_72724e22","in_reply_to":"aca27ab9_b7bdd29d","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"1f729146_5e37cd3f","updated":"2026-07-02 02:40:37.000000000","message":"Starting fresh with comments","commit_id":"2b319b57eb8da8b05a53dc0557d47ba178ef5420"}],"specs/approved/emergency-bulk-operations.rst":[{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"0f5ed4b53bee73f4fb90aabf71828f9432887440","unresolved":true,"context_lines":[{"line_number":25,"context_line":"2. Write custom scripts that may not properly handle all edge cases"},{"line_number":26,"context_line":"3. Use database queries and manual intervention, which is risky during"},{"line_number":27,"context_line":"   critical situations"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"This makes it difficult to respond quickly and safely to datacenter-wide"},{"line_number":30,"context_line":"emergencies that require coordinated action across many nodes."},{"line_number":31,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"e2683d86_36a7f507","line":28,"updated":"2025-06-13 22:49:09.000000000","message":"another case specifically mentioned by some operators: when you recover, you may wish to ensure the conductor doesn\u0027t power on all nodes (a motivator for db interventions in some cases)","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":25,"context_line":"2. Write custom scripts that may not properly handle all edge cases"},{"line_number":26,"context_line":"3. Use database queries and manual intervention, which is risky during"},{"line_number":27,"context_line":"   critical situations"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"This makes it difficult to respond quickly and safely to datacenter-wide"},{"line_number":30,"context_line":"emergencies that require coordinated action across many nodes."},{"line_number":31,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"f52edf33_35a7430d","line":28,"in_reply_to":"e2683d86_36a7f507","updated":"2025-06-17 16:17:06.000000000","message":"Or that people need some way to articulate an ordered preference. That may need to be delineated form just a command interface.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":false,"context_lines":[{"line_number":25,"context_line":"2. Write custom scripts that may not properly handle all edge cases"},{"line_number":26,"context_line":"3. Use database queries and manual intervention, which is risky during"},{"line_number":27,"context_line":"   critical situations"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"This makes it difficult to respond quickly and safely to datacenter-wide"},{"line_number":30,"context_line":"emergencies that require coordinated action across many nodes."},{"line_number":31,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"948f2f8f_e746204a","line":28,"in_reply_to":"f52edf33_35a7430d","updated":"2025-07-03 01:17:01.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"0f5ed4b53bee73f4fb90aabf71828f9432887440","unresolved":true,"context_lines":[{"line_number":38,"context_line":"During a critical network infrastructure failure affecting multiple racks,"},{"line_number":39,"context_line":"the operations team needs to:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"1. Quickly identify all nodes in the affected network segments"},{"line_number":42,"context_line":"2. Gracefully shut down workloads to prevent data corruption"},{"line_number":43,"context_line":"3. Power cycle network-dependent nodes to clear stuck states"},{"line_number":44,"context_line":"4. Monitor the recovery progress across multiple racks"}],"source_content_type":"text/x-rst","patch_set":4,"id":"51c4a7d3_53367f07","line":41,"updated":"2025-06-13 22:49:09.000000000","message":"Can Ironic do this generally?","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":38,"context_line":"During a critical network infrastructure failure affecting multiple racks,"},{"line_number":39,"context_line":"the operations team needs to:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"1. Quickly identify all nodes in the affected network segments"},{"line_number":42,"context_line":"2. Gracefully shut down workloads to prevent data corruption"},{"line_number":43,"context_line":"3. Power cycle network-dependent nodes to clear stuck states"},{"line_number":44,"context_line":"4. Monitor the recovery progress across multiple racks"}],"source_content_type":"text/x-rst","patch_set":4,"id":"e1232d2f_9fe9b4a6","line":41,"in_reply_to":"51c4a7d3_53367f07","updated":"2025-06-17 16:17:06.000000000","message":"Is a sub-section of the infrastructure really a justifiable reason for bulk operations?\n\nIf I loose a row or trip a side of a quad panel PDU, that is a relatively isolated failure...\n\nTo Jay\u0027s question, I think it can only do so if you can also independently track and match a portion of physical structural location. For example, I had a structure\n\u003cdatacenter\u003e-\u003crow ID\u003e-\u003ccabinet-id\u003e-\u003cphysical location by RU\u003e, but everyone I know treats this sort of differently.\n\nA physical location detail would sort of make a lot of sense.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":false,"context_lines":[{"line_number":38,"context_line":"During a critical network infrastructure failure affecting multiple racks,"},{"line_number":39,"context_line":"the operations team needs to:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"1. Quickly identify all nodes in the affected network segments"},{"line_number":42,"context_line":"2. Gracefully shut down workloads to prevent data corruption"},{"line_number":43,"context_line":"3. Power cycle network-dependent nodes to clear stuck states"},{"line_number":44,"context_line":"4. Monitor the recovery progress across multiple racks"}],"source_content_type":"text/x-rst","patch_set":4,"id":"64c357b9_1c9e47b8","line":41,"in_reply_to":"e1232d2f_9fe9b4a6","updated":"2025-07-03 01:17:01.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":39,"context_line":"the operations team needs to:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"1. Quickly identify all nodes in the affected network segments"},{"line_number":42,"context_line":"2. Gracefully shut down workloads to prevent data corruption"},{"line_number":43,"context_line":"3. Power cycle network-dependent nodes to clear stuck states"},{"line_number":44,"context_line":"4. Monitor the recovery progress across multiple racks"},{"line_number":45,"context_line":"5. Coordinate with network operations team for restoration"}],"source_content_type":"text/x-rst","patch_set":4,"id":"fe0e41dd_ef0cc2cc","line":42,"updated":"2025-06-17 16:17:06.000000000","message":"This one makes me think we\u0027re talking about a SAN failure, and at that point, there really is no way to gracefully shutdown. Your quickly in a \"just power it down\" or \"reboot it once the SAN is back\".\n\n(Linux kernel has historically, by default, marks read-only devices which are unresponsive for beyond 60 seconds. At which point, there is no preventing data corruption, the damage is done.)","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":39,"context_line":"the operations team needs to:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"1. Quickly identify all nodes in the affected network segments"},{"line_number":42,"context_line":"2. Gracefully shut down workloads to prevent data corruption"},{"line_number":43,"context_line":"3. Power cycle network-dependent nodes to clear stuck states"},{"line_number":44,"context_line":"4. Monitor the recovery progress across multiple racks"},{"line_number":45,"context_line":"5. Coordinate with network operations team for restoration"}],"source_content_type":"text/x-rst","patch_set":4,"id":"bc6e70f3_e3816fdd","line":42,"in_reply_to":"fe0e41dd_ef0cc2cc","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":40,"context_line":""},{"line_number":41,"context_line":"1. Quickly identify all nodes in the affected network segments"},{"line_number":42,"context_line":"2. Gracefully shut down workloads to prevent data corruption"},{"line_number":43,"context_line":"3. Power cycle network-dependent nodes to clear stuck states"},{"line_number":44,"context_line":"4. Monitor the recovery progress across multiple racks"},{"line_number":45,"context_line":"5. Coordinate with network operations team for restoration"},{"line_number":46,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"77e45262_52a26bf4","line":43,"updated":"2025-06-17 16:17:06.000000000","message":"Is this in relation to existing logic which handles power state management? Since stuck states shouldn\u0027t really be a thing for very long. More detail here would be good. Yes, it is lazy, its last updated first if memory sense. I guess clarity into the problems is really needed.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":40,"context_line":""},{"line_number":41,"context_line":"1. Quickly identify all nodes in the affected network segments"},{"line_number":42,"context_line":"2. Gracefully shut down workloads to prevent data corruption"},{"line_number":43,"context_line":"3. Power cycle network-dependent nodes to clear stuck states"},{"line_number":44,"context_line":"4. Monitor the recovery progress across multiple racks"},{"line_number":45,"context_line":"5. Coordinate with network operations team for restoration"},{"line_number":46,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"fb2a284c_e4d5e0dc","line":43,"in_reply_to":"77e45262_52a26bf4","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"1. Power Distribution Unit (PDU) Failure"},{"line_number":58,"context_line":"   - Sudden power loss to multiple racks"},{"line_number":59,"context_line":"   - Need to gracefully shut down nodes before UPS depletion"},{"line_number":60,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"2. Cooling System Failure"}],"source_content_type":"text/x-rst","patch_set":4,"id":"6a9ac366_fc0ca6a8","line":59,"updated":"2025-06-17 16:17:06.000000000","message":"So... Most larger datacenters, typically centralize power management such that UPSes are before the PDUs.\n\n[transfer switch with (grid|generator)] -\u003e UPS Bypass panels with UPS attached -\u003e Transformers -\u003e PDU distribution panel -\u003e Circuits out to the Racks -\u003e PDU power-strip -\u003e Server.\n\nThere *is* a trend of doing Direct Current powered data centers, as well.\n\nIn those cases, for example what google has done, build small packs into the racks as well, but less common and more along the lines of custom hardware and not COTS.\n\nSome of this has shifted over the years with increased density, but facility high capacity UPSes are almost always before the PDU cabinets. There are separate levels of PDUs, but this feels more like \"I lost a power strip\" and most data center operators intentionally spread loads across power strips, breakers, and even transformers. In such a failure, that is often not a \"power down\" anything case, but identify the reduced capacity/issue and then addressing that specific issue.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"1. Power Distribution Unit (PDU) Failure"},{"line_number":58,"context_line":"   - Sudden power loss to multiple racks"},{"line_number":59,"context_line":"   - Need to gracefully shut down nodes before UPS depletion"},{"line_number":60,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"2. Cooling System Failure"}],"source_content_type":"text/x-rst","patch_set":4,"id":"8b2a7a49_03508242","line":59,"in_reply_to":"6a9ac366_fc0ca6a8","updated":"2025-07-03 01:17:01.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":59,"context_line":"   - Need to gracefully shut down nodes before UPS depletion"},{"line_number":60,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"2. Cooling System Failure"},{"line_number":63,"context_line":"   - Rising temperatures in affected areas"},{"line_number":64,"context_line":"   - Need to prioritize shutdown of heat-sensitive hardware"},{"line_number":65,"context_line":"   - Coordinate with environmental monitoring systems"}],"source_content_type":"text/x-rst","patch_set":4,"id":"98b18f38_991e0c48","line":62,"updated":"2025-06-17 16:17:06.000000000","message":"This has happened to me soooooooo many times.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":59,"context_line":"   - Need to gracefully shut down nodes before UPS depletion"},{"line_number":60,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"2. Cooling System Failure"},{"line_number":63,"context_line":"   - Rising temperatures in affected areas"},{"line_number":64,"context_line":"   - Need to prioritize shutdown of heat-sensitive hardware"},{"line_number":65,"context_line":"   - Coordinate with environmental monitoring systems"}],"source_content_type":"text/x-rst","patch_set":4,"id":"e1e87133_98dbcbde","line":62,"in_reply_to":"98b18f38_991e0c48","updated":"2025-08-05 03:28:03.000000000","message":"Acknowledged","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":63,"context_line":"   - Rising temperatures in affected areas"},{"line_number":64,"context_line":"   - Need to prioritize shutdown of heat-sensitive hardware"},{"line_number":65,"context_line":"   - Coordinate with environmental monitoring systems"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"3. Security Breach"},{"line_number":68,"context_line":"   - Need to quickly isolate compromised nodes"},{"line_number":69,"context_line":"   - Prevent lateral movement across the infrastructure"},{"line_number":70,"context_line":"   - Maintain audit trail of emergency actions"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"4. Natural Disaster Preparation"},{"line_number":73,"context_line":"   - Preemptive shutdown of non-critical systems"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3508e1d9_377ac23f","line":70,"range":{"start_line":66,"start_character":0,"end_line":70,"end_character":46},"updated":"2025-06-17 16:17:06.000000000","message":"Breaches, while a disaster, might need to be considered out of scope?\n\nUnless we want to expose \"shutdown all network interfaces\" as a thing?!","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"6e999fd26f033e51c3fbe9a1837cb5fd30d62567","unresolved":false,"context_lines":[{"line_number":63,"context_line":"   - Rising temperatures in affected areas"},{"line_number":64,"context_line":"   - Need to prioritize shutdown of heat-sensitive hardware"},{"line_number":65,"context_line":"   - Coordinate with environmental monitoring systems"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"3. Security Breach"},{"line_number":68,"context_line":"   - Need to quickly isolate compromised nodes"},{"line_number":69,"context_line":"   - Prevent lateral movement across the infrastructure"},{"line_number":70,"context_line":"   - Maintain audit trail of emergency actions"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"4. Natural Disaster Preparation"},{"line_number":73,"context_line":"   - Preemptive shutdown of non-critical systems"}],"source_content_type":"text/x-rst","patch_set":4,"id":"04bf3774_05f6a511","line":70,"range":{"start_line":66,"start_character":0,"end_line":70,"end_character":46},"in_reply_to":"3508e1d9_377ac23f","updated":"2025-06-25 21:05:38.000000000","message":"Yeah, good call, I just put out a bunch of suggestions to understand the field.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"0f5ed4b53bee73f4fb90aabf71828f9432887440","unresolved":true,"context_lines":[{"line_number":87,"context_line":"7. Storage System Failures"},{"line_number":88,"context_line":"   - Critical storage infrastructure issues"},{"line_number":89,"context_line":"   - Need to safely shut down dependent nodes"},{"line_number":90,"context_line":"   - Prevent data corruption"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":93,"context_line":"framework in Ironic that can handle various types of emergencies with"}],"source_content_type":"text/x-rst","patch_set":4,"id":"6e9414c5_309937b7","line":90,"updated":"2025-06-13 22:49:09.000000000","message":"These use cases should be more specific and less plentiful. Lets pick one or two solid examples and flesh them out like the primary scenario","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"65b4687d60a74c8788947eb3b295bc2f5d296b38","unresolved":true,"context_lines":[{"line_number":87,"context_line":"7. Storage System Failures"},{"line_number":88,"context_line":"   - Critical storage infrastructure issues"},{"line_number":89,"context_line":"   - Need to safely shut down dependent nodes"},{"line_number":90,"context_line":"   - Prevent data corruption"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":93,"context_line":"framework in Ironic that can handle various types of emergencies with"}],"source_content_type":"text/x-rst","patch_set":4,"id":"afc2b929_c7342e7f","line":90,"in_reply_to":"6e9414c5_309937b7","updated":"2025-06-17 16:17:06.000000000","message":"I concur, and I\u0027d focus around power + cooling.\n\nFor example, storage failures, the damage is done so quickly you can\u0027t really respond that quickly. There is no safe shutdown in those cases.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"6e999fd26f033e51c3fbe9a1837cb5fd30d62567","unresolved":false,"context_lines":[{"line_number":87,"context_line":"7. Storage System Failures"},{"line_number":88,"context_line":"   - Critical storage infrastructure issues"},{"line_number":89,"context_line":"   - Need to safely shut down dependent nodes"},{"line_number":90,"context_line":"   - Prevent data corruption"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":93,"context_line":"framework in Ironic that can handle various types of emergencies with"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9788a96a_014cca4d","line":90,"in_reply_to":"afc2b929_c7342e7f","updated":"2025-06-25 21:05:38.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":92,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":93,"context_line":"framework in Ironic that can handle various types of emergencies with"},{"line_number":94,"context_line":"appropriate safety measures and coordination capabilities."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Proposed change"},{"line_number":97,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":98,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff3a0be6_208e9992","line":95,"updated":"2025-06-17 17:00:20.000000000","message":"So, an item to frame here:\n\nIs ironic up? Is it down?","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":92,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":93,"context_line":"framework in Ironic that can handle various types of emergencies with"},{"line_number":94,"context_line":"appropriate safety measures and coordination capabilities."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Proposed change"},{"line_number":97,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":98,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"bb099a4a_6ee08cbd","line":95,"in_reply_to":"2a4cde8f_2c1f12a5","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"6e999fd26f033e51c3fbe9a1837cb5fd30d62567","unresolved":true,"context_lines":[{"line_number":92,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":93,"context_line":"framework in Ironic that can handle various types of emergencies with"},{"line_number":94,"context_line":"appropriate safety measures and coordination capabilities."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Proposed change"},{"line_number":97,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":98,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"2a4cde8f_2c1f12a5","line":95,"in_reply_to":"ff3a0be6_208e9992","updated":"2025-06-25 21:05:38.000000000","message":"I would argue that this draft is about handling both? Me and Jay discussed that going online and offline are pretty similar so I think tooling to support both is worth having in the same spec.\n\nAdding a line to clarify that","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":108,"context_line":"2. Emergency Recovery"},{"line_number":109,"context_line":"   - Power on nodes"},{"line_number":110,"context_line":"   - Clear maintenance mode"},{"line_number":111,"context_line":"   - Restore normal operation state"},{"line_number":112,"context_line":"   - Validate node health"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"3. Node Targeting"}],"source_content_type":"text/x-rst","patch_set":4,"id":"708ff68e_b5a08609","line":111,"updated":"2025-06-17 17:00:20.000000000","message":"Could we clarify what action this would be?","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"6e999fd26f033e51c3fbe9a1837cb5fd30d62567","unresolved":true,"context_lines":[{"line_number":108,"context_line":"2. Emergency Recovery"},{"line_number":109,"context_line":"   - Power on nodes"},{"line_number":110,"context_line":"   - Clear maintenance mode"},{"line_number":111,"context_line":"   - Restore normal operation state"},{"line_number":112,"context_line":"   - Validate node health"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"3. Node Targeting"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7ff11569_c91c5258","line":111,"in_reply_to":"708ff68e_b5a08609","updated":"2025-06-25 21:05:38.000000000","message":"Is your confusion just having the phrase \"Power on nodes\"? I would rather remove it here and define \"Restore normal operation state\" further down the page if that suits.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":108,"context_line":"2. Emergency Recovery"},{"line_number":109,"context_line":"   - Power on nodes"},{"line_number":110,"context_line":"   - Clear maintenance mode"},{"line_number":111,"context_line":"   - Restore normal operation state"},{"line_number":112,"context_line":"   - Validate node health"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"3. Node Targeting"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7eea9916_01085644","line":111,"in_reply_to":"7ff11569_c91c5258","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":109,"context_line":"   - Power on nodes"},{"line_number":110,"context_line":"   - Clear maintenance mode"},{"line_number":111,"context_line":"   - Restore normal operation state"},{"line_number":112,"context_line":"   - Validate node health"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"3. Node Targeting"},{"line_number":115,"context_line":"   - Global operations (all managed nodes)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"41746b24_344d12c8","line":112,"updated":"2025-06-17 17:00:20.000000000","message":"Are we just talking about forcing power sync to occur?","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":false,"context_lines":[{"line_number":109,"context_line":"   - Power on nodes"},{"line_number":110,"context_line":"   - Clear maintenance mode"},{"line_number":111,"context_line":"   - Restore normal operation state"},{"line_number":112,"context_line":"   - Validate node health"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"3. Node Targeting"},{"line_number":115,"context_line":"   - Global operations (all managed nodes)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f8adf4d_fba1062f","line":112,"in_reply_to":"2394b83c_f34b8929","updated":"2025-07-03 01:17:01.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"6e999fd26f033e51c3fbe9a1837cb5fd30d62567","unresolved":true,"context_lines":[{"line_number":109,"context_line":"   - Power on nodes"},{"line_number":110,"context_line":"   - Clear maintenance mode"},{"line_number":111,"context_line":"   - Restore normal operation state"},{"line_number":112,"context_line":"   - Validate node health"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"3. Node Targeting"},{"line_number":115,"context_line":"   - Global operations (all managed nodes)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"2394b83c_f34b8929","line":112,"in_reply_to":"41746b24_344d12c8","updated":"2025-06-25 21:05:38.000000000","message":"I haven\u0027t gotten that far in writing yet, but from a high-level, we assume that the database + conductors are running, and go from there\n\nI will write in soon details about what each endpoint in the API will do exactly.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"0f5ed4b53bee73f4fb90aabf71828f9432887440","unresolved":true,"context_lines":[{"line_number":114,"context_line":"3. Node Targeting"},{"line_number":115,"context_line":"   - Global operations (all managed nodes)"},{"line_number":116,"context_line":"   - Targeted operations using:"},{"line_number":117,"context_line":"     * Instance name patterns (e.g., prod-*) [#]_"},{"line_number":118,"context_line":"     * Resource class (e.g., compute-high-performance)"},{"line_number":119,"context_line":"     * Project metadata"},{"line_number":120,"context_line":"     * Conductor group"}],"source_content_type":"text/x-rst","patch_set":4,"id":"f832fd0f_39814d0a","line":117,"updated":"2025-06-13 22:49:09.000000000","message":"this will require elevating node.instance_name to a top level attribute, which we should do anyway","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":false,"context_lines":[{"line_number":114,"context_line":"3. Node Targeting"},{"line_number":115,"context_line":"   - Global operations (all managed nodes)"},{"line_number":116,"context_line":"   - Targeted operations using:"},{"line_number":117,"context_line":"     * Instance name patterns (e.g., prod-*) [#]_"},{"line_number":118,"context_line":"     * Resource class (e.g., compute-high-performance)"},{"line_number":119,"context_line":"     * Project metadata"},{"line_number":120,"context_line":"     * Conductor group"}],"source_content_type":"text/x-rst","patch_set":4,"id":"0219d58c_92744cf8","line":117,"in_reply_to":"d0d7fd02_10a15906","updated":"2025-07-03 01:17:01.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":114,"context_line":"3. Node Targeting"},{"line_number":115,"context_line":"   - Global operations (all managed nodes)"},{"line_number":116,"context_line":"   - Targeted operations using:"},{"line_number":117,"context_line":"     * Instance name patterns (e.g., prod-*) [#]_"},{"line_number":118,"context_line":"     * Resource class (e.g., compute-high-performance)"},{"line_number":119,"context_line":"     * Project metadata"},{"line_number":120,"context_line":"     * Conductor group"}],"source_content_type":"text/x-rst","patch_set":4,"id":"d0d7fd02_10a15906","line":117,"in_reply_to":"f832fd0f_39814d0a","updated":"2025-06-17 17:00:20.000000000","message":"++","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":120,"context_line":"     * Conductor group"},{"line_number":121,"context_line":"     * Custom node properties"},{"line_number":122,"context_line":"     * Maintenance state"},{"line_number":123,"context_line":"     * Power state"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"4. Operation Control"},{"line_number":126,"context_line":"   - Dry-run mode to preview affected nodes"}],"source_content_type":"text/x-rst","patch_set":4,"id":"99102811_3e849362","line":123,"updated":"2025-06-17 17:00:20.000000000","message":"I\u0027d consider, if we\u0027re focused/worried about PDU failures (think https://www.vertiv.com/en-in/products-catalog/critical-power/power-distribution/vertiv-liebert-rxv-distribution-unit-test/), then I would consider also adding a user oriented location field which could be searched/filtered against.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":120,"context_line":"     * Conductor group"},{"line_number":121,"context_line":"     * Custom node properties"},{"line_number":122,"context_line":"     * Maintenance state"},{"line_number":123,"context_line":"     * Power state"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"4. Operation Control"},{"line_number":126,"context_line":"   - Dry-run mode to preview affected nodes"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dc54745b_652a13b9","line":123,"in_reply_to":"0c95adfb_0770b677","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"6e999fd26f033e51c3fbe9a1837cb5fd30d62567","unresolved":true,"context_lines":[{"line_number":120,"context_line":"     * Conductor group"},{"line_number":121,"context_line":"     * Custom node properties"},{"line_number":122,"context_line":"     * Maintenance state"},{"line_number":123,"context_line":"     * Power state"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"4. Operation Control"},{"line_number":126,"context_line":"   - Dry-run mode to preview affected nodes"}],"source_content_type":"text/x-rst","patch_set":4,"id":"0c95adfb_0770b677","line":123,"in_reply_to":"99102811_3e849362","updated":"2025-06-25 21:05:38.000000000","message":"This sounds like it deserves its own feature in a separate spec?","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":129,"context_line":"   - Concurrent operation limits"},{"line_number":130,"context_line":"   - Operation status tracking"},{"line_number":131,"context_line":"   - Detailed operation logs"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Dry Run Mode"},{"line_number":134,"context_line":"~~~~~~~~~~~~"},{"line_number":135,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"c3f5d464_2231af2b","line":132,"updated":"2025-06-17 17:00:20.000000000","message":"I\u0027ll toss this out there, and we can discuss:\n\nOne of the cases where one of the operators I know would love, is a bulk/emergency shutdown, but then allowing for natural recovery of the data center with one aspect:\n\n1) Can\u0027t just turn everything back on at once, need to be able to stage systems out.\n1a) Think in-rush current combined with spinup power consumption.\n1b) This may take up to several minutes for a given node.\n2) power up critical infrastructure first: For example localized conductors, network nodes, various rounds of high power nodes and then allow the recovery to trickle through the remaining nodes.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":129,"context_line":"   - Concurrent operation limits"},{"line_number":130,"context_line":"   - Operation status tracking"},{"line_number":131,"context_line":"   - Detailed operation logs"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Dry Run Mode"},{"line_number":134,"context_line":"~~~~~~~~~~~~"},{"line_number":135,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7cf98b53_897f0d21","line":132,"in_reply_to":"1845ad61_bdeb0fe8","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":129,"context_line":"   - Concurrent operation limits"},{"line_number":130,"context_line":"   - Operation status tracking"},{"line_number":131,"context_line":"   - Detailed operation logs"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Dry Run Mode"},{"line_number":134,"context_line":"~~~~~~~~~~~~"},{"line_number":135,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1845ad61_bdeb0fe8","line":132,"in_reply_to":"7ffe66eb_a27270f5","updated":"2025-07-10 14:22:41.000000000","message":"Thanks!","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"6e999fd26f033e51c3fbe9a1837cb5fd30d62567","unresolved":true,"context_lines":[{"line_number":129,"context_line":"   - Concurrent operation limits"},{"line_number":130,"context_line":"   - Operation status tracking"},{"line_number":131,"context_line":"   - Detailed operation logs"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Dry Run Mode"},{"line_number":134,"context_line":"~~~~~~~~~~~~"},{"line_number":135,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7ffe66eb_a27270f5","line":132,"in_reply_to":"c3f5d464_2231af2b","updated":"2025-06-25 21:05:38.000000000","message":"Yeah, so I am planning on having this in this spec, just bare-with.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"0f5ed4b53bee73f4fb90aabf71828f9432887440","unresolved":true,"context_lines":[{"line_number":179,"context_line":"                }"},{"line_number":180,"context_line":"            ],"},{"line_number":181,"context_line":"            \"required_permissions\": ["},{"line_number":182,"context_line":"                \"baremetal:node:power_off\","},{"line_number":183,"context_line":"                \"baremetal:node:set_maintenance\""},{"line_number":184,"context_line":"            ],"},{"line_number":185,"context_line":"            \"potential_risks\": ["}],"source_content_type":"text/x-rst","patch_set":4,"id":"2a5539a9_8d294381","line":182,"updated":"2025-06-13 22:49:09.000000000","message":"Arguably even being able to see the results of the command should be locked behind some elevated permissions. Also, we probably can\u0027t do this in a reliable way in the presence of non-default policy.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":false,"context_lines":[{"line_number":179,"context_line":"                }"},{"line_number":180,"context_line":"            ],"},{"line_number":181,"context_line":"            \"required_permissions\": ["},{"line_number":182,"context_line":"                \"baremetal:node:power_off\","},{"line_number":183,"context_line":"                \"baremetal:node:set_maintenance\""},{"line_number":184,"context_line":"            ],"},{"line_number":185,"context_line":"            \"potential_risks\": ["}],"source_content_type":"text/x-rst","patch_set":4,"id":"8886ac8a_e5109e92","line":182,"in_reply_to":"2a5539a9_8d294381","updated":"2025-07-03 01:17:01.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":229,"context_line":""},{"line_number":230,"context_line":"Add new tables:"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"1. emergency_operation"},{"line_number":233,"context_line":"   - id (UUID)"},{"line_number":234,"context_line":"   - operation_type (enum: shutdown, recover)"},{"line_number":235,"context_line":"   - status (enum: pending, in_progress, completed, failed)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"71c3b24e_d0de29d2","line":232,"updated":"2025-06-17 17:00:20.000000000","message":"Just to make sure, this is 1:1 to the request, not 1 request to many entries.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":229,"context_line":""},{"line_number":230,"context_line":"Add new tables:"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"1. emergency_operation"},{"line_number":233,"context_line":"   - id (UUID)"},{"line_number":234,"context_line":"   - operation_type (enum: shutdown, recover)"},{"line_number":235,"context_line":"   - status (enum: pending, in_progress, completed, failed)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1f09d8f0_4f8dfba9","line":232,"in_reply_to":"2cdf9026_b62a07e5","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":true,"context_lines":[{"line_number":229,"context_line":""},{"line_number":230,"context_line":"Add new tables:"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"1. emergency_operation"},{"line_number":233,"context_line":"   - id (UUID)"},{"line_number":234,"context_line":"   - operation_type (enum: shutdown, recover)"},{"line_number":235,"context_line":"   - status (enum: pending, in_progress, completed, failed)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"2cdf9026_b62a07e5","line":232,"in_reply_to":"71c3b24e_d0de29d2","updated":"2025-07-03 01:17:01.000000000","message":"Yep - 1 entry per operation","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"f7c80b0a4a1a726d9be23c73306a0f426753eefd","unresolved":true,"context_lines":[{"line_number":243,"context_line":"2. emergency_operation_node"},{"line_number":244,"context_line":"   - operation_id (UUID, foreign key)"},{"line_number":245,"context_line":"   - node_id (UUID, foreign key)"},{"line_number":246,"context_line":"   - status (enum: pending, in_progress, completed, failed)"},{"line_number":247,"context_line":"   - error_message (text)"},{"line_number":248,"context_line":"   - started_at (timestamp)"},{"line_number":249,"context_line":"   - completed_at (timestamp)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7e2f9e76_3dd3c07f","line":246,"range":{"start_line":246,"start_character":12,"end_line":246,"end_character":59},"updated":"2025-06-17 17:00:20.000000000","message":"So, I\u0027ll be honest, this is kind of concerning performance wise when one of the motivation is speed for such a failure.\n\nInserts are slow, updates are going to be a little slower, Perhaps the exception shouldn\u0027t be the rule?\n\nI guess another thing is we don\u0027t want to check two tables to return a node, so do we consider instead having an actual node state?","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"d181fc5a99c2090033514a6964138ba8a9e91e7a","unresolved":false,"context_lines":[{"line_number":243,"context_line":"2. emergency_operation_node"},{"line_number":244,"context_line":"   - operation_id (UUID, foreign key)"},{"line_number":245,"context_line":"   - node_id (UUID, foreign key)"},{"line_number":246,"context_line":"   - status (enum: pending, in_progress, completed, failed)"},{"line_number":247,"context_line":"   - error_message (text)"},{"line_number":248,"context_line":"   - started_at (timestamp)"},{"line_number":249,"context_line":"   - completed_at (timestamp)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9441c3fc_a1c14a55","line":246,"range":{"start_line":246,"start_character":12,"end_line":246,"end_character":59},"in_reply_to":"3cf1d3cc_a0452987","updated":"2025-07-03 01:17:01.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"55a7e5fd23be24d7501f1b091a75c68fb41a405f","unresolved":true,"context_lines":[{"line_number":243,"context_line":"2. emergency_operation_node"},{"line_number":244,"context_line":"   - operation_id (UUID, foreign key)"},{"line_number":245,"context_line":"   - node_id (UUID, foreign key)"},{"line_number":246,"context_line":"   - status (enum: pending, in_progress, completed, failed)"},{"line_number":247,"context_line":"   - error_message (text)"},{"line_number":248,"context_line":"   - started_at (timestamp)"},{"line_number":249,"context_line":"   - completed_at (timestamp)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3cf1d3cc_a0452987","line":246,"range":{"start_line":246,"start_character":12,"end_line":246,"end_character":59},"in_reply_to":"7e2f9e76_3dd3c07f","updated":"2025-06-17 18:48:14.000000000","message":"yeah, maybe node.power_state[EMERGENCY_x] for off/on/","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"0f5ed4b53bee73f4fb90aabf71828f9432887440","unresolved":true,"context_lines":[{"line_number":389,"context_line":""},{"line_number":390,"context_line":".. [#] Note: The instance_name field is not currently available in Ironic but is planned"},{"line_number":391,"context_line":"    for a future spec. This emergency operations feature will be implemented to work"},{"line_number":392,"context_line":"    with this field once it becomes available."},{"line_number":393,"context_line":""},{"line_number":394,"context_line":"1. Power fault recovery spec"},{"line_number":395,"context_line":"2. Manual cleaning spec"}],"source_content_type":"text/x-rst","patch_set":4,"id":"12cea795_93d65ea6","line":392,"updated":"2025-06-13 22:49:09.000000000","message":"If you want bonus points, go ahead and file this RFE. CID should be able to help review to ensure it\u0027s good.","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ce031d403306aaf9cd90f04ef2e76bb1eb2b4e2e","unresolved":false,"context_lines":[{"line_number":389,"context_line":""},{"line_number":390,"context_line":".. [#] Note: The instance_name field is not currently available in Ironic but is planned"},{"line_number":391,"context_line":"    for a future spec. This emergency operations feature will be implemented to work"},{"line_number":392,"context_line":"    with this field once it becomes available."},{"line_number":393,"context_line":""},{"line_number":394,"context_line":"1. Power fault recovery spec"},{"line_number":395,"context_line":"2. Manual cleaning spec"}],"source_content_type":"text/x-rst","patch_set":4,"id":"fb091af4_d3405cb8","line":392,"in_reply_to":"12cea795_93d65ea6","updated":"2025-06-25 21:11:46.000000000","message":"Done","commit_id":"ce60b465930d98cc364210b8b953db67b3e65037"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"ace324d4abebb19058d0e1bf451a1a8d63cf7644","unresolved":true,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"1. Power Distribution Unit (PDU) Failure or other Power Failures"},{"line_number":36,"context_line":"   - Sudden power loss to multiple racks"},{"line_number":37,"context_line":"   - Need to shut down nodes before UPS depletion"},{"line_number":38,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"2. Cooling System Failure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"12690617_45f7e2e2","line":37,"range":{"start_line":37,"start_character":0,"end_line":37,"end_character":49},"updated":"2025-07-10 15:06:45.000000000","message":"In our DC we have ~10 min UPS coverage before the racks go down in case of a power cut. We have a notification system in place to which instances can subscribe and where they get notified in case a power cut is detected. It is then up to the instance/user/application to decide how to react, e.g. flush caches, cleaning shut down services, ignore, ... on the Ironic/OpenStack/hypervisor layer we always said we would keep the physical machine running as long as we can, aka do nothing :) That seems to be different from your use case as you trigger a shutdown for the underlying infrastructure ... can you elaborate on why that makes sense in your env?","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"46d943cc5298604e93ed3f3c1d03bcdd36e2e259","unresolved":true,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"1. Power Distribution Unit (PDU) Failure or other Power Failures"},{"line_number":36,"context_line":"   - Sudden power loss to multiple racks"},{"line_number":37,"context_line":"   - Need to shut down nodes before UPS depletion"},{"line_number":38,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"2. Cooling System Failure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"7273eac0_db1a981c","line":37,"range":{"start_line":37,"start_character":0,"end_line":37,"end_character":49},"in_reply_to":"12690617_45f7e2e2","updated":"2025-07-10 16:07:07.000000000","message":"Just to chime in, I think CERN\u0027s approach might make a lot of sense, and maybe ironic needs a \"I\u0027m about to loose power soon loudspeaker of some sort\" to enable workloads to commit their work and shutdown... that being said we\u0027re agnostic to the workloads and operators do need a way to facilitate controlled power down of the workloads, if only to prevent the massive in-rush surge which would result of having heavy loads just power back on automatically.\n\nSpeaking from experience as well, some operators need to keep some workloads online as long as possible or prevent them from going down entirely and there is a trade-off decision of \"okay, if I shed the un-necessary load, how much time can I buy on the UPS infrastructure.  That is one of those active decisions to make in response to the current state of the disaster. While Ironic cannot provide that clarity, the swiss army knife of massively parallelism through the conductor to being to shed the workload for operators with hundreds or thousands of machines is a huge time saver.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"7b79d02a3ead9f691bb78bc4c90a97c3adb5db33","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"1. Power Distribution Unit (PDU) Failure or other Power Failures"},{"line_number":36,"context_line":"   - Sudden power loss to multiple racks"},{"line_number":37,"context_line":"   - Need to shut down nodes before UPS depletion"},{"line_number":38,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"2. Cooling System Failure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"cad893cb_dd7f16d4","line":37,"range":{"start_line":37,"start_character":0,"end_line":37,"end_character":49},"in_reply_to":"2d6b7cf7_4ab6ecfa","updated":"2025-09-23 15:39:03.000000000","message":"I\u0027m sort of wondering, if we have an API resource that could go \"yay/nay\" to a request if the instance or node uuid, is if the node knows it, should begin powering down. Downside is, that is a huge DB hit for thousands of nodes doing such polling. I guess there are several different approaches here which could be beneficial but really depends on the use case/model/operational approach.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"1. Power Distribution Unit (PDU) Failure or other Power Failures"},{"line_number":36,"context_line":"   - Sudden power loss to multiple racks"},{"line_number":37,"context_line":"   - Need to shut down nodes before UPS depletion"},{"line_number":38,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"2. Cooling System Failure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"2d6b7cf7_4ab6ecfa","line":37,"range":{"start_line":37,"start_character":0,"end_line":37,"end_character":49},"in_reply_to":"38d98323_de30e523","updated":"2025-08-05 03:28:03.000000000","message":"I agree. I have removed the UPS usecase, explained why at the bottom of the doc, and added the equipment protection idea","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"b5ebd63473b13f5b33081c215d04195476c42b5d","unresolved":true,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"1. Power Distribution Unit (PDU) Failure or other Power Failures"},{"line_number":36,"context_line":"   - Sudden power loss to multiple racks"},{"line_number":37,"context_line":"   - Need to shut down nodes before UPS depletion"},{"line_number":38,"context_line":"   - Coordinate power restoration across affected racks"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"2. Cooling System Failure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"38d98323_de30e523","line":37,"range":{"start_line":37,"start_character":0,"end_line":37,"end_character":49},"in_reply_to":"7273eac0_db1a981c","updated":"2025-07-11 08:17:25.000000000","message":"What you describe is exactly how our \"loudspeaker\" works: it figures out from an API whether there is a power cut, and puts a message on a message bus. Applications can subscribe and react as they wish. All fully outside of Ironic and at the application/instance level.\n\nThere are clearly also use cases on the infrastructure level like emergency shutdowns, e.g. due to unexpected cooling loss and risk of overheating, usually in localised areas of a data center. In this case you probably cannot inform instances and let others take the decision as you need to protect the equipment. If Ironic had means to shut off nodes in the overheated area, that would be really useful, and I understand this is one of the use cases covered by this spec.\n\nThe approach of shedding unnecessary load is also an idea we were  considering, i.e. switching of our batch computing to provide additional UPS time to the more sensitive services. The question is how big the effect will be since you will need some time to assert there is a cut coming and you will need time to drain/flush/shutdown the load you want to shed ... given we\u0027re talking about minutes in total for the UPS, how much will you gain? Complicated :)","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":45,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":46,"context_line":"framework in Ironic that can handle various types of emergencies with"},{"line_number":47,"context_line":"appropriate safety measures and coordination capabilities."},{"line_number":48,"context_line":"There should be a framework to support quickly shutting Ironic down,"},{"line_number":49,"context_line":"as well as quickly bringing it back up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Example: Power Failure Response"}],"source_content_type":"text/x-rst","patch_set":6,"id":"73c6ae6c_f90ee0a4","line":48,"range":{"start_line":48,"start_character":63,"end_line":48,"end_character":67},"updated":"2025-07-10 14:22:41.000000000","message":"nit, an Ironic managed fleet down :)","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":45,"context_line":"These scenarios highlight the need for a robust, built-in emergency operations"},{"line_number":46,"context_line":"framework in Ironic that can handle various types of emergencies with"},{"line_number":47,"context_line":"appropriate safety measures and coordination capabilities."},{"line_number":48,"context_line":"There should be a framework to support quickly shutting Ironic down,"},{"line_number":49,"context_line":"as well as quickly bringing it back up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Example: Power Failure Response"}],"source_content_type":"text/x-rst","patch_set":6,"id":"ffdec17d_14e216e9","line":48,"range":{"start_line":48,"start_character":63,"end_line":48,"end_character":67},"in_reply_to":"73c6ae6c_f90ee0a4","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"ace324d4abebb19058d0e1bf451a1a8d63cf7644","unresolved":true,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Consider a scenario where a PDU failure affects racks 12-18 in a datacenter"},{"line_number":55,"context_line":"with 200+ nodes. The operator needs to perform a coordinated shutdown."},{"line_number":56,"context_line":"Using descriptive `instance_name`values which the operator had already set for his key nodes(e.g., `k8s-control-plane-01`, `postgres-primary`, `etcd-01`), the operator can:"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"1. **Emergency Shutdown**: Execute `openstack baremetal emergency shutdown"},{"line_number":59,"context_line":"   --resource-class compute --graceful --timeout 300` to safely power"}],"source_content_type":"text/x-rst","patch_set":6,"id":"c4464750_2a380bdc","line":56,"range":{"start_line":56,"start_character":0,"end_line":56,"end_character":92},"updated":"2025-07-10 15:06:45.000000000","message":"Is this duplicating the instance name into Ironic? This also seems to assume the Ironic admin is aware of the instances, but the Ironic operator/admin and the instance owner will in many cases be in totally different units. Would the project be a sensible selector? I imagine starting up the \"Active Directory\", \"Ceph\", \"Databases\", ... projects in order.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"b5ebd63473b13f5b33081c215d04195476c42b5d","unresolved":true,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Consider a scenario where a PDU failure affects racks 12-18 in a datacenter"},{"line_number":55,"context_line":"with 200+ nodes. The operator needs to perform a coordinated shutdown."},{"line_number":56,"context_line":"Using descriptive `instance_name`values which the operator had already set for his key nodes(e.g., `k8s-control-plane-01`, `postgres-primary`, `etcd-01`), the operator can:"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"1. **Emergency Shutdown**: Execute `openstack baremetal emergency shutdown"},{"line_number":59,"context_line":"   --resource-class compute --graceful --timeout 300` to safely power"}],"source_content_type":"text/x-rst","patch_set":6,"id":"cec6322d_12b5279e","line":56,"range":{"start_line":56,"start_character":0,"end_line":56,"end_character":92},"in_reply_to":"366d13ce_881da97e","updated":"2025-07-11 08:17:25.000000000","message":"Checking with my facility colleagues, they will power on the DC by rows of racks, so yes, from our perspective location is likely something very useful when orchestrating power management, see also my comment above on localised cooling issues above.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"46d943cc5298604e93ed3f3c1d03bcdd36e2e259","unresolved":true,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Consider a scenario where a PDU failure affects racks 12-18 in a datacenter"},{"line_number":55,"context_line":"with 200+ nodes. The operator needs to perform a coordinated shutdown."},{"line_number":56,"context_line":"Using descriptive `instance_name`values which the operator had already set for his key nodes(e.g., `k8s-control-plane-01`, `postgres-primary`, `etcd-01`), the operator can:"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"1. **Emergency Shutdown**: Execute `openstack baremetal emergency shutdown"},{"line_number":59,"context_line":"   --resource-class compute --graceful --timeout 300` to safely power"}],"source_content_type":"text/x-rst","patch_set":6,"id":"366d13ce_881da97e","line":56,"range":{"start_line":56,"start_character":0,"end_line":56,"end_character":92},"in_reply_to":"c4464750_2a380bdc","updated":"2025-07-10 16:07:07.000000000","message":"There is discussion, and I think even a WIP patch to do so, but I concur with your concern. I\u0027m unsure it would really be a sensible selector unless you need to re-power the etcd cluster, for example.\n\nThis is kind of why I\u0027ve been thinking location, because then it potentially becomes easy enough to convert over to to the physical locatino which would then relate to what PDU/Transformer/UPS/Generator/Grid Connection power is drawn from as well.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":true,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Consider a scenario where a PDU failure affects racks 12-18 in a datacenter"},{"line_number":55,"context_line":"with 200+ nodes. The operator needs to perform a coordinated shutdown."},{"line_number":56,"context_line":"Using descriptive `instance_name`values which the operator had already set for his key nodes(e.g., `k8s-control-plane-01`, `postgres-primary`, `etcd-01`), the operator can:"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"1. **Emergency Shutdown**: Execute `openstack baremetal emergency shutdown"},{"line_number":59,"context_line":"   --resource-class compute --graceful --timeout 300` to safely power"}],"source_content_type":"text/x-rst","patch_set":6,"id":"e5d4f352_d2183564","line":56,"range":{"start_line":56,"start_character":0,"end_line":56,"end_character":92},"in_reply_to":"cec6322d_12b5279e","updated":"2025-08-05 03:28:03.000000000","message":"After discussing with Julia, I agree. instance_name is more for recovery (i.e. bring k8s control plane nodes up first) where-as location based is more useful for shutdown. I am still not 100% sure how to input that information into Ironic but I\u0027ll get some ideas down.\n\nBut also from the below point (#86 which I will resolve now) I agree this is a useful feature both ways\n\nThe WIP is here: https://review.opendev.org/c/openstack/ironic/+/952790 and is also in the spec","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Consider a scenario where a PDU failure affects racks 12-18 in a datacenter"},{"line_number":55,"context_line":"with 200+ nodes. The operator needs to perform a coordinated shutdown."},{"line_number":56,"context_line":"Using descriptive `instance_name`values which the operator had already set for his key nodes(e.g., `k8s-control-plane-01`, `postgres-primary`, `etcd-01`), the operator can:"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"1. **Emergency Shutdown**: Execute `openstack baremetal emergency shutdown"},{"line_number":59,"context_line":"   --resource-class compute --graceful --timeout 300` to safely power"}],"source_content_type":"text/x-rst","patch_set":6,"id":"363a387b_08890989","line":56,"range":{"start_line":56,"start_character":0,"end_line":56,"end_character":92},"in_reply_to":"e5d4f352_d2183564","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"ace324d4abebb19058d0e1bf451a1a8d63cf7644","unresolved":true,"context_lines":[{"line_number":83,"context_line":"   - Power off nodes"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"2. Emergency Recovery"},{"line_number":86,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":87,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":88,"context_line":"   - Validate node health"},{"line_number":89,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"4d12e8a4_870aec3c","line":86,"updated":"2025-07-10 15:06:45.000000000","message":"It would be really neat to go by racks, or rows of racks since I understand from our facility experts this is what they do to avoid power surges.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":83,"context_line":"   - Power off nodes"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"2. Emergency Recovery"},{"line_number":86,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":87,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":88,"context_line":"   - Validate node health"},{"line_number":89,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"c97e83bb_4bf2a5bb","line":86,"in_reply_to":"26f59982_e3c1ed30","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"46d943cc5298604e93ed3f3c1d03bcdd36e2e259","unresolved":true,"context_lines":[{"line_number":83,"context_line":"   - Power off nodes"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"2. Emergency Recovery"},{"line_number":86,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":87,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":88,"context_line":"   - Validate node health"},{"line_number":89,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"56df3c41_f2315bc2","line":86,"in_reply_to":"4d12e8a4_870aec3c","updated":"2025-07-10 16:07:07.000000000","message":"I\u0027ve heard the exact same feedback another operator. That they presently have some tools which queries their CMDB/hardware inventory to produce a list of physical asset nodes, and then act upon Ironic. If they could just tell ironic, that obviously would be a lot cleaner for them.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"b5ebd63473b13f5b33081c215d04195476c42b5d","unresolved":true,"context_lines":[{"line_number":83,"context_line":"   - Power off nodes"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"2. Emergency Recovery"},{"line_number":86,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":87,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":88,"context_line":"   - Validate node health"},{"line_number":89,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"582b1da0_e88a4c47","line":86,"in_reply_to":"56df3c41_f2315bc2","updated":"2025-07-11 08:17:25.000000000","message":"We do the first part (ask our DCIM), and then use the BMCs, bypassing Ironic at the moment. This brings in difficulties about the state of the nodes once you need to bring them back: think services which have a cold stand-by, services which have an old control plane shutoff as they are in the middle of an upgrade, nodes which are put aside due to problems ... you need to make sure you only bring back up what should be brought back, so we\u0027re working on refining this. Ironic/Nova have the state of course, and other monitoring we have as well, so it is mostly a matter of integrating that information into the tooling.\n\nOne aspect for us is also the dependency on Ironic: going separately with a list of nodes and BMC creds (plus their pre-powercut power status) is separate and less integrated/clean, but has no dependencies on Ironic, its APIs and DBs, which you would need to bring up first, or have them in a special power area, if you rely on them. Depends on the local setup, I guess.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"7b79d02a3ead9f691bb78bc4c90a97c3adb5db33","unresolved":true,"context_lines":[{"line_number":83,"context_line":"   - Power off nodes"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"2. Emergency Recovery"},{"line_number":86,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":87,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":88,"context_line":"   - Validate node health"},{"line_number":89,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"26f59982_e3c1ed30","line":86,"in_reply_to":"582b1da0_e88a4c47","updated":"2025-09-23 15:39:03.000000000","message":"I guess it also depends on actual use/access model. Specifically if the Network or DC Ops folks have the BMC credentials and access to gain it.\n\nSome folks I know don\u0027t provide those credentials, and also run additional UPSes for critical control resources. Everyone is different though.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":92,"context_line":"   - Targeted operations using any available node field (including the new instance_name field https://review.opendev.org/c/openstack/ironic/+/952790)"},{"line_number":93,"context_line":"   - This would be available as a selector to the shutdown and recover operations"},{"line_number":94,"context_line":"   - Emergency operations will use the same node filtering capabilities as the `ironic node-list` command, supporting all existing filters such as `--resource-class`, `--driver`, `--provision-state`, `--maintenance`, `--associated`, and `--fields` (including `instance_name`)"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"4. Operation Control"},{"line_number":97,"context_line":"   - Operation timeout settings"},{"line_number":98,"context_line":"   - Graceful shutdown timeout"}],"source_content_type":"text/x-rst","patch_set":6,"id":"2aa1ecef_4584a33f","line":95,"updated":"2025-07-10 14:22:41.000000000","message":"I still think we *really* need a location field. 😊\n\nI\u0027m really not a huge fan of substring matching in databases, but honestly it is kind of what is needed. Under the hood we\u0027ll functionally need to trigger an like or ilike query against the database, and we\u0027ll also *really* want to make sure the fields which could be queried are all indexed to enable a rapid match.\n\nWe also likely need to discourage queries against the longer fields where this is the case, because of database design/behavior as it relates to query performance.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[{"line_number":92,"context_line":"   - Targeted operations using any available node field (including the new instance_name field https://review.opendev.org/c/openstack/ironic/+/952790)"},{"line_number":93,"context_line":"   - This would be available as a selector to the shutdown and recover operations"},{"line_number":94,"context_line":"   - Emergency operations will use the same node filtering capabilities as the `ironic node-list` command, supporting all existing filters such as `--resource-class`, `--driver`, `--provision-state`, `--maintenance`, `--associated`, and `--fields` (including `instance_name`)"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"4. Operation Control"},{"line_number":97,"context_line":"   - Operation timeout settings"},{"line_number":98,"context_line":"   - Graceful shutdown timeout"}],"source_content_type":"text/x-rst","patch_set":6,"id":"b36987c3_eb08114f","line":95,"in_reply_to":"2aa1ecef_4584a33f","updated":"2025-09-25 21:20:11.000000000","message":"++ I am OK with a location field.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":92,"context_line":"   - Targeted operations using any available node field (including the new instance_name field https://review.opendev.org/c/openstack/ironic/+/952790)"},{"line_number":93,"context_line":"   - This would be available as a selector to the shutdown and recover operations"},{"line_number":94,"context_line":"   - Emergency operations will use the same node filtering capabilities as the `ironic node-list` command, supporting all existing filters such as `--resource-class`, `--driver`, `--provision-state`, `--maintenance`, `--associated`, and `--fields` (including `instance_name`)"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"4. Operation Control"},{"line_number":97,"context_line":"   - Operation timeout settings"},{"line_number":98,"context_line":"   - Graceful shutdown timeout"}],"source_content_type":"text/x-rst","patch_set":6,"id":"b14dd6dd_8954f5ca","line":95,"in_reply_to":"b36987c3_eb08114f","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":100,"context_line":"   - Operation status tracking"},{"line_number":101,"context_line":"   - Detailed operation logs"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"In the case of power failures for example, it is likely that the operator would only want to power on a subset key of nodes. This spec proposes that `instance_name` can be used to identify high-priority nodes."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"Note that for recovery, we assume that database nodes and conductor nodes are available,"},{"line_number":106,"context_line":"or can be made available manually."}],"source_content_type":"text/x-rst","patch_set":6,"id":"ccc0cc7d_70abc3fc","line":103,"updated":"2025-07-10 14:22:41.000000000","message":"I guess my only concern with instance name is it implies prior intentional context as to the name mapping which is not always True. Partly why I\u0027m thinking location would be useful because so any of these issues are raw location related.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":100,"context_line":"   - Operation status tracking"},{"line_number":101,"context_line":"   - Detailed operation logs"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"In the case of power failures for example, it is likely that the operator would only want to power on a subset key of nodes. This spec proposes that `instance_name` can be used to identify high-priority nodes."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"Note that for recovery, we assume that database nodes and conductor nodes are available,"},{"line_number":106,"context_line":"or can be made available manually."}],"source_content_type":"text/x-rst","patch_set":6,"id":"420bcf4a_6440b606","line":103,"in_reply_to":"ccc0cc7d_70abc3fc","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":178,"context_line":"   - Conductors execute power-on operations:"},{"line_number":179,"context_line":"     - Power on nodes by invoking the `power_on` method of the node\u0027s"},{"line_number":180,"context_line":"       configured power driver (e.g., IPMI, Redfish)."},{"line_number":181,"context_line":"     - If `--validate-health` specified, perform health checks after power-on"},{"line_number":182,"context_line":"     - Report operation status back to API"},{"line_number":183,"context_line":"   - Operation state updated to \"executing\""},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"5. **Operation Monitoring**"}],"source_content_type":"text/x-rst","patch_set":6,"id":"162fd02f_12e4aa48","line":182,"range":{"start_line":181,"start_character":0,"end_line":182,"end_character":42},"updated":"2025-07-10 14:22:41.000000000","message":"I\u0027ll just note this seems like a whole other additional feature.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":178,"context_line":"   - Conductors execute power-on operations:"},{"line_number":179,"context_line":"     - Power on nodes by invoking the `power_on` method of the node\u0027s"},{"line_number":180,"context_line":"       configured power driver (e.g., IPMI, Redfish)."},{"line_number":181,"context_line":"     - If `--validate-health` specified, perform health checks after power-on"},{"line_number":182,"context_line":"     - Report operation status back to API"},{"line_number":183,"context_line":"   - Operation state updated to \"executing\""},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"5. **Operation Monitoring**"}],"source_content_type":"text/x-rst","patch_set":6,"id":"e78932e7_e0632ce9","line":182,"range":{"start_line":181,"start_character":0,"end_line":182,"end_character":42},"in_reply_to":"162fd02f_12e4aa48","updated":"2025-08-05 03:28:03.000000000","message":"Removed - I agree","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":191,"context_line":"--------------------"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"- **State Consistency**: All node state changes are recorded in the database before conductor operations begin"},{"line_number":194,"context_line":"- **Conductor Independence**: Conductors operate independently but report back to centralized API (i.e. not changing the current conductor-to-node relationship)"},{"line_number":195,"context_line":"- **Failure Handling**: Failed operations are logged but don\u0027t prevent other operations from proceeding"},{"line_number":196,"context_line":"- **Audit Trail**: All state changes and operations are logged for compliance and troubleshooting"},{"line_number":197,"context_line":"- **Graceful Degradation**: System continues operating even if some conductors are unavailable"}],"source_content_type":"text/x-rst","patch_set":6,"id":"e009a128_900ad3b9","line":194,"range":{"start_line":194,"start_character":0,"end_line":194,"end_character":160},"updated":"2025-07-10 14:22:41.000000000","message":"Well, it doesn\u0027t report back but is visible.\n\nI guess one thing I\u0027m wondering outloud is if we out to have conductors identify the task and then pickup the action and commit state back to and database endpoint which could be visible. As an aggregate operation, its going to be split up regardless.\n\nFood for thought at least.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":191,"context_line":"--------------------"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"- **State Consistency**: All node state changes are recorded in the database before conductor operations begin"},{"line_number":194,"context_line":"- **Conductor Independence**: Conductors operate independently but report back to centralized API (i.e. not changing the current conductor-to-node relationship)"},{"line_number":195,"context_line":"- **Failure Handling**: Failed operations are logged but don\u0027t prevent other operations from proceeding"},{"line_number":196,"context_line":"- **Audit Trail**: All state changes and operations are logged for compliance and troubleshooting"},{"line_number":197,"context_line":"- **Graceful Degradation**: System continues operating even if some conductors are unavailable"}],"source_content_type":"text/x-rst","patch_set":6,"id":"678678a3_8a84912c","line":194,"range":{"start_line":194,"start_character":0,"end_line":194,"end_character":160},"in_reply_to":"e009a128_900ad3b9","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"ace324d4abebb19058d0e1bf451a1a8d63cf7644","unresolved":true,"context_lines":[{"line_number":218,"context_line":""},{"line_number":219,"context_line":"1. When the API initiates recovery, it updates the node\u0027s `power_state` from"},{"line_number":220,"context_line":"   `EMERGENCY_SHUTDOWN_OFF` to `EMERGENCY_RECOVERY_OFF`. At this stage, the"},{"line_number":221,"context_line":"   node\u0027s `maintenance` flag is also cleared in the database."},{"line_number":222,"context_line":"2. The conductor receives the request and invokes the `power_on` method from"},{"line_number":223,"context_line":"   the node\u0027s power driver. The state may briefly transition to"},{"line_number":224,"context_line":"   `EMERGENCY_RECOVERY_ON` to indicate the power-on is in progress."}],"source_content_type":"text/x-rst","patch_set":6,"id":"72b206fc_52878f24","line":221,"range":{"start_line":221,"start_character":4,"end_line":221,"end_character":61},"updated":"2025-07-10 15:06:45.000000000","message":"Does this take into account the power synchronisation between Nova and Ironic? IIRC, the default behaviour is that when a node goes down, the Nova DB is updated, but when a node comes up, the DB state in Nova is enforced on the node in Ironic. There is an option to control and change this, so that Ironic becomes the source of truth, but the moment the maintenance flag is cleared this target power sync mechanism will come into action, so the proposal needs to make sure it interferes with the different settings correctly.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"46d943cc5298604e93ed3f3c1d03bcdd36e2e259","unresolved":true,"context_lines":[{"line_number":218,"context_line":""},{"line_number":219,"context_line":"1. When the API initiates recovery, it updates the node\u0027s `power_state` from"},{"line_number":220,"context_line":"   `EMERGENCY_SHUTDOWN_OFF` to `EMERGENCY_RECOVERY_OFF`. At this stage, the"},{"line_number":221,"context_line":"   node\u0027s `maintenance` flag is also cleared in the database."},{"line_number":222,"context_line":"2. The conductor receives the request and invokes the `power_on` method from"},{"line_number":223,"context_line":"   the node\u0027s power driver. The state may briefly transition to"},{"line_number":224,"context_line":"   `EMERGENCY_RECOVERY_ON` to indicate the power-on is in progress."}],"source_content_type":"text/x-rst","patch_set":6,"id":"d9558a68_deb4a217","line":221,"range":{"start_line":221,"start_character":4,"end_line":221,"end_character":61},"in_reply_to":"72b206fc_52878f24","updated":"2025-07-10 16:07:07.000000000","message":"This is a great callout. I suspect its going to require some investigation of the nova code, and I think it might just ignore the node, but we\u0027ll need to double check for sure. Power states which are unknown also trigger different logic, so we may need to patch Nova to keep it happy.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":218,"context_line":""},{"line_number":219,"context_line":"1. When the API initiates recovery, it updates the node\u0027s `power_state` from"},{"line_number":220,"context_line":"   `EMERGENCY_SHUTDOWN_OFF` to `EMERGENCY_RECOVERY_OFF`. At this stage, the"},{"line_number":221,"context_line":"   node\u0027s `maintenance` flag is also cleared in the database."},{"line_number":222,"context_line":"2. The conductor receives the request and invokes the `power_on` method from"},{"line_number":223,"context_line":"   the node\u0027s power driver. The state may briefly transition to"},{"line_number":224,"context_line":"   `EMERGENCY_RECOVERY_ON` to indicate the power-on is in progress."}],"source_content_type":"text/x-rst","patch_set":6,"id":"7520b3f9_efda3f28","line":221,"range":{"start_line":221,"start_character":4,"end_line":221,"end_character":61},"in_reply_to":"d9558a68_deb4a217","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":261,"context_line":"- `concurrent_limit` (Integer) - Maximum concurrent operations"},{"line_number":262,"context_line":"- `dry_run` (Boolean) - Whether this was a dry run"},{"line_number":263,"context_line":"- `filters` (JSON) - Applied node filters for the operation"},{"line_number":264,"context_line":"- `error_details` (JSON) - Detailed error information for failed operations"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"Node Table Updates"},{"line_number":267,"context_line":"~~~~~~~~~~~~~~~~~"}],"source_content_type":"text/x-rst","patch_set":6,"id":"d5f32d5e_ddfa2e8d","line":264,"updated":"2025-07-10 14:22:41.000000000","message":"I\u0027d consider dividing by conductor, so an API request would turn into, for example 8 entries, one per conductor, so we don\u0027t have mutliple conductors trying to lock/operate on a single row.\n\nHappy to discuss if you were thinking a different model. The other aspect to consider is what if a conductor itself looses power/connectivity in this entire sequence, and who owns/orchestrates at the highest level.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":261,"context_line":"- `concurrent_limit` (Integer) - Maximum concurrent operations"},{"line_number":262,"context_line":"- `dry_run` (Boolean) - Whether this was a dry run"},{"line_number":263,"context_line":"- `filters` (JSON) - Applied node filters for the operation"},{"line_number":264,"context_line":"- `error_details` (JSON) - Detailed error information for failed operations"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"Node Table Updates"},{"line_number":267,"context_line":"~~~~~~~~~~~~~~~~~"}],"source_content_type":"text/x-rst","patch_set":6,"id":"058f8587_8004d7b6","line":264,"in_reply_to":"d5f32d5e_ddfa2e8d","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"0303ea4d14147f6bb127fb8eb9cd0046d85c4d0e","unresolved":true,"context_lines":[{"line_number":267,"context_line":"~~~~~~~~~~~~~~~~~"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"Add to existing `nodes` table:"},{"line_number":270,"context_line":"- `emergency_operation_id` (UUID, Foreign Key) - Reference to current emergency operation affecting this node"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"It is critical that an index is created on the `emergency_operation_id` column"},{"line_number":273,"context_line":"to ensure efficient querying of nodes associated with an operation. This field"}],"source_content_type":"text/x-rst","patch_set":6,"id":"b405d449_029ca70e","line":270,"updated":"2025-07-10 14:22:41.000000000","message":"So the purpose here, I guess is to be able to explicitly undo a single operation\u0027s request... I guess that makes sense.","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"ea5df8febddd7d5f4beafbceafe04b5e6819e80b","unresolved":false,"context_lines":[{"line_number":267,"context_line":"~~~~~~~~~~~~~~~~~"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"Add to existing `nodes` table:"},{"line_number":270,"context_line":"- `emergency_operation_id` (UUID, Foreign Key) - Reference to current emergency operation affecting this node"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"It is critical that an index is created on the `emergency_operation_id` column"},{"line_number":273,"context_line":"to ensure efficient querying of nodes associated with an operation. This field"}],"source_content_type":"text/x-rst","patch_set":6,"id":"10bef78e_f5813e4e","line":270,"in_reply_to":"b405d449_029ca70e","updated":"2025-08-05 03:28:03.000000000","message":"Done","commit_id":"74498c4a1224987111cd210d059cc8528159b82c"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"fcb9e3d4706e007a430aa9e7956ba31c59893a36","unresolved":false,"context_lines":[{"line_number":63,"context_line":"   --fields location --location \"rack:12-18\"` to immediately power off all nodes"},{"line_number":64,"context_line":"   in the affected physical location before thermal damage occurs. Location-based"},{"line_number":65,"context_line":"   targeting is ideal for shutdown scenarios as facility operators typically manage"},{"line_number":66,"context_line":"   power by physical zones (racks, rows, PDUs) to avoid power surges."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"2. **Application-Aware Phased Recovery**: After cooling is restored and temperatures"},{"line_number":69,"context_line":"   return to safe levels, prioritize critical infrastructure using targeted recovery"}],"source_content_type":"text/x-rst","patch_set":8,"id":"ec4f3247_a25a9133","line":66,"range":{"start_line":66,"start_character":47,"end_line":66,"end_character":69},"updated":"2025-09-23 16:16:58.000000000","message":"It is more about lags and voltage drop management, FWIW. 😊","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"fcb9e3d4706e007a430aa9e7956ba31c59893a36","unresolved":true,"context_lines":[{"line_number":72,"context_line":"   - `--fields instance_name --instance_name \"k8s-control-plane-*\"` (Kubernetes control plane)"},{"line_number":73,"context_line":"   - `--fields instance_name --instance_name \"postgres-*\"` (database nodes)"},{"line_number":74,"context_line":"   - `--fields instance_name --instance_name \"ceph-*\"` (storage cluster nodes)"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"This dual-selector approach prevents expensive hardware damage through location-based"},{"line_number":77,"context_line":"shutdown while ensuring critical application services are restored first in priority"},{"line_number":78,"context_line":"order during recovery, transforming a potential equipment loss scenario into a"}],"source_content_type":"text/x-rst","patch_set":8,"id":"19e6ec38_06db3ada","line":75,"updated":"2025-09-23 16:16:58.000000000","message":"Question, is this proposing client based filter/match/recovery signaling?","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[{"line_number":72,"context_line":"   - `--fields instance_name --instance_name \"k8s-control-plane-*\"` (Kubernetes control plane)"},{"line_number":73,"context_line":"   - `--fields instance_name --instance_name \"postgres-*\"` (database nodes)"},{"line_number":74,"context_line":"   - `--fields instance_name --instance_name \"ceph-*\"` (storage cluster nodes)"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"This dual-selector approach prevents expensive hardware damage through location-based"},{"line_number":77,"context_line":"shutdown while ensuring critical application services are restored first in priority"},{"line_number":78,"context_line":"order during recovery, transforming a potential equipment loss scenario into a"}],"source_content_type":"text/x-rst","patch_set":8,"id":"78cd9cdf_17b970c4","line":75,"in_reply_to":"19e6ec38_06db3ada","updated":"2025-09-25 21:20:11.000000000","message":"Would that be a requirement to do a partial match in the API as presented here?","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":72,"context_line":"   - `--fields instance_name --instance_name \"k8s-control-plane-*\"` (Kubernetes control plane)"},{"line_number":73,"context_line":"   - `--fields instance_name --instance_name \"postgres-*\"` (database nodes)"},{"line_number":74,"context_line":"   - `--fields instance_name --instance_name \"ceph-*\"` (storage cluster nodes)"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"This dual-selector approach prevents expensive hardware damage through location-based"},{"line_number":77,"context_line":"shutdown while ensuring critical application services are restored first in priority"},{"line_number":78,"context_line":"order during recovery, transforming a potential equipment loss scenario into a"}],"source_content_type":"text/x-rst","patch_set":8,"id":"13979d61_41aedcba","line":75,"in_reply_to":"78cd9cdf_17b970c4","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36770,"name":"cid","display_name":"cid","email":"cid@gr-oss.io","username":"cidelight","status":"@gr-oss upstream: Doing good IRONIC things..."},"change_message_id":"e6de4bad1084b458046d1cd5f81e36bc908bf906","unresolved":true,"context_lines":[{"line_number":86,"context_line":"Add a new emergency operations API endpoint that supports bulk actions"},{"line_number":87,"context_line":"with flexible node targeting. The system is designed as a **best-effort operation"},{"line_number":88,"context_line":"optimized for speed over precision** - prioritizing rapid response during emergencies"},{"line_number":89,"context_line":"over guaranteed operation completion or detailed failure tracking.wn"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"The endpoint will support:"},{"line_number":92,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"b004615b_f341c6a5","line":89,"range":{"start_line":89,"start_character":65,"end_line":89,"end_character":68},"updated":"2025-09-24 20:25:10.000000000","message":"Possible unintended characters.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":86,"context_line":"Add a new emergency operations API endpoint that supports bulk actions"},{"line_number":87,"context_line":"with flexible node targeting. The system is designed as a **best-effort operation"},{"line_number":88,"context_line":"optimized for speed over precision** - prioritizing rapid response during emergencies"},{"line_number":89,"context_line":"over guaranteed operation completion or detailed failure tracking.wn"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"The endpoint will support:"},{"line_number":92,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"87ed1bf6_f9068afe","line":89,"range":{"start_line":89,"start_character":65,"end_line":89,"end_character":68},"in_reply_to":"b004615b_f341c6a5","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"fcb9e3d4706e007a430aa9e7956ba31c59893a36","unresolved":false,"context_lines":[{"line_number":92,"context_line":""},{"line_number":93,"context_line":"1. Emergency Shutdown"},{"line_number":94,"context_line":"   - Set maintenance mode with emergency reason"},{"line_number":95,"context_line":"   - Update node state to reflect emergency status"},{"line_number":96,"context_line":"   - Power off nodes"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"2. Emergency Recovery"}],"source_content_type":"text/x-rst","patch_set":8,"id":"8a7145b0_e7d99cc4","line":95,"updated":"2025-09-23 16:16:58.000000000","message":"I *believe this is power state, please correct me if I\u0027m wrong.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":false,"context_lines":[{"line_number":92,"context_line":""},{"line_number":93,"context_line":"1. Emergency Shutdown"},{"line_number":94,"context_line":"   - Set maintenance mode with emergency reason"},{"line_number":95,"context_line":"   - Update node state to reflect emergency status"},{"line_number":96,"context_line":"   - Power off nodes"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"2. Emergency Recovery"}],"source_content_type":"text/x-rst","patch_set":8,"id":"cf1c785c_9a257b83","line":95,"in_reply_to":"8a7145b0_e7d99cc4","updated":"2025-09-25 21:20:11.000000000","message":"Would this be new power state(s) for \"emergency-off\" vs \"off\"?","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36770,"name":"cid","display_name":"cid","email":"cid@gr-oss.io","username":"cidelight","status":"@gr-oss upstream: Doing good IRONIC things..."},"change_message_id":"e6de4bad1084b458046d1cd5f81e36bc908bf906","unresolved":true,"context_lines":[{"line_number":97,"context_line":""},{"line_number":98,"context_line":"2. Emergency Recovery"},{"line_number":99,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":100,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":101,"context_line":"   - Validate node health"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"3. Node Targeting"}],"source_content_type":"text/x-rst","patch_set":8,"id":"882a60c1_61d16d07","line":100,"range":{"start_line":100,"start_character":0,"end_line":100,"end_character":60},"updated":"2025-09-24 20:25:10.000000000","message":"What would we consider \u0027normal operation state\u0027? Only asking if this node is powered on is probably all we need. Unless we envision this feature to preserve the state of nodes that were in transitional or in-progress states? For example, a node deploying an image, cleaning, etc.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[{"line_number":97,"context_line":""},{"line_number":98,"context_line":"2. Emergency Recovery"},{"line_number":99,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":100,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":101,"context_line":"   - Validate node health"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"3. Node Targeting"}],"source_content_type":"text/x-rst","patch_set":8,"id":"fcba831c_f5b9937e","line":100,"range":{"start_line":100,"start_character":0,"end_line":100,"end_character":60},"in_reply_to":"882a60c1_61d16d07","updated":"2025-09-25 21:20:11.000000000","message":"I think there is (maybe unstated here and should be explicit?) an idea of this emergency situation pausing things like the power status loop on affected nodes. This is reflecting that we\u0027d ahve to unroll that.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":97,"context_line":""},{"line_number":98,"context_line":"2. Emergency Recovery"},{"line_number":99,"context_line":"   - Default should be to only power on a subset of nodes, but provide an option to power on all nodes"},{"line_number":100,"context_line":"   - Restore normal operation state / clear maintenance mode"},{"line_number":101,"context_line":"   - Validate node health"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"3. Node Targeting"}],"source_content_type":"text/x-rst","patch_set":8,"id":"2aac455b_06365ed1","line":100,"range":{"start_line":100,"start_character":0,"end_line":100,"end_character":60},"in_reply_to":"fcba831c_f5b9937e","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36770,"name":"cid","display_name":"cid","email":"cid@gr-oss.io","username":"cidelight","status":"@gr-oss upstream: Doing good IRONIC things..."},"change_message_id":"e6de4bad1084b458046d1cd5f81e36bc908bf906","unresolved":true,"context_lines":[{"line_number":151,"context_line":"The emergency operations system follows Ironic\u0027s standard API-to-conductor architecture that ensures"},{"line_number":152,"context_line":"reliable state management and operation tracking across distributed Ironic deployments."},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"Emergency Shutdown Flow"},{"line_number":155,"context_line":"-----------------------"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"1. **API Request Processing**"}],"source_content_type":"text/x-rst","patch_set":8,"id":"f10c14c2_7457d6ef","line":154,"range":{"start_line":154,"start_character":0,"end_line":154,"end_character":23},"updated":"2025-09-24 20:25:10.000000000","message":"Since an emergency action will be high-stakes, as a safeguard, somehow, somewhere, we may want to include a confirmation step to avoid accidental exec.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":151,"context_line":"The emergency operations system follows Ironic\u0027s standard API-to-conductor architecture that ensures"},{"line_number":152,"context_line":"reliable state management and operation tracking across distributed Ironic deployments."},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"Emergency Shutdown Flow"},{"line_number":155,"context_line":"-----------------------"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"1. **API Request Processing**"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1fc3d55b_b5c065f4","line":154,"range":{"start_line":154,"start_character":0,"end_line":154,"end_character":23},"in_reply_to":"283e53c8_e638b70e","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[{"line_number":151,"context_line":"The emergency operations system follows Ironic\u0027s standard API-to-conductor architecture that ensures"},{"line_number":152,"context_line":"reliable state management and operation tracking across distributed Ironic deployments."},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"Emergency Shutdown Flow"},{"line_number":155,"context_line":"-----------------------"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"1. **API Request Processing**"}],"source_content_type":"text/x-rst","patch_set":8,"id":"283e53c8_e638b70e","line":154,"range":{"start_line":154,"start_character":0,"end_line":154,"end_character":23},"in_reply_to":"f10c14c2_7457d6ef","updated":"2025-09-25 21:20:11.000000000","message":"I think this would be best done via RBAC: you should need significant, abnormally-high levels of access to do this.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[{"line_number":173,"context_line":"     - `power_state` updated to `EMERGENCY_SHUTDOWN_ON` (if powered on) or `EMERGENCY_SHUTDOWN_OFF` (if powered off)"},{"line_number":174,"context_line":"     - `emergency_operation_id` field set to the operation ID"},{"line_number":175,"context_line":"   - Each node state change is logged in audit trail"},{"line_number":176,"context_line":"   - Operation state updated to \"state_updated\""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"4. **Conductor Registration and Progress Tracking**"},{"line_number":179,"context_line":"   - Each conductor participating in the operation creates a row in emergency_operation_conductors table"}],"source_content_type":"text/x-rst","patch_set":8,"id":"44eed3d6_8dd49a6d","line":176,"updated":"2025-09-25 21:20:11.000000000","message":"how does this impact periodic jobs like the power status loop? similarly for emergency_operation_conductors","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":173,"context_line":"     - `power_state` updated to `EMERGENCY_SHUTDOWN_ON` (if powered on) or `EMERGENCY_SHUTDOWN_OFF` (if powered off)"},{"line_number":174,"context_line":"     - `emergency_operation_id` field set to the operation ID"},{"line_number":175,"context_line":"   - Each node state change is logged in audit trail"},{"line_number":176,"context_line":"   - Operation state updated to \"state_updated\""},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"4. **Conductor Registration and Progress Tracking**"},{"line_number":179,"context_line":"   - Each conductor participating in the operation creates a row in emergency_operation_conductors table"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9627878c_80ff8085","line":176,"in_reply_to":"44eed3d6_8dd49a6d","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36770,"name":"cid","display_name":"cid","email":"cid@gr-oss.io","username":"cidelight","status":"@gr-oss upstream: Doing good IRONIC things..."},"change_message_id":"e6de4bad1084b458046d1cd5f81e36bc908bf906","unresolved":true,"context_lines":[{"line_number":183,"context_line":"   - Individual power operations (power_off) are executed per node"},{"line_number":184,"context_line":"   - Operation state updated to \"executing\""},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"5. **Operation Monitoring**"},{"line_number":187,"context_line":"   - API tracks progress across all individual node operations"},{"line_number":188,"context_line":"   - Failed operations are logged with detailed error information"},{"line_number":189,"context_line":"   - Operation state updated to \"completed\" or \"failed\""}],"source_content_type":"text/x-rst","patch_set":8,"id":"d7761a66_469d123c","line":186,"range":{"start_line":186,"start_character":0,"end_line":186,"end_character":27},"updated":"2025-09-24 20:25:10.000000000","message":"Here, or somewhere else, we might want to handle cases of debounce actions, such as double-click or overlapping incidents","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":183,"context_line":"   - Individual power operations (power_off) are executed per node"},{"line_number":184,"context_line":"   - Operation state updated to \"executing\""},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"5. **Operation Monitoring**"},{"line_number":187,"context_line":"   - API tracks progress across all individual node operations"},{"line_number":188,"context_line":"   - Failed operations are logged with detailed error information"},{"line_number":189,"context_line":"   - Operation state updated to \"completed\" or \"failed\""}],"source_content_type":"text/x-rst","patch_set":8,"id":"ad052e33_1e67bf0f","line":186,"range":{"start_line":186,"start_character":0,"end_line":186,"end_character":27},"in_reply_to":"d7761a66_469d123c","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36770,"name":"cid","display_name":"cid","email":"cid@gr-oss.io","username":"cidelight","status":"@gr-oss upstream: Doing good IRONIC things..."},"change_message_id":"e6de4bad1084b458046d1cd5f81e36bc908bf906","unresolved":true,"context_lines":[{"line_number":188,"context_line":"   - Failed operations are logged with detailed error information"},{"line_number":189,"context_line":"   - Operation state updated to \"completed\" or \"failed\""},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"Emergency Recovery Flow"},{"line_number":192,"context_line":"-----------------------"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"1. **API Request Processing**"}],"source_content_type":"text/x-rst","patch_set":8,"id":"19a181c4_a3568d62","line":191,"range":{"start_line":191,"start_character":0,"end_line":191,"end_character":23},"updated":"2025-09-24 20:25:10.000000000","message":"This might already be implied, but since we\u0027ll be doing best-effort shutdown, we may need to add a verification step to check each node\u0027s actual state before proceeding with recovery flow.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":188,"context_line":"   - Failed operations are logged with detailed error information"},{"line_number":189,"context_line":"   - Operation state updated to \"completed\" or \"failed\""},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"Emergency Recovery Flow"},{"line_number":192,"context_line":"-----------------------"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"1. **API Request Processing**"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3c2a185e_a96e37e9","line":191,"range":{"start_line":191,"start_character":0,"end_line":191,"end_character":23},"in_reply_to":"19a181c4_a3568d62","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36770,"name":"cid","display_name":"cid","email":"cid@gr-oss.io","username":"cidelight","status":"@gr-oss upstream: Doing good IRONIC things..."},"change_message_id":"e6de4bad1084b458046d1cd5f81e36bc908bf906","unresolved":true,"context_lines":[{"line_number":272,"context_line":"Open Questions"},{"line_number":273,"context_line":"--------------"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"- Should there be different types of shutdown operations based on emergency type"},{"line_number":276,"context_line":"  (i.e. immediate shutdown for thermal protection vs. graceful shutdown for"},{"line_number":277,"context_line":"  planned maintenance scenarios)?"},{"line_number":278,"context_line":""},{"line_number":279,"context_line":"Data model impact"},{"line_number":280,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"5faa1d1e_4a74b09c","line":277,"range":{"start_line":275,"start_character":0,"end_line":277,"end_character":33},"updated":"2025-09-24 20:25:10.000000000","message":"++\n\nI think this will make for a good follow-up feature that will be fairly straightforward.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":272,"context_line":"Open Questions"},{"line_number":273,"context_line":"--------------"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"- Should there be different types of shutdown operations based on emergency type"},{"line_number":276,"context_line":"  (i.e. immediate shutdown for thermal protection vs. graceful shutdown for"},{"line_number":277,"context_line":"  planned maintenance scenarios)?"},{"line_number":278,"context_line":""},{"line_number":279,"context_line":"Data model impact"},{"line_number":280,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7fe35865_e1bc2221","line":277,"range":{"start_line":275,"start_character":0,"end_line":277,"end_character":33},"in_reply_to":"5faa1d1e_4a74b09c","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"fcb9e3d4706e007a430aa9e7956ba31c59893a36","unresolved":true,"context_lines":[{"line_number":337,"context_line":"~~~~~~~~~~~~~~~~~"},{"line_number":338,"context_line":""},{"line_number":339,"context_line":"Add to existing `nodes` table:"},{"line_number":340,"context_line":"- `emergency_operation_id` (UUID, Foreign Key) - Reference to current emergency operation affecting this node"},{"line_number":341,"context_line":"- `location` (String) - Free-form location identifier set by operators for location-based targeting"},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"Progress tracking is achieved through the emergency_operation_conductors table:"}],"source_content_type":"text/x-rst","patch_set":8,"id":"db1d999b_ef0349f0","line":340,"updated":"2025-09-23 16:16:58.000000000","message":"I guess this is slightly concerning for me because its another thing to clean up and another column to update. Dunno.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":337,"context_line":"~~~~~~~~~~~~~~~~~"},{"line_number":338,"context_line":""},{"line_number":339,"context_line":"Add to existing `nodes` table:"},{"line_number":340,"context_line":"- `emergency_operation_id` (UUID, Foreign Key) - Reference to current emergency operation affecting this node"},{"line_number":341,"context_line":"- `location` (String) - Free-form location identifier set by operators for location-based targeting"},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"Progress tracking is achieved through the emergency_operation_conductors table:"}],"source_content_type":"text/x-rst","patch_set":8,"id":"dd5fcd7a_6452af0a","line":340,"in_reply_to":"58c870ef_9f32834c","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[{"line_number":337,"context_line":"~~~~~~~~~~~~~~~~~"},{"line_number":338,"context_line":""},{"line_number":339,"context_line":"Add to existing `nodes` table:"},{"line_number":340,"context_line":"- `emergency_operation_id` (UUID, Foreign Key) - Reference to current emergency operation affecting this node"},{"line_number":341,"context_line":"- `location` (String) - Free-form location identifier set by operators for location-based targeting"},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"Progress tracking is achieved through the emergency_operation_conductors table:"}],"source_content_type":"text/x-rst","patch_set":8,"id":"58c870ef_9f32834c","line":340,"in_reply_to":"db1d999b_ef0349f0","updated":"2025-09-25 21:20:11.000000000","message":"Yeah I agree, can we do this without adding an additional field to node?","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"43f7f16a8108045b724b904bf0786114d6fba493","unresolved":true,"context_lines":[{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* EMERGENCY_SHUTDOWN_OFF - Node is in emergency shutdown state (powered off)"},{"line_number":367,"context_line":"* EMERGENCY_SHUTDOWN_ON - Node is in emergency shutdown state (powered on)"},{"line_number":368,"context_line":"* EMERGENCY_RECOVERY_OFF - Node is in emergency recovery state (powered off)"},{"line_number":369,"context_line":"* EMERGENCY_RECOVERY_ON - Node is in emergency recovery state (powered on)"},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"This approach leverages the existing power state tracking mechanism, providing better performance characteristics for bulk operations while maintaining state visibility through the existing node power state field."}],"source_content_type":"text/x-rst","patch_set":8,"id":"5af9579d_63070991","line":368,"updated":"2025-09-25 21:20:11.000000000","message":"I\u0027m confused about the *RECOVERY* states. Also can we be explicit about when power_state vs target_power_state is set?\n\ne.g. \n- power_state\u003dPOWER_ON\n- emergency api called\n- target_power_state\u003dEMERGENCY_SHUTDOWN_OFF or would we immediately update the power_state to EMERGENCY_SHUTDOWN_ON?\n\nI guess I\u0027m just confused as to how all these interact.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* EMERGENCY_SHUTDOWN_OFF - Node is in emergency shutdown state (powered off)"},{"line_number":367,"context_line":"* EMERGENCY_SHUTDOWN_ON - Node is in emergency shutdown state (powered on)"},{"line_number":368,"context_line":"* EMERGENCY_RECOVERY_OFF - Node is in emergency recovery state (powered off)"},{"line_number":369,"context_line":"* EMERGENCY_RECOVERY_ON - Node is in emergency recovery state (powered on)"},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"This approach leverages the existing power state tracking mechanism, providing better performance characteristics for bulk operations while maintaining state visibility through the existing node power state field."}],"source_content_type":"text/x-rst","patch_set":8,"id":"e96c46ff_3da6a615","line":368,"in_reply_to":"5af9579d_63070991","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"fcb9e3d4706e007a430aa9e7956ba31c59893a36","unresolved":true,"context_lines":[{"line_number":535,"context_line":"3. Infrastructure should remain running as long as possible during power events"},{"line_number":536,"context_line":""},{"line_number":537,"context_line":"Emergency bulk operations remain valuable for equipment protection scenarios where"},{"line_number":538,"context_line":"immediate infrastructure action is required."},{"line_number":539,"context_line":""},{"line_number":540,"context_line":"References"},{"line_number":541,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"b63fe0db_9b72491b","line":538,"updated":"2025-09-23 16:16:58.000000000","message":"Eh, or \"I\u0027m using 6.3 Gph, if I get my generator load down to 50% then I can run at 4.5 Gph, then I can last X number of additional hours.\"  I\u0027ve had to do this secondary system load shed before and its \"not fun\", but its more a calculated move to buy time for a diesel delivery to become available.","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"},{"author":{"_account_id":36102,"name":"Adam McArthur","display_name":"Adam McArthur","email":"adam@mcaq.me","username":"adamcarthur","status":"PhD Candidate in Computational Imaging. Open Source Software Engineer at G-Research."},"change_message_id":"223a0865c0224bf76a6a9ce8ed2a858273f1bfd1","unresolved":false,"context_lines":[{"line_number":535,"context_line":"3. Infrastructure should remain running as long as possible during power events"},{"line_number":536,"context_line":""},{"line_number":537,"context_line":"Emergency bulk operations remain valuable for equipment protection scenarios where"},{"line_number":538,"context_line":"immediate infrastructure action is required."},{"line_number":539,"context_line":""},{"line_number":540,"context_line":"References"},{"line_number":541,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"17db95f4_614892ef","line":538,"in_reply_to":"b63fe0db_9b72491b","updated":"2026-07-02 02:40:37.000000000","message":"Done","commit_id":"1aec96a5cbeab084ca253e81574e6823ea907dfd"}]}