)]}'
{"doc/source/admin/troubleshooting.rst":[{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":599,"context_line":"The baremetal node failed to power on"},{"line_number":600,"context_line":"-------------------------------------"},{"line_number":601,"context_line":""},{"line_number":602,"context_line":"Power failures after the deployment or cleaning operation started, or where"},{"line_number":603,"context_line":"the BMC is incorrectly reporting the physical node power state to Ironic."},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"The ramdisk did not [re]boot"},{"line_number":606,"context_line":"----------------------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"eff954b3_090ddd1a","line":603,"range":{"start_line":602,"start_character":0,"end_line":603,"end_character":73},"updated":"2021-07-06 18:34:42.000000000","message":"How can the operator check this is the case, what are potential causes, and how can the operator fix these?","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":605,"context_line":"The ramdisk did not [re]boot"},{"line_number":606,"context_line":"----------------------------"},{"line_number":607,"context_line":""},{"line_number":608,"context_line":"The deployment or cleaning ramdisk failed to boot or reboot. Typically"},{"line_number":609,"context_line":"issues such as this can present when physical network interface boot process"},{"line_number":610,"context_line":"is inconsistent *OR* the machine is *NOT* honoring persistent boot requests."},{"line_number":611,"context_line":""},{"line_number":612,"context_line":"Most often, failures such as this can be the result of a physical network"},{"line_number":613,"context_line":"misconfiguration where there are multiple network boot services and DHCP"}],"source_content_type":"text/x-rst","patch_set":6,"id":"967ab60b_acabedb6","line":610,"range":{"start_line":608,"start_character":61,"end_line":610,"end_character":15},"updated":"2021-07-06 18:34:42.000000000","message":"Is the wording correct? Also: I struggle to understand what this means and what I should do in this case ... and is repeated in the next paragraph.\n\nHow about:\n\n\"The deployment or cleaning ramdisk failed to boot or reboot. This can happen when:\n- the physical network is misconfigured\n- the BMC is not honoring boot requests\n- ...\"\n\nand explain each item in one of the next paragraphs.","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":615,"context_line":""},{"line_number":616,"context_line":"If you determine that the BMC is *NOT* honoring persistent boot requests,"},{"line_number":617,"context_line":"such as the style used by Ironic to inform the machine to boot to network,"},{"line_number":618,"context_line":"please report a bug with your hardware manufacter."},{"line_number":619,"context_line":""},{"line_number":620,"context_line":"In rare occassions, there may be a race between the physical network"},{"line_number":621,"context_line":"interfaces timing out on network boot operation, and switch fabric"}],"source_content_type":"text/x-rst","patch_set":6,"id":"57082a3c_8e8a83c2","line":618,"range":{"start_line":618,"start_character":39,"end_line":618,"end_character":49},"updated":"2021-07-06 18:34:42.000000000","message":"manufacturer","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":618,"context_line":"please report a bug with your hardware manufacter."},{"line_number":619,"context_line":""},{"line_number":620,"context_line":"In rare occassions, there may be a race between the physical network"},{"line_number":621,"context_line":"interfaces timing out on network boot operation, and switch fabric"},{"line_number":622,"context_line":"allowing for packets to be forwarded to the network. This is often"},{"line_number":623,"context_line":"referred to as \"portfast\" as a networking feature hich should be enabled."},{"line_number":624,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"914f4df5_06e768e1","line":621,"range":{"start_line":621,"start_character":38,"end_line":621,"end_character":47},"updated":"2021-07-06 18:34:42.000000000","message":"operations?","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":620,"context_line":"In rare occassions, there may be a race between the physical network"},{"line_number":621,"context_line":"interfaces timing out on network boot operation, and switch fabric"},{"line_number":622,"context_line":"allowing for packets to be forwarded to the network. This is often"},{"line_number":623,"context_line":"referred to as \"portfast\" as a networking feature hich should be enabled."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"The Ironic Python Agent is heartbeating, but nothing is happening"},{"line_number":626,"context_line":"-----------------------------------------------------------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"04e7219a_92673828","line":623,"range":{"start_line":623,"start_character":50,"end_line":623,"end_character":54},"updated":"2021-07-06 18:34:42.000000000","message":"which","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":627,"context_line":""},{"line_number":628,"context_line":"The node may be in maintenance mode."},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"When nodes are in maintenance mode, the conductor will suppress heartbeat"},{"line_number":631,"context_line":"operations from performing any action. Often maintenance mode may be due"},{"line_number":632,"context_line":"to communication failures with the BMC."},{"line_number":633,"context_line":""},{"line_number":634,"context_line":"The node lookup is failing"},{"line_number":635,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"a39b8904_78079eb0","line":632,"range":{"start_line":630,"start_character":0,"end_line":632,"end_character":39},"updated":"2021-07-06 18:34:42.000000000","message":"add how a node can be checked to be in maintenance mode","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":638,"context_line":"issue to the ``ironic-api`` service, but sometimes lookups can fail for"},{"line_number":639,"context_line":"reasons beyond connectivity."},{"line_number":640,"context_line":""},{"line_number":641,"context_line":"When not network connectivity, this may be because the BMC address cannot"},{"line_number":642,"context_line":"be discovered from within the agent ramdisk *or* corrilated back to the node"},{"line_number":643,"context_line":"due to the use of DNS for an internal address, and there are no network ports"},{"line_number":644,"context_line":"registered *OR* the port address(es) contain a typo, or the MAC address is"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1fdae4dd_5a8ce928","line":641,"range":{"start_line":641,"start_character":55,"end_line":641,"end_character":66},"updated":"2021-07-06 18:34:42.000000000","message":"is this really the BMC interface?","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":639,"context_line":"reasons beyond connectivity."},{"line_number":640,"context_line":""},{"line_number":641,"context_line":"When not network connectivity, this may be because the BMC address cannot"},{"line_number":642,"context_line":"be discovered from within the agent ramdisk *or* corrilated back to the node"},{"line_number":643,"context_line":"due to the use of DNS for an internal address, and there are no network ports"},{"line_number":644,"context_line":"registered *OR* the port address(es) contain a typo, or the MAC address is"},{"line_number":645,"context_line":"registered to a different physical node in Ironic."}],"source_content_type":"text/x-rst","patch_set":6,"id":"6099cb7a_4966898b","line":642,"range":{"start_line":642,"start_character":49,"end_line":642,"end_character":59},"updated":"2021-07-06 18:34:42.000000000","message":"correlated","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":640,"context_line":""},{"line_number":641,"context_line":"When not network connectivity, this may be because the BMC address cannot"},{"line_number":642,"context_line":"be discovered from within the agent ramdisk *or* corrilated back to the node"},{"line_number":643,"context_line":"due to the use of DNS for an internal address, and there are no network ports"},{"line_number":644,"context_line":"registered *OR* the port address(es) contain a typo, or the MAC address is"},{"line_number":645,"context_line":"registered to a different physical node in Ironic."},{"line_number":646,"context_line":""},{"line_number":647,"context_line":"Best practice is to ensure there is at least one physical network interface"},{"line_number":648,"context_line":"known by Ironic to facilitate network booting *and* identification"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3898d2bf_f83fe95e","line":645,"range":{"start_line":643,"start_character":0,"end_line":645,"end_character":50},"updated":"2021-07-06 18:34:42.000000000","message":"as above, for clarity I would prefer:\n\n\"This can be due to:\n- the address cannot be discovered\n- the address cannot be correlated\n- ... \"","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":924,"context_line":"  * The ``nova-conductor`` could not schedule instance creation to any cell."},{"line_number":925,"context_line":""},{"line_number":926,"context_line":"Actions:"},{"line_number":927,"context_line":"  * Verify that the ``nova-scheduler`` and/or ``placement`` serivce is"},{"line_number":928,"context_line":"    running and procssing requests."},{"line_number":929,"context_line":"  * Verify ``nova-scheduler`` logs."},{"line_number":930,"context_line":"  * Verify the ``placement`` service logs."}],"source_content_type":"text/x-rst","patch_set":6,"id":"17b46b70_22361c3d","line":927,"range":{"start_line":927,"start_character":60,"end_line":927,"end_character":67},"updated":"2021-07-06 18:34:42.000000000","message":"service","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":938,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":939,"context_line":""},{"line_number":940,"context_line":"Possible causes:"},{"line_number":941,"context_line":"  * There was an error creating the instance. The error happened before"},{"line_number":942,"context_line":"    the ``nova-compute`` process could reserve the Ironic node. Typicaly this"},{"line_number":943,"context_line":"    occurs when multiple requests are being fulfilled at the same time and"},{"line_number":944,"context_line":"    the same node is suggested by the scheduling and placement processes."},{"line_number":945,"context_line":"  * Baremetal nodes could have a lack of physical ports when compared to the"},{"line_number":946,"context_line":"    instance being created. Every virtual port requested in an instance"},{"line_number":947,"context_line":"    requires at least one physical instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"2caf6c35_b8178996","line":944,"range":{"start_line":941,"start_character":4,"end_line":944,"end_character":73},"updated":"2021-07-06 18:34:42.000000000","message":"Is this really still an issue? I have never observed this and would think the current code base is protected against such races.","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":973,"context_line":"Node in ``clean fail`` state"},{"line_number":974,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":975,"context_line":""},{"line_number":976,"context_line":"Possible causes:"},{"line_number":977,"context_line":""},{"line_number":978,"context_line":"* Connectivity failure between Ironic and the agent ramdisk during the"},{"line_number":979,"context_line":"  cleaning process. In this case, typically the ramdisk continues to"},{"line_number":980,"context_line":"  run and the operation timed out."},{"line_number":981,"context_line":"* A failure occured with one of the cleaning actions, such that operator"},{"line_number":982,"context_line":"  intervention is required to identify the root cause and determine the"},{"line_number":983,"context_line":"  appropriate path forward. Often this is along the lines of a failure"},{"line_number":984,"context_line":"  within a hardware manager, or a device failure during the cleaning"},{"line_number":985,"context_line":"  process, such as a disk having a non-default password resulting in"},{"line_number":986,"context_line":"  security mode not being able to be entered inside the ramdisk."},{"line_number":987,"context_line":"* Conductor logs as well as additional logging may be present in the form"},{"line_number":988,"context_line":"  of logs downloaded by the conductor. Please review these logs, as"},{"line_number":989,"context_line":"  they will often indicate exactly what has occured."},{"line_number":990,"context_line":""},{"line_number":991,"context_line":"Node locked"},{"line_number":992,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"45da9b89_9f08bce5","line":989,"range":{"start_line":976,"start_character":0,"end_line":989,"end_character":52},"updated":"2021-07-06 18:34:42.000000000","message":"Shouldn\u0027t be a \"node show\" be the first step? It usually has the error logged.","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":1012,"context_line":"Possible causes:"},{"line_number":1013,"context_line":"  * The Ironic node associated with the instance has been"},{"line_number":1014,"context_line":"    deleted from Ironic or due to some sort of transient"},{"line_number":1015,"context_line":"    failure, the node was in the process of shifting between"},{"line_number":1016,"context_line":"    the management of ``nova-compute`` instances when."},{"line_number":1017,"context_line":"  * If the baremetal node is not missing, then ``nova-compute``"},{"line_number":1018,"context_line":"    should automatically heal it\u0027s internal cache and hash ring"},{"line_number":1019,"context_line":"    as time moves forward, and a ``reset-state`` command can be"}],"source_content_type":"text/x-rst","patch_set":6,"id":"4461fa75_e73e79f7","line":1016,"range":{"start_line":1015,"start_character":13,"end_line":1016,"end_character":54},"updated":"2021-07-06 18:34:42.000000000","message":"incomplete?","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":1015,"context_line":"    failure, the node was in the process of shifting between"},{"line_number":1016,"context_line":"    the management of ``nova-compute`` instances when."},{"line_number":1017,"context_line":"  * If the baremetal node is not missing, then ``nova-compute``"},{"line_number":1018,"context_line":"    should automatically heal it\u0027s internal cache and hash ring"},{"line_number":1019,"context_line":"    as time moves forward, and a ``reset-state`` command can be"},{"line_number":1020,"context_line":"    issued."},{"line_number":1021,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"8784fc43_0db9551d","line":1018,"range":{"start_line":1018,"start_character":30,"end_line":1018,"end_character":34},"updated":"2021-07-06 18:34:42.000000000","message":"its","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":1020,"context_line":"    issued."},{"line_number":1021,"context_line":""},{"line_number":1022,"context_line":"Actions:"},{"line_number":1023,"context_line":"  * If missing, Re-add the node to Ironic with a matching node ``uuid``,"},{"line_number":1024,"context_line":"    ``instance_uuid`` and then utilize the ``adopt`` functionality"},{"line_number":1025,"context_line":"    to move the node into an appropriate state. From this state, the"},{"line_number":1026,"context_line":"    nova ``reset-state`` command can be utilized and commands issued"}],"source_content_type":"text/x-rst","patch_set":6,"id":"f76ddb83_4f548e64","line":1023,"range":{"start_line":1023,"start_character":16,"end_line":1023,"end_character":23},"updated":"2021-07-06 18:34:42.000000000","message":"re-add","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"04ceaeaaa78935345e8da2b59c74d86e3a9d5467","unresolved":true,"context_lines":[{"line_number":1025,"context_line":"    to move the node into an appropriate state. From this state, the"},{"line_number":1026,"context_line":"    nova ``reset-state`` command can be utilized and commands issued"},{"line_number":1027,"context_line":"    as would normally be expected."},{"line_number":1028,"context_line":"  * If the baremetal node has not been recreated, Use the"},{"line_number":1029,"context_line":"    Nova ``reset-state`` and ``force-delete`` command to"},{"line_number":1030,"context_line":"    reset the instance state:"},{"line_number":1031,"context_line":"    ``nova reset-state INSTANCE_UUID`` and"}],"source_content_type":"text/x-rst","patch_set":6,"id":"47a1bbbd_3303469c","line":1028,"range":{"start_line":1028,"start_character":50,"end_line":1028,"end_character":53},"updated":"2021-07-06 18:34:42.000000000","message":"use","commit_id":"6ec232b6eb584d3d9cf2142a0817076a96c56d9e"}],"doc/source/deploy/troubleshooting.rst":[{"author":{"_account_id":6618,"name":"Ruby Loo","email":"opensrloo@gmail.com","username":"rloo"},"change_message_id":"f87db09fffeb77200d89f02b65f1e7e8805fc79d","unresolved":false,"context_lines":[{"line_number":72,"context_line":"   again."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Instance Stuck in SCHEDULING"},{"line_number":76,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"Possible cause:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3aaa91ec_c91e395e","line":75,"updated":"2016-06-30 14:45:43.000000000","message":"Maybe add \u0027Nova\u0027 to this.\n\nI suspect we might want to split out the troubleshooting, and have maybe one section related to nova but this is good for now.","commit_id":"d3b5cdf5cfa4956e96343ba4676665cc062f8de2"},{"author":{"_account_id":10697,"name":"Thiago Paiva Brito","email":"thiagop@lsd.ufcg.edu.br","username":"thiagop","inactive":true},"change_message_id":"e289964aeb7d71d27e8d462ea740cd3bf70c8678","unresolved":false,"context_lines":[{"line_number":72,"context_line":"   again."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Instance Stuck in SCHEDULING"},{"line_number":76,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"Possible cause:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3aaa91ec_1b8c04d9","line":75,"in_reply_to":"3aaa91ec_c91e395e","updated":"2016-06-30 15:11:28.000000000","message":"Or \"Baremetal Instance\" to be more specific to our cases. Can be a problem with VM\u0027s too, but we\u0027re targeting Ironic users. :)","commit_id":"d3b5cdf5cfa4956e96343ba4676665cc062f8de2"},{"author":{"_account_id":6618,"name":"Ruby Loo","email":"opensrloo@gmail.com","username":"rloo"},"change_message_id":"f87db09fffeb77200d89f02b65f1e7e8805fc79d","unresolved":false,"context_lines":[{"line_number":76,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"Possible cause:"},{"line_number":79,"context_line":"  * The scheduling process failed and did not transition the instance in the"},{"line_number":80,"context_line":"    error state. This can happen when using nova-cells as cell scheduling is"},{"line_number":81,"context_line":"    known to be flaky."},{"line_number":82,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3aaa91ec_292edd8b","line":79,"updated":"2016-06-30 14:45:43.000000000","message":"s/in/to/","commit_id":"d3b5cdf5cfa4956e96343ba4676665cc062f8de2"},{"author":{"_account_id":6618,"name":"Ruby Loo","email":"opensrloo@gmail.com","username":"rloo"},"change_message_id":"f87db09fffeb77200d89f02b65f1e7e8805fc79d","unresolved":false,"context_lines":[{"line_number":84,"context_line":"  * Verify nova-cells logs on both API and metal cells"},{"line_number":85,"context_line":"  * Ensure all nova-cells configuration are present (especially capabilities)"},{"line_number":86,"context_line":"  * Validate if both parent and child cells are on the same codebase"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Instance in ERROR, no node with instance UUID"},{"line_number":90,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3aaa91ec_096b21b8","line":87,"updated":"2016-06-30 14:45:43.000000000","message":"This is all related to nova, is it specific to ironic? (I mean, if it was a VM, would this also be a problem with nova?)","commit_id":"d3b5cdf5cfa4956e96343ba4676665cc062f8de2"},{"author":{"_account_id":10697,"name":"Thiago Paiva Brito","email":"thiagop@lsd.ufcg.edu.br","username":"thiagop","inactive":true},"change_message_id":"e289964aeb7d71d27e8d462ea740cd3bf70c8678","unresolved":false,"context_lines":[{"line_number":89,"context_line":"Instance in ERROR, no node with instance UUID"},{"line_number":90,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"Possible cause:"},{"line_number":93,"context_line":"  * There was an error creating the instance. The error happened before Nova"},{"line_number":94,"context_line":"    reserved the Ironic node."},{"line_number":95,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3aaa91ec_5b804c8b","line":92,"range":{"start_line":92,"start_character":9,"end_line":92,"end_character":14},"updated":"2016-06-30 15:11:28.000000000","message":"if not a table, please uses \"causes\" so people are encouraged to add more cases. :)","commit_id":"d3b5cdf5cfa4956e96343ba4676665cc062f8de2"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"97facc63652e3eb066c92abe73f2cf87bb81c81c","unresolved":false,"context_lines":[{"line_number":105,"context_line":""},{"line_number":106,"context_line":"Possible causes:"},{"line_number":107,"context_line":"  * There was an error creating the instance. The error happened after nova"},{"line_number":108,"context_line":"    reserved the Ironic node."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"Actions:"},{"line_number":111,"context_line":"  * As admin, run ``nova show INSTANCE_UUID`` and look at the error message."}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_7f5aa92b","line":108,"updated":"2016-07-11 13:26:44.000000000","message":"this is not something I ever see happening. in my setup nova always retries any failures from ironic, so instance_uuid is never set :( I think it\u0027s worth mentioning as a possibility","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"97facc63652e3eb066c92abe73f2cf87bb81c81c","unresolved":false,"context_lines":[{"line_number":153,"context_line":"  * Verify DHCP service is running."},{"line_number":154,"context_line":"  * Verify no other DHCP server is running on the same network."},{"line_number":155,"context_line":"  * Verify PXE server information is handed out by the DHCP server."},{"line_number":156,"context_line":"  * Verify PXE server is reachable from the DHCP subnet."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Node in ERROR"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_df411d3b","line":156,"updated":"2016-07-11 13:26:44.000000000","message":"* Verify that the switch does not block DHCP requests and/or responses to and from the Ironic host.","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"e9fc7a79ed8e6a6ccae2c60641ea69a81422da60","unresolved":false,"context_lines":[{"line_number":153,"context_line":"  * Verify DHCP service is running."},{"line_number":154,"context_line":"  * Verify no other DHCP server is running on the same network."},{"line_number":155,"context_line":"  * Verify PXE server information is handed out by the DHCP server."},{"line_number":156,"context_line":"  * Verify PXE server is reachable from the DHCP subnet."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Node in ERROR"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5f7c97a3_1d5f2542","line":156,"in_reply_to":"1aa78d24_df411d3b","updated":"2018-05-03 22:54:48.000000000","message":"Spanning-tree hold-down timers may be another possibility and prevent a port from being usable with-in the first 30 seconds.","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"97facc63652e3eb066c92abe73f2cf87bb81c81c","unresolved":false,"context_lines":[{"line_number":211,"context_line":"  * The deletion process took too long and Nova timed out."},{"line_number":212,"context_line":"  * The deletion process was interrupted."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"Actions:"},{"line_number":215,"context_line":"  * Try transitioning the node to another state via ironic"},{"line_number":216,"context_line":"    node-set-provision-state. This will most likely fail because a transition"},{"line_number":217,"context_line":"    is in progress."}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_bf3c11c0","line":214,"updated":"2016-07-11 13:26:44.000000000","message":"I would start with \"Try restarting deletion\". As stupid as it sounds, sometimes it actually helps.","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"97facc63652e3eb066c92abe73f2cf87bb81c81c","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  * The deletion process was interrupted."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"Actions:"},{"line_number":215,"context_line":"  * Try transitioning the node to another state via ironic"},{"line_number":216,"context_line":"    node-set-provision-state. This will most likely fail because a transition"},{"line_number":217,"context_line":"    is in progress."},{"line_number":218,"context_line":"  * Manually edit the database to change the node\u0027s provision_state to \"error\"."}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_1f4c456f","line":215,"updated":"2016-07-11 13:26:44.000000000","message":"lets be specific about another state. which state do you suggest here?","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":20835,"name":"Gerald Lonlas","email":"glonlas@internap.com","username":"glonlas"},"change_message_id":"fd4439f0cd56949138708c9f5e8b4659e2dbda8a","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  * The deletion process was interrupted."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"Actions:"},{"line_number":215,"context_line":"  * Try transitioning the node to another state via ironic"},{"line_number":216,"context_line":"    node-set-provision-state. This will most likely fail because a transition"},{"line_number":217,"context_line":"    is in progress."},{"line_number":218,"context_line":"  * Manually edit the database to change the node\u0027s provision_state to \"error\"."}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a74a57a_fe43c7ac","line":215,"in_reply_to":"1aa78d24_1f4c456f","updated":"2016-11-28 19:57:10.000000000","message":"Why to not go to \"abort\" to get a clean failed and restart?\n`` ironic node-set-provision-state NODE_UUID abort``","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"97facc63652e3eb066c92abe73f2cf87bb81c81c","unresolved":false,"context_lines":[{"line_number":215,"context_line":"  * Try transitioning the node to another state via ironic"},{"line_number":216,"context_line":"    node-set-provision-state. This will most likely fail because a transition"},{"line_number":217,"context_line":"    is in progress."},{"line_number":218,"context_line":"  * Manually edit the database to change the node\u0027s provision_state to \"error\"."},{"line_number":219,"context_line":"  * Use the Ironic CLI to transition the node to the \"deleted\" state."},{"line_number":220,"context_line":"  * Finally, delete the instance in \"ERROR DELETING\" by using"},{"line_number":221,"context_line":"    ``nova delete INSTANCE_UUID``."}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_ff46994d","line":218,"updated":"2016-07-11 13:26:44.000000000","message":"Lets not have \"edit the database\" in the official documents... I understand it\u0027s needed sometimes, but we can officially bless it.","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"97facc63652e3eb066c92abe73f2cf87bb81c81c","unresolved":false,"context_lines":[{"line_number":233,"context_line":"    the node."},{"line_number":234,"context_line":"  * Verify all ironic-conductors to ensure no conductors are actively using the"},{"line_number":235,"context_line":"    node."},{"line_number":236,"context_line":"  * Manually edit the database to remove the node\u0027s reservation."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"Baremetal instance in ERROR and node stuck in DEPLOYING"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_5f740d9b","line":236,"updated":"2016-07-11 13:26:44.000000000","message":"ditto about manual editing. we can\u0027t really recommend it. also restarting the conductor usually removes the lock, we\u0027d better recommend it.","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"97facc63652e3eb066c92abe73f2cf87bb81c81c","unresolved":false,"context_lines":[{"line_number":249,"context_line":"    node-set-provision-state. This will most likely fail because a transition"},{"line_number":250,"context_line":"    is in progress."},{"line_number":251,"context_line":"  * Manually edit the database to change the node\u0027s provision_state to"},{"line_number":252,"context_line":"    \"wait call-back\"."},{"line_number":253,"context_line":"  * Reboot the node, let it boot back into IPA."},{"line_number":254,"context_line":"  * The node should transition into DEPLOY FAILED on it\u0027s own."},{"line_number":255,"context_line":"  * Use the Ironic CLI to transition the node to the \"deleted\" state."}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_3f7781a0","line":252,"updated":"2016-07-11 13:26:44.000000000","message":"It\u0027s not needed. I think now it\u0027s enough to restart the conductor.","commit_id":"816f8c734eb2412517d279b30164a7a11a961458"}]}