)]}'
{"doc/source/admin/troubleshooting.rst":[{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"eae567e1845b0f3874c4efbd4c535352866dd790","unresolved":false,"context_lines":[{"line_number":408,"context_line":"sorts of operations can vary based on the specific environment and operating"},{"line_number":409,"context_line":"configuration."},{"line_number":410,"context_line":""},{"line_number":411,"context_line":"What can cause these sorts of failures?"},{"line_number":412,"context_line":"---------------------------------------"},{"line_number":413,"context_line":""},{"line_number":414,"context_line":"Typical causes of such failures are going to be largely rooted in the concept"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ff570b3c_b900f803","line":411,"updated":"2020-06-03 18:15:08.000000000","message":"docs: Title level inconsistent:","commit_id":"5c76ee9a4b45e7cd61aa408ac5aa7418ce1d5ab6"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":402,"context_line":"and the conductor is downloading, uploading, or attempting to perform some"},{"line_number":403,"context_line":"sort of Input/Output operation."},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"Often these sorts of operations should time out, but there are cases in"},{"line_number":406,"context_line":"operating systems where operations are blocked until completion. These"},{"line_number":407,"context_line":"sorts of operations can vary based on the specific environment and operating"},{"line_number":408,"context_line":"configuration."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_e3dd9241","line":405,"range":{"start_line":405,"start_character":0,"end_line":405,"end_character":5},"updated":"2020-06-04 06:17:06.000000000","message":"Rather than \"Often\", how about: \"In case the conductor gets stuck, these operations ...\"","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":402,"context_line":"and the conductor is downloading, uploading, or attempting to perform some"},{"line_number":403,"context_line":"sort of Input/Output operation."},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"Often these sorts of operations should time out, but there are cases in"},{"line_number":406,"context_line":"operating systems where operations are blocked until completion. These"},{"line_number":407,"context_line":"sorts of operations can vary based on the specific environment and operating"},{"line_number":408,"context_line":"configuration."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_4cd2a201","line":405,"range":{"start_line":405,"start_character":0,"end_line":405,"end_character":5},"in_reply_to":"ff570b3c_e3dd9241","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":416,"context_line":"`iostat \u003chttps://man7.org/linux/man-pages/man1/iostat.1.html\u003e`_ tool to"},{"line_number":417,"context_line":"identify the percentage of CPU time spent waiting on storage devices."},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"The fields that will be particularlly important is the ``iowait``, ``await``,"},{"line_number":420,"context_line":"and ``tps`` fields, which can be read about in the ``iostat`` manual page."},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_802c8cac","line":419,"range":{"start_line":419,"start_character":24,"end_line":419,"end_character":37},"updated":"2020-06-04 06:17:06.000000000","message":"Nit: particularly","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"9389af743447fcd26841acdcf91cf58b46bda83f","unresolved":false,"context_lines":[{"line_number":416,"context_line":"`iostat \u003chttps://man7.org/linux/man-pages/man1/iostat.1.html\u003e`_ tool to"},{"line_number":417,"context_line":"identify the percentage of CPU time spent waiting on storage devices."},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"The fields that will be particularlly important is the ``iowait``, ``await``,"},{"line_number":420,"context_line":"and ``tps`` fields, which can be read about in the ``iostat`` manual page."},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_551a30c5","line":419,"range":{"start_line":419,"start_character":48,"end_line":419,"end_character":50},"updated":"2020-06-04 12:37:42.000000000","message":"are","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":416,"context_line":"`iostat \u003chttps://man7.org/linux/man-pages/man1/iostat.1.html\u003e`_ tool to"},{"line_number":417,"context_line":"identify the percentage of CPU time spent waiting on storage devices."},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"The fields that will be particularlly important is the ``iowait``, ``await``,"},{"line_number":420,"context_line":"and ``tps`` fields, which can be read about in the ``iostat`` manual page."},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_cc07d281","line":419,"range":{"start_line":419,"start_character":48,"end_line":419,"end_character":50},"in_reply_to":"ff570b3c_551a30c5","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":416,"context_line":"`iostat \u003chttps://man7.org/linux/man-pages/man1/iostat.1.html\u003e`_ tool to"},{"line_number":417,"context_line":"identify the percentage of CPU time spent waiting on storage devices."},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"The fields that will be particularlly important is the ``iowait``, ``await``,"},{"line_number":420,"context_line":"and ``tps`` fields, which can be read about in the ``iostat`` manual page."},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_ac0a5e86","line":419,"range":{"start_line":419,"start_character":24,"end_line":419,"end_character":37},"in_reply_to":"ff570b3c_802c8cac","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"9389af743447fcd26841acdcf91cf58b46bda83f","unresolved":false,"context_lines":[{"line_number":417,"context_line":"identify the percentage of CPU time spent waiting on storage devices."},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"The fields that will be particularlly important is the ``iowait``, ``await``,"},{"line_number":420,"context_line":"and ``tps`` fields, which can be read about in the ``iostat`` manual page."},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"},{"line_number":423,"context_line":"caches or distributed ``tftpboot`` or ``httpboot`` folders, IO operations"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_b57aaced","line":420,"range":{"start_line":420,"start_character":12,"end_line":420,"end_character":18},"updated":"2020-06-04 12:37:42.000000000","message":"nit: ones","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":417,"context_line":"identify the percentage of CPU time spent waiting on storage devices."},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"The fields that will be particularlly important is the ``iowait``, ``await``,"},{"line_number":420,"context_line":"and ``tps`` fields, which can be read about in the ``iostat`` manual page."},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"},{"line_number":423,"context_line":"caches or distributed ``tftpboot`` or ``httpboot`` folders, IO operations"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_6cd32602","line":420,"range":{"start_line":420,"start_character":12,"end_line":420,"end_character":18},"in_reply_to":"ff570b3c_b57aaced","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"},{"line_number":423,"context_line":"caches or distributed ``tftpboot`` or ``httpboot`` folders, IO operations"},{"line_number":424,"context_line":"failing on these can, depending on operating system and underlying client"},{"line_number":425,"context_line":"settings can cause threads to be stuck in a blocking wait state, which is"},{"line_number":426,"context_line":"realistically undetectable short the operating system logging connectivity"},{"line_number":427,"context_line":"errors or even lock manager access errors."},{"line_number":428,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_c3296e8a","line":425,"range":{"start_line":425,"start_character":8,"end_line":425,"end_character":12},"updated":"2020-06-04 06:17:06.000000000","message":"Nit: s/ can/, /","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":422,"context_line":"In the case of network file systems, for backing components such as image"},{"line_number":423,"context_line":"caches or distributed ``tftpboot`` or ``httpboot`` folders, IO operations"},{"line_number":424,"context_line":"failing on these can, depending on operating system and underlying client"},{"line_number":425,"context_line":"settings can cause threads to be stuck in a blocking wait state, which is"},{"line_number":426,"context_line":"realistically undetectable short the operating system logging connectivity"},{"line_number":427,"context_line":"errors or even lock manager access errors."},{"line_number":428,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_ac2ffef3","line":425,"range":{"start_line":425,"start_character":8,"end_line":425,"end_character":12},"in_reply_to":"ff570b3c_c3296e8a","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":428,"context_line":""},{"line_number":429,"context_line":"For example with"},{"line_number":430,"context_line":"`nfs \u003chttps://www.man7.org/linux/man-pages/man5/nfs.5.html\u003e`_,"},{"line_number":431,"context_line":"the under client recovery behavior, ``soft``, ``hard``, ``softreval``,"},{"line_number":432,"context_line":"``nosoftreval``, will largely impact this behavior, but also NFS server"},{"line_number":433,"context_line":"settings can impact this behavior. A solid sign that this is a failure,"},{"line_number":434,"context_line":"is when an ``ls /path/to/nfs`` command hangs for a period of time."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_8348165c","line":431,"range":{"start_line":431,"start_character":4,"end_line":431,"end_character":16},"updated":"2020-06-04 06:17:06.000000000","message":"\"under client\"?","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":428,"context_line":""},{"line_number":429,"context_line":"For example with"},{"line_number":430,"context_line":"`nfs \u003chttps://www.man7.org/linux/man-pages/man5/nfs.5.html\u003e`_,"},{"line_number":431,"context_line":"the under client recovery behavior, ``soft``, ``hard``, ``softreval``,"},{"line_number":432,"context_line":"``nosoftreval``, will largely impact this behavior, but also NFS server"},{"line_number":433,"context_line":"settings can impact this behavior. A solid sign that this is a failure,"},{"line_number":434,"context_line":"is when an ``ls /path/to/nfs`` command hangs for a period of time."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_ac189ed2","line":431,"range":{"start_line":431,"start_character":4,"end_line":431,"end_character":16},"in_reply_to":"ff570b3c_8348165c","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":441,"context_line":""},{"line_number":442,"context_line":"If the node has a populated ``reservation`` field, and has not timed out or"},{"line_number":443,"context_line":"proceeded to a ``fail`` state, then the conductor process will likely need to"},{"line_number":444,"context_line":"be restated. This is because the worker thread is hung with-in the conductor."},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"Manual intervention with-in ironic\u0027s database is *not* advised to try and"},{"line_number":447,"context_line":"\"un-wedge\" the machine in this state, and restarting the conductor is"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_23666aee","line":444,"range":{"start_line":444,"start_character":3,"end_line":444,"end_character":11},"updated":"2020-06-04 06:17:06.000000000","message":"Nit: restarted","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":441,"context_line":""},{"line_number":442,"context_line":"If the node has a populated ``reservation`` field, and has not timed out or"},{"line_number":443,"context_line":"proceeded to a ``fail`` state, then the conductor process will likely need to"},{"line_number":444,"context_line":"be restated. This is because the worker thread is hung with-in the conductor."},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"Manual intervention with-in ironic\u0027s database is *not* advised to try and"},{"line_number":447,"context_line":"\"un-wedge\" the machine in this state, and restarting the conductor is"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_ec0e168a","line":444,"range":{"start_line":444,"start_character":3,"end_line":444,"end_character":11},"in_reply_to":"ff570b3c_23666aee","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":443,"context_line":"proceeded to a ``fail`` state, then the conductor process will likely need to"},{"line_number":444,"context_line":"be restated. This is because the worker thread is hung with-in the conductor."},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"Manual intervention with-in ironic\u0027s database is *not* advised to try and"},{"line_number":447,"context_line":"\"un-wedge\" the machine in this state, and restarting the conductor is"},{"line_number":448,"context_line":"encouraged."},{"line_number":449,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_23702a72","line":446,"range":{"start_line":446,"start_character":28,"end_line":446,"end_character":34},"updated":"2020-06-04 06:17:06.000000000","message":"Ironic\u0027s","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":443,"context_line":"proceeded to a ``fail`` state, then the conductor process will likely need to"},{"line_number":444,"context_line":"be restated. This is because the worker thread is hung with-in the conductor."},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"Manual intervention with-in ironic\u0027s database is *not* advised to try and"},{"line_number":447,"context_line":"\"un-wedge\" the machine in this state, and restarting the conductor is"},{"line_number":448,"context_line":"encouraged."},{"line_number":449,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_4c2902e4","line":446,"range":{"start_line":446,"start_character":28,"end_line":446,"end_character":34},"in_reply_to":"ff570b3c_23702a72","updated":"2020-06-04 17:35:14.000000000","message":"Done","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":451,"context_line":"   Ironic\u0027s conductor, upon restart, clears reservations for nodes which"},{"line_number":452,"context_line":"   were previously managed by the conductor before restart."},{"line_number":453,"context_line":""},{"line_number":454,"context_line":"IF a distributed or network file system is in use, it is highly recommended"},{"line_number":455,"context_line":"that the operating system of the node running the conductor be rebooted as"},{"line_number":456,"context_line":"the running conductor may not even be able to exit in the state of an IO"},{"line_number":457,"context_line":"failure, again dependent upon site and server configuration."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_43defe6c","line":454,"range":{"start_line":454,"start_character":0,"end_line":454,"end_character":2},"updated":"2020-06-04 06:17:06.000000000","message":"Nit: \"If\"?","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"42174d8412030a10605245e51d079b9ca181a73c","unresolved":false,"context_lines":[{"line_number":469,"context_line":""},{"line_number":470,"context_line":"* Conversion to raw image files upon download to the conductor, from the"},{"line_number":471,"context_line":"  ``[DEFAULT]force_raw_images`` option, in particular with the ``iscsi``"},{"line_number":472,"context_line":"  deployment interface. Users using glance and the ``direct`` deployment"},{"line_number":473,"context_line":"  interface may also experience issues here as the conductor will cache"},{"line_number":474,"context_line":"  the image to be written."},{"line_number":475,"context_line":""},{"line_number":476,"context_line":"* Write of a QCOW2 file over the ``iscsi`` deployment interface from the"},{"line_number":477,"context_line":"  conductor to the node being deployed can result in large amounts of"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_e346327b","line":474,"range":{"start_line":472,"start_character":24,"end_line":474,"end_character":26},"updated":"2020-06-04 06:17:06.000000000","message":"The conductor will cache the image in direct mode? I thought with direct, the node will bypass the conductor and connect to the image source directly, no?","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"5e44347f7373cf92e8553a62e783206fd4198910","unresolved":false,"context_lines":[{"line_number":469,"context_line":""},{"line_number":470,"context_line":"* Conversion to raw image files upon download to the conductor, from the"},{"line_number":471,"context_line":"  ``[DEFAULT]force_raw_images`` option, in particular with the ``iscsi``"},{"line_number":472,"context_line":"  deployment interface. Users using glance and the ``direct`` deployment"},{"line_number":473,"context_line":"  interface may also experience issues here as the conductor will cache"},{"line_number":474,"context_line":"  the image to be written."},{"line_number":475,"context_line":""},{"line_number":476,"context_line":"* Write of a QCOW2 file over the ``iscsi`` deployment interface from the"},{"line_number":477,"context_line":"  conductor to the node being deployed can result in large amounts of"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ff570b3c_cc635241","line":474,"range":{"start_line":472,"start_character":24,"end_line":474,"end_character":26},"in_reply_to":"ff570b3c_e346327b","updated":"2020-06-04 17:35:14.000000000","message":"There is a very fine variance with the indirect download code path.  I don\u0027t remember how that gets followed anymore. :\\","commit_id":"6eb7d82bcc4592af098b549814682dc064f0b014"}]}