)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"7795bc5ed170f99b6b88331ac99c0875f4c85a4e","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix wait for detach code to handle \u0027the not found\u0027 from libvirt"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Disk may be detached by external operation but the domain xml don\u0027t"},{"line_number":10,"context_line":"remove device section. So we may manage to get device info(conf) from"},{"line_number":11,"context_line":"xml but fail to detach because it doesn\u0027t exist. We should check if the"},{"line_number":12,"context_line":"target device exists and should be handling the not found from libvirt"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5a74a57a_ebf38708","line":9,"updated":"2016-11-24 11:00:36.000000000","message":"I\u0027m not sure to really understand, the errors you are handling/reporting are because libvirt does not find the target disk in the domain definition.","commit_id":"2e6cb02d193ee3d00d830aeee6d67baccce419bf"},{"author":{"_account_id":21479,"name":"int32bit","email":"krystism@gmail.com","username":"int32bit"},"change_message_id":"6c3069155c2bcca5c57a7beb42e6611fceaf8769","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix wait for detach code to handle \u0027the not found\u0027 from libvirt"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Disk may be detached by external operation but the domain xml don\u0027t"},{"line_number":10,"context_line":"remove device section. So we may manage to get device info(conf) from"},{"line_number":11,"context_line":"xml but fail to detach because it doesn\u0027t exist. We should check if the"},{"line_number":12,"context_line":"target device exists and should be handling the not found from libvirt"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5a74a57a_ce7c19ee","line":9,"in_reply_to":"5a74a57a_ebf38708","updated":"2016-11-24 11:32:21.000000000","message":"It seems we currently get device info just from xml:\nhttps://github.com/openstack/nova/blob/master/nova/virt/libvirt/guest.py#L299-L303\nBut sometimes it\u0027s not 100% reliable. For example, someone may have successfully detach device but fail to tell libvirt to update xml. I think this bug just hit it.","commit_id":"2e6cb02d193ee3d00d830aeee6d67baccce419bf"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"fd02927f03ee16c96e9c3dd6c6a60127497dcd14","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Fix wait for detach code to handle \u0027disk not found error\u0027"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"We currently retrieve device info from instance xml which is not"},{"line_number":10,"context_line":"100% realiable. The disk may have been detached but failed to tell"},{"line_number":11,"context_line":"libvirt to update xml."},{"line_number":12,"context_line":"If the disk doesn\u0027t exist, try to detach it will raise a LibvirtError"},{"line_number":13,"context_line":"exception, but our loop code in `_do_wait_and_retry_detach` doesn\u0027t"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"5a74a57a_8b67264c","line":10,"updated":"2016-11-27 09:39:17.000000000","message":"reliable","commit_id":"9ae1d6d58c62279b67f75ecfe6f9f0579585ba73"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"fd02927f03ee16c96e9c3dd6c6a60127497dcd14","unresolved":false,"context_lines":[{"line_number":9,"context_line":"We currently retrieve device info from instance xml which is not"},{"line_number":10,"context_line":"100% realiable. The disk may have been detached but failed to tell"},{"line_number":11,"context_line":"libvirt to update xml."},{"line_number":12,"context_line":"If the disk doesn\u0027t exist, try to detach it will raise a LibvirtError"},{"line_number":13,"context_line":"exception, but our loop code in `_do_wait_and_retry_detach` doesn\u0027t"},{"line_number":14,"context_line":"catch it and will response an error. We should double check if the"},{"line_number":15,"context_line":"target device exists and should be handling the `disk not found error`"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"5a74a57a_cb5d9e7a","line":12,"updated":"2016-11-27 09:39:17.000000000","message":"\"trying to detach\"","commit_id":"9ae1d6d58c62279b67f75ecfe6f9f0579585ba73"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"fd02927f03ee16c96e9c3dd6c6a60127497dcd14","unresolved":false,"context_lines":[{"line_number":11,"context_line":"libvirt to update xml."},{"line_number":12,"context_line":"If the disk doesn\u0027t exist, try to detach it will raise a LibvirtError"},{"line_number":13,"context_line":"exception, but our loop code in `_do_wait_and_retry_detach` doesn\u0027t"},{"line_number":14,"context_line":"catch it and will response an error. We should double check if the"},{"line_number":15,"context_line":"target device exists and should be handling the `disk not found error`"},{"line_number":16,"context_line":"from libvirt and consider the job complete."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"5a74a57a_4b4a6ead","line":14,"updated":"2016-11-27 09:39:17.000000000","message":"s/response an/respond with/","commit_id":"9ae1d6d58c62279b67f75ecfe6f9f0579585ba73"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"9ca4ceee4b002f01864077155cf68155d34deac4","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix wait for detach code to handle \u0027disk not found error\u0027"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"We currently retrieve device info from instance xml which is not"},{"line_number":10,"context_line":"100% reliable. The disk may have been detached but failed to tell"},{"line_number":11,"context_line":"libvirt to update xml."},{"line_number":12,"context_line":"If the disk doesn\u0027t exist, trying to detach it will raise a LibvirtError"},{"line_number":13,"context_line":"exception, but our loop code in `_do_wait_and_retry_detach` doesn\u0027t"},{"line_number":14,"context_line":"catch it and will respond with error. We should double check if the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"5a74a57a_f27c98ed","line":11,"range":{"start_line":9,"start_character":0,"end_line":11,"end_character":22},"updated":"2016-11-30 00:36:37.000000000","message":"I think this is actually a race. Our code does an initial detach from persistent and transient domain in one call, then calls the retryable function afterward, which first checks the domain xml before retrying the transient domain detach. If the transient domain detach (which is asynchronous) completes after the domain xml check, we will fail the detach with the LibvirtError.","commit_id":"07956a4b25faa8d4362264d073cb1c09433a3dc6"}],"nova/tests/unit/virt/libvirt/test_guest.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"114c46aa95b79a74e7b21a167883c0be8be71738","unresolved":false,"context_lines":[{"line_number":280,"context_line":"            fakelibvirt.libvirtError,"},{"line_number":281,"context_line":"            msg\u003d\"invalid argument: no target device vdb\","},{"line_number":282,"context_line":"            error_code\u003dfakelibvirt.VIR_ERR_OPERATION_FAILED,"},{"line_number":283,"context_line":"            error_message\u003d\"disk %s not found\","},{"line_number":284,"context_line":"            error_domain\u003dfakelibvirt.VIR_FROM_DOMAIN)"},{"line_number":285,"context_line":"        mock_detach.side_effect \u003d [None, fake_exc]"},{"line_number":286,"context_line":"        retry_detach \u003d self.guest.detach_device_with_retry("}],"source_content_type":"text/x-python","patch_set":5,"id":"5a74a57a_683b3e39","line":283,"range":{"start_line":283,"start_character":32,"end_line":283,"end_character":34},"updated":"2016-11-29 21:04:28.000000000","message":"Just hardcode vdb here?","commit_id":"ed5b064891c5f273c0ce15d411878b21b53b5b07"}],"nova/virt/libvirt/guest.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"e87f6c91d8f44e77a22073fb1185363b82e0dfde","unresolved":false,"context_lines":[{"line_number":384,"context_line":"                    msg \u003d e.get_error_message()"},{"line_number":385,"context_line":"                    # FIXME Should we check error message ?"},{"line_number":386,"context_line":"                    if (code \u003d\u003d libvirt.VIR_ERR_INVALID_ARG"},{"line_number":387,"context_line":"                        and msg.find(\"no target device\") \u003e\u003d 0):"},{"line_number":388,"context_line":"                        raise exception.DeviceNotFound(message\u003de.message)"},{"line_number":389,"context_line":"                    else:"},{"line_number":390,"context_line":"                        # Raise error since the device still existed"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a74a57a_4c659643","line":387,"range":{"start_line":387,"start_character":24,"end_line":387,"end_character":63},"updated":"2016-11-23 18:44:37.000000000","message":"If this were translated in libvirt and the locale wasn\u0027t english, this would fail. So if the code is unique to the not found error, we should just use that. We\u0027d probably have to dig into the libvirt 1.3.1 code to see if that\u0027s the case, we don\u0027t always get unique codes from libvirt errors.","commit_id":"a4005ec036b8e40f1edb4c9f9b25384bb702893e"},{"author":{"_account_id":21479,"name":"int32bit","email":"krystism@gmail.com","username":"int32bit"},"change_message_id":"e6fab27d23f25f780a49bc94d948c4a70faba00e","unresolved":false,"context_lines":[{"line_number":384,"context_line":"                    msg \u003d e.get_error_message()"},{"line_number":385,"context_line":"                    # FIXME Should we check error message ?"},{"line_number":386,"context_line":"                    if (code \u003d\u003d libvirt.VIR_ERR_INVALID_ARG"},{"line_number":387,"context_line":"                        and msg.find(\"no target device\") \u003e\u003d 0):"},{"line_number":388,"context_line":"                        raise exception.DeviceNotFound(message\u003de.message)"},{"line_number":389,"context_line":"                    else:"},{"line_number":390,"context_line":"                        # Raise error since the device still existed"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a74a57a_c6725049","line":387,"range":{"start_line":387,"start_character":24,"end_line":387,"end_character":63},"in_reply_to":"5a74a57a_4752d114","updated":"2016-11-24 04:24:38.000000000","message":"Yes, that\u0027s because I tested on my environment(rbd backend), the libvirt always raise VIR_ERR_INVALID(\"no target device\"):\nhttps://github.com/libvirt/libvirt/blob/master/src/qemu/qemu_driver.c#L8055-L8059\nBut the bug seems trigger this exception:\nhttps://github.com/libvirt/libvirt/blob/master/src/qemu/qemu_hotplug.c#L4285-L4289\n\nI think both should be checked.","commit_id":"a4005ec036b8e40f1edb4c9f9b25384bb702893e"},{"author":{"_account_id":21479,"name":"int32bit","email":"krystism@gmail.com","username":"int32bit"},"change_message_id":"e6fab27d23f25f780a49bc94d948c4a70faba00e","unresolved":false,"context_lines":[{"line_number":384,"context_line":"                    msg \u003d e.get_error_message()"},{"line_number":385,"context_line":"                    # FIXME Should we check error message ?"},{"line_number":386,"context_line":"                    if (code \u003d\u003d libvirt.VIR_ERR_INVALID_ARG"},{"line_number":387,"context_line":"                        and msg.find(\"no target device\") \u003e\u003d 0):"},{"line_number":388,"context_line":"                        raise exception.DeviceNotFound(message\u003de.message)"},{"line_number":389,"context_line":"                    else:"},{"line_number":390,"context_line":"                        # Raise error since the device still existed"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a74a57a_06936850","line":387,"range":{"start_line":387,"start_character":24,"end_line":387,"end_character":63},"in_reply_to":"5a74a57a_4c659643","updated":"2016-11-24 04:24:38.000000000","message":"Done","commit_id":"a4005ec036b8e40f1edb4c9f9b25384bb702893e"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fb2ec8723cd90cfaba416a1fdadb8ab2f1a9f5d4","unresolved":false,"context_lines":[{"line_number":384,"context_line":"                    msg \u003d e.get_error_message()"},{"line_number":385,"context_line":"                    # FIXME Should we check error message ?"},{"line_number":386,"context_line":"                    if (code \u003d\u003d libvirt.VIR_ERR_INVALID_ARG"},{"line_number":387,"context_line":"                        and msg.find(\"no target device\") \u003e\u003d 0):"},{"line_number":388,"context_line":"                        raise exception.DeviceNotFound(message\u003de.message)"},{"line_number":389,"context_line":"                    else:"},{"line_number":390,"context_line":"                        # Raise error since the device still existed"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a74a57a_4752d114","line":387,"range":{"start_line":387,"start_character":24,"end_line":387,"end_character":63},"in_reply_to":"5a74a57a_4c659643","updated":"2016-11-23 18:57:30.000000000","message":"This also doesn\u0027t look like what we\u0027re hitting in the ceph job:\n\nhttp://logs.openstack.org/00/398800/1/gate/gate-tempest-dsvm-full-devstack-plugin-ceph-ubuntu-xenial/a387fb0/logs/screen-n-cpu.txt.gz?level\u003dTRACE#_2016-11-17_11_39_00_649\n\nlibvirtError: operation failed: disk vdb not found\n\nI\u0027m pretty sure the \u0027operation failed\u0027 is a specific code, but other than that we probably just have to work with the error message, but we could probably get by with looking for \u0027not found\u0027 in there. It doesn\u0027t work for non-English locales, but that\u0027s about as good as we can do it looks like. It\u0027s not great, but we do it elsewhere, see:\n\n2ce193e16d7ca5b93671d7f0d2e3b35761f5d386","commit_id":"a4005ec036b8e40f1edb4c9f9b25384bb702893e"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"7795bc5ed170f99b6b88331ac99c0875f4c85a4e","unresolved":false,"context_lines":[{"line_number":382,"context_line":"                except libvirt.libvirtError as e:"},{"line_number":383,"context_line":"                    code \u003d e.get_error_code()"},{"line_number":384,"context_line":"                    msg \u003d e.get_error_message()"},{"line_number":385,"context_line":"                    if ((code \u003d\u003d libvirt.VIR_ERR_INVALID_ARG and"},{"line_number":386,"context_line":"                         \u0027no target device\u0027 in msg)"},{"line_number":387,"context_line":"                        or (code \u003d\u003d libvirt.VIR_ERR_OPERATION_FAILED and"},{"line_number":388,"context_line":"                            \u0027not found\u0027 in msg)):"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_eb09c72d","line":385,"updated":"2016-11-24 11:00:36.000000000","message":"Hum both of these errors could happen only when libvirt is verifying that the disk passed exists in the domain definition.\nSince to get the target disk we actually have looked to the domain definition I wondering to know if we are not dealing with a race condition or something?","commit_id":"2e6cb02d193ee3d00d830aeee6d67baccce419bf"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"04fbb3629d5d424c1c18a3032b8d380eafdfeff8","unresolved":false,"context_lines":[{"line_number":382,"context_line":"                except libvirt.libvirtError as e:"},{"line_number":383,"context_line":"                    code \u003d e.get_error_code()"},{"line_number":384,"context_line":"                    msg \u003d e.get_error_message()"},{"line_number":385,"context_line":"                    if ((code \u003d\u003d libvirt.VIR_ERR_INVALID_ARG and"},{"line_number":386,"context_line":"                         \u0027no target device\u0027 in msg)"},{"line_number":387,"context_line":"                        or (code \u003d\u003d libvirt.VIR_ERR_OPERATION_FAILED and"},{"line_number":388,"context_line":"                            \u0027not found\u0027 in msg)):"},{"line_number":389,"context_line":"                        raise exception.DeviceNotFound(message\u003dmsg)"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a74a57a_d25ab729","line":386,"range":{"start_line":385,"start_character":23,"end_line":386,"end_character":51},"updated":"2016-11-26 15:10:19.000000000","message":"Yeah I\u0027m not really sure we should have this check in here, it\u0027s not what we\u0027re hitting in the CI system with the bug reported. I\u0027d like to omit this.","commit_id":"64aebab5aa352d339db639e248eb752a18480642"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"04fbb3629d5d424c1c18a3032b8d380eafdfeff8","unresolved":false,"context_lines":[{"line_number":388,"context_line":"                            \u0027not found\u0027 in msg)):"},{"line_number":389,"context_line":"                        raise exception.DeviceNotFound(message\u003dmsg)"},{"line_number":390,"context_line":"                    else:"},{"line_number":391,"context_line":"                        raise e"},{"line_number":392,"context_line":""},{"line_number":393,"context_line":"                reason \u003d _(\"Unable to detach from guest transient domain.\")"},{"line_number":394,"context_line":"                raise exception.DeviceDetachFailed(device\u003ddevice,"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a74a57a_727643a5","line":391,"updated":"2016-11-26 15:10:19.000000000","message":"Doesn\u0027t this change the call stack?\n\nNormally I use \"with excutils.save_and_reraise_exception() as ctxt:\" with stuff like this. See how it\u0027s handled here:\n\nhttps://review.openstack.org/#/c/374975/2/nova/virt/libvirt/firewall.py@266\n\nThe only difference is in this block we raise DeviceNotFound potentially, but I think that\u0027s fine.","commit_id":"64aebab5aa352d339db639e248eb752a18480642"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"114c46aa95b79a74e7b21a167883c0be8be71738","unresolved":false,"context_lines":[{"line_number":385,"context_line":"                        if errcode \u003d\u003d libvirt.VIR_ERR_OPERATION_FAILED:"},{"line_number":386,"context_line":"                            errmsg \u003d ex.get_error_message()"},{"line_number":387,"context_line":"                            if \u0027not found\u0027 in errmsg:"},{"line_number":388,"context_line":"                                raise exception.DeviceNotFound(message\u003derrmsg)"},{"line_number":389,"context_line":""},{"line_number":390,"context_line":"                reason \u003d _(\"Unable to detach from guest transient domain.\")"},{"line_number":391,"context_line":"                raise exception.DeviceDetachFailed(device\u003ddevice,"}],"source_content_type":"text/x-python","patch_set":5,"id":"5a74a57a_28022670","line":388,"range":{"start_line":388,"start_character":63,"end_line":388,"end_character":77},"updated":"2016-11-29 21:04:28.000000000","message":"I think we want this to be \"device\u003ddevice\".","commit_id":"ed5b064891c5f273c0ce15d411878b21b53b5b07"}]}