)]}'
{"id":"openstack%2Fdevstack~731723","triplet_id":"openstack%2Fdevstack~master~Ie55de13b34e7df546bf5e2fa0f1fce5a143a72ad","project":"openstack/devstack","branch":"master","hashtags":[],"change_id":"Ie55de13b34e7df546bf5e2fa0f1fce5a143a72ad","subject":"DNM: Add Nova debug logging when detaching devices for focal failures","status":"ABANDONED","created":"2020-05-29 10:36:07.000000000","updated":"2020-07-16 11:17:32.000000000","total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"98e9f23f9f53fd0f815a0306af8b43aa86abd7c9","_number":731723,"virtual_id_number":731723,"owner":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"date":"2020-05-29 13:24:46.000000000","_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},{"tag":"autogenerated:zuul:check","value":1,"date":"2020-05-29 12:19:54.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":0},"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Review-Priority":{"all":[{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},{"value":0,"permitted_voting_range":{"min":0,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Branch Freeze"," 0":"No Priority","+1":"Important Change","+2":"Gate Blocker Fix / Urgent Change"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-05-29 12:19:54.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"c2b782ddde7e7abbbfb4482190c6d58108f9be05","author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"date":"2020-05-29 10:36:07.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"c6dd859291abe5d005fc1a99384fa04c235f725c","author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"date":"2020-05-29 10:43:52.000000000","message":"Patch Set 1:\n\nUsing this change to debug device detach failures seen in Ib4416dc2f5e003fd770f5240a8f78213c56af8e6 logs below:\n\nhttps://zuul.opendev.org/t/openstack/build/a5aca94014224f1fa1acbb64e3f72f54\n\nSpecifically n-cpu appears to somehow miss libvirtd reporting that it no longer can find the device once it has been correctly detached:\n\n\n# Successful detach from the persistent config\n\n655416 2020-05-28 17:46:03.802+0000: 57169: debug : qemuMonitorDelDevice:2850 : mon:0x7ffb5408bde0 vm:0x7ffb5c036de0 fd:35\n655417 2020-05-28 17:46:03.802+0000: 57169: info : qemuMonitorSend:993 : QEMU_MONITOR_SEND_MSG: mon\u003d0x7ffb5408bde0 msg\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-367\"}^M\n655418  fd\u003d-1                                                                          \n655419 2020-05-28 17:46:03.802+0000: 57165: info : qemuMonitorIOWrite:450 : QEMU_MONITOR_IO_WRITE: mon\u003d0x7ffb5408bde0 buf\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-367\"}^M\n655420  len\u003d79 ret\u003d79 errno\u003d0                                                          \n655421 2020-05-28 17:46:03.803+0000: 57165: debug : qemuMonitorJSONIOProcessLine:220 : Line [{\"return\": {}, \"id\": \"libvirt-367\"}]\n655422 2020-05-28 17:46:03.803+0000: 57165: info : qemuMonitorJSONIOProcessLine:239 : QEMU_MONITOR_RECV_REPLY: mon\u003d0x7ffb5408bde0 reply\u003d{\"return\": {}, \"id\": \"libvirt-367\"}\n\n\n# Successful detach from the live config\n\n655445 2020-05-28 17:46:08.814+0000: 57167: debug : qemuMonitorDelDevice:2850 : mon:0x7ffb5408bde0 vm:0x7ffb5c036de0 fd:35\n655446 2020-05-28 17:46:08.814+0000: 57167: info : qemuMonitorSend:993 : QEMU_MONITOR_SEND_MSG: mon\u003d0x7ffb5408bde0 msg\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-369\"}^M\n655447  fd\u003d-1                                                                          \n655448 2020-05-28 17:46:08.814+0000: 57165: info : qemuMonitorIOWrite:450 : QEMU_MONITOR_IO_WRITE: mon\u003d0x7ffb5408bde0 buf\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-369\"}^M\n655449  len\u003d79 ret\u003d79 errno\u003d0                                                          \n655450 2020-05-28 17:46:08.815+0000: 57165: debug : qemuMonitorJSONIOProcessLine:220 : Line [{\"return\": {}, \"id\": \"libvirt-369\"}]\n655451 2020-05-28 17:46:08.815+0000: 57165: info : qemuMonitorJSONIOProcessLine:239 : QEMU_MONITOR_RECV_REPLY: mon\u003d0x7ffb5408bde0 reply\u003d{\"return\": {}, \"id\": \"libvirt-369\"}\n\n# Failure to find the device not caught by n-cpu\n\n655483 2020-05-28 17:46:15.824+0000: 57166: debug : qemuMonitorDelDevice:2850 : mon:0x7ffb5408bde0 vm:0x7ffb5c036de0 fd:35\n655484 2020-05-28 17:46:15.824+0000: 57166: info : qemuMonitorSend:993 : QEMU_MONITOR_SEND_MSG: mon\u003d0x7ffb5408bde0 msg\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-371\"}^M\n655485  fd\u003d-1                                                                          \n655486 2020-05-28 17:46:15.824+0000: 57165: info : qemuMonitorIOWrite:450 : QEMU_MONITOR_IO_WRITE: mon\u003d0x7ffb5408bde0 buf\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-371\"}^M\n655487  len\u003d79 ret\u003d79 errno\u003d0                                                          \n655488 2020-05-28 17:46:15.825+0000: 57165: debug : qemuMonitorJSONIOProcessLine:220 : Line [{\"id\": \"libvirt-371\", \"error\": {\"class\": \"DeviceNotFound\", \"desc\": \"Device \u0027virtio-disk1\u0027 not found\"}}]\n655489 2020-05-28 17:46:15.825+0000: 57165: info : qemuMonitorJSONIOProcessLine:239 : QEMU_MONITOR_RECV_REPLY: mon\u003d0x7ffb5408bde0 reply\u003d{\"id\": \"libvirt-371\", \"error\": {\"class\": \"DeviceNotFound\", \"desc\": \"Device \u0027virtio-disk1\u0027 not found\"}}\n655490 2020-05-28 17:46:15.825+0000: 57166: debug : qemuDomainObjExitMonitorInternal:9892 : Exited monitor (mon\u003d0x7ffb5408bde0 vm\u003d0x7ffb5c036de0 name\u003dinstance-0000007b)","accounts_in_message":[],"_revision_number":1},{"id":"4d904d2e4ace3188eb8d56126d790c094716705f","author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"date":"2020-05-29 10:47:04.000000000","message":"Patch Set 1:\n\nNote focal is using libvirt 6.0.0 : \n\nhttps://zuul.opendev.org/t/openstack/build/a5aca94014224f1fa1acbb64e3f72f54/log/controller/logs/dpkg-l.txt#509","accounts_in_message":[],"_revision_number":1},{"id":"ba1d5217bc1fc44170a3040e52560d230d37cbec","author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"date":"2020-05-29 10:49:31.000000000","message":"Patch Set 1:\n\nhttps://github.com/libvirt/libvirt/commit/229a0358f0678d414b765e533b53c74e40d5189c maybe?","accounts_in_message":[],"_revision_number":1},{"id":"525094d9ff59738227cb3229cde83bfd7c7a4fc4","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-05-29 12:19:54.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\n\n- grenade https://zuul.opendev.org/t/openstack/build/f9563e071aa94f47b6e21bbed87778e5 : SUCCESS in 58m 30s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/ec37c041ef1a42df867c0ee66ca59a7d : SUCCESS in 1h 16m 56s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/e9f8f204b8d3496fbd73ce62fe989ebb : SUCCESS in 6m 52s\n- devstack https://zuul.opendev.org/t/openstack/build/f2653e023ed44178bf1df9da965e0974 : SUCCESS in 18m 32s\n- devstack-ipv6 https://zuul.opendev.org/t/openstack/build/39300850bfbd451288984b3b1f84b990 : SUCCESS in 31m 43s\n- devstack-platform-opensuse-15 https://zuul.opendev.org/t/openstack/build/edf18a83556f4ac3ad1a971277fa9692 : SUCCESS in 1h 20m 55s\n- devstack-platform-fedora-latest https://zuul.opendev.org/t/openstack/build/3ed24d45d46247e8b117aeaa0e6aa666 : SUCCESS in 1h 02m 13s (non-voting)\n- devstack-platform-centos-8 https://zuul.opendev.org/t/openstack/build/c456ad5cc16d44f7a1ba035b16a2898d : SUCCESS in 1h 32m 46s (non-voting)\n- devstack-platform-focal https://zuul.opendev.org/t/openstack/build/603604367de043699977cb0d6107359c : FAILURE in 1h 19m 45s (non-voting)\n- devstack-multinode https://zuul.opendev.org/t/openstack/build/973c8304e8f64a74a0f402af9c1e640e : SUCCESS in 36m 06s\n- devstack-unit-tests https://zuul.opendev.org/t/openstack/build/624c50341b544133a6ddce89afba41ef : SUCCESS in 4m 19s\n- openstack-tox-bashate https://zuul.opendev.org/t/openstack/build/78ee047a797a4fd392e17ae6ba1ce125 : SUCCESS in 3m 23s\n- ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa https://zuul.opendev.org/t/openstack/build/65c92d356fee4fe894fb45b3af56d182 : SUCCESS in 53m 21s (non-voting)\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/5f400ea88cc84f1c87ffcce3aa940a22 : SUCCESS in 31m 25s (non-voting)\n- neutron-grenade-multinode https://zuul.opendev.org/t/openstack/build/9c5cb9eb79a4482f8dfc73854be5135a : SUCCESS in 58m 52s\n- neutron-tempest-linuxbridge https://zuul.opendev.org/t/openstack/build/92cfec65fb6347ea93490297c84325c1 : SUCCESS in 45m 32s\n- tempest-multinode-full-py3 https://zuul.opendev.org/t/openstack/build/67bd19fbfdc9424caef03710d6f81066 : SUCCESS in 1h 41m 24s (non-voting)\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/ddb3424a44f646098b6eaff323b2ef6a : SUCCESS in 47m 47s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/9e38c90699b84570bb693b7c11764715 : SUCCESS in 54m 25s\n- nova-live-migration https://zuul.opendev.org/t/openstack/build/82e2d0f7d0474eac90419a7faf312ed7 : SUCCESS in 51m 59s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"44acf2ae44450f03169db5299bd99b68d509e79a","author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"date":"2020-05-29 13:24:18.000000000","message":"Patch Set 1:\n\nStill seeing this on the latest test run without the not found exception being logged. I think there are two issues here.\n\n1. n-cpu\u0027s polling is incorrectly finding the device somehow, I\u0027m adding logging to look into this more now.\n\n2. libvirtd / python-libvirt isn\u0027t raising the libvirt exception to n-cpu when it isn\u0027t able to find the device after it is removed from the live config.\n\n\nhttps://dfbc42988c24d4bd13be-34abc3b080ea1b24d7f61aad28452048.ssl.cf1.rackcdn.com/731723/1/check/devstack-platform-focal/6036043/testr_results.html\n\ntempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest\n\nmon\u003d0x7fe76c158400 vm\u003d0x7fe754006020 name\u003dinstance-00000075 uuid\u003dbe03a14a-97f1-4bfc-bc02-08f77c4ace6f\n\nhttps://dfbc42988c24d4bd13be-34abc3b080ea1b24d7f61aad28452048.ssl.cf1.rackcdn.com/731723/1/check/devstack-platform-focal/6036043/controller/logs/libvirt/libvirtd_log.txt\n\nhttps://dfbc42988c24d4bd13be-34abc3b080ea1b24d7f61aad28452048.ssl.cf1.rackcdn.com/731723/1/check/devstack-platform-focal/6036043/controller/logs/screen-n-cpu.txt\n\n# Device vdb successfully removed from the persistent config\n\n48931 May 29 11:35:37.041738 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: DEBUG nova.virt.libvirt.guest [None req-63e7d467-0179-4612-af8c-1975f779ece7 tempest-ServerStableDeviceRescueTest-1936931408 tempest-ServerStableDeviceRescueTest-1936931408] detach device xml: \u003cdisk type\u003d\"block\" device\u003d\"disk\"\u003e\n48932 May 29 11:35:37.041738 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003cdriver name\u003d\"qemu\" type\u003d\"raw\" cache\u003d\"none\" io\u003d\"native\"/\u003e \n48933 May 29 11:35:37.041738 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003csource dev\u003d\"/dev/sdc\"/\u003e\n48934 May 29 11:35:37.041738 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003ctarget dev\u003d\"vdb\" bus\u003d\"virtio\"/\u003e \n48935 May 29 11:35:37.041738 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003cserial\u003eb31f7064-104e-4472-9c7c-f3c8d7fb9ff9\u003c/serial\u003e\n48936 May 29 11:35:37.041738 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003caddress type\u003d\"pci\" domain\u003d\"0x0000\" bus\u003d\"0x00\" slot\u003d\"0x07\" function\u003d\"0x0\"/\u003e\n48937 May 29 11:35:37.041738 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: \u003c/disk\u003e\n\n\n649421 2020-05-29 11:35:37.044+0000: 56768: debug : qemuMonitorDelDevice:2850 : mon:0x7fe768114960 vm:0x7fe754006020 fd:38\n649422 2020-05-29 11:35:37.044+0000: 56768: info : qemuMonitorSend:993 : QEMU_MONITOR_SEND_MSG: mon\u003d0x7fe768114960 msg\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-367\"}^M\n649423  fd\u003d-1                                                                          \n649424 2020-05-29 11:35:37.044+0000: 56766: info : qemuMonitorIOWrite:450 : QEMU_MONITOR_IO_WRITE: mon\u003d0x7fe768114960 buf\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-367\"}^M\n649425  len\u003d79 ret\u003d79 errno\u003d0                                                          \n649426 2020-05-29 11:35:37.045+0000: 56766: debug : qemuMonitorJSONIOProcessLine:220 : Line [{\"return\": {}, \"id\": \"libvirt-367\"}]\n\n\n# Device vdb successfully removed from the live config\n\n49231 May 29 11:35:42.059604 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: DEBUG oslo.service.loopingcall [None req-63e7d467-0179-4612-af8c-1975f779ece7 tempest-ServerStableDeviceRescueTest-1936931408 tempest-ServerStableDeviceRescueTest-1936931408] Waiting for function nova.virt.libvirt.guest.Guest      .detach_device_with_retry.\u003clocals\u003e._do_wait_and_retry_detach to return. {{(pid\u003d82434) func /usr/local/lib/python3.8/dist-packages/oslo_service/loopingcall.py:435}}\n49232 May 29 11:35:42.062071 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: DEBUG nova.virt.libvirt.guest [-] detach device xml: \u003cdisk type\u003d\"block\" device\u003d\"disk\"\u003e\n49233 May 29 11:35:42.062071 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003cdriver name\u003d\"qemu\" type\u003d\"raw\" cache\u003d\"none\" io\u003d\"native\"/\u003e\n49234 May 29 11:35:42.062071 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003csource dev\u003d\"/dev/sdc\"/\u003e\n49235 May 29 11:35:42.062071 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003ctarget dev\u003d\"vdb\" bus\u003d\"virtio\"/\u003e\n49236 May 29 11:35:42.062071 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003cserial\u003eb31f7064-104e-4472-9c7c-f3c8d7fb9ff9\u003c/serial\u003e\n49237 May 29 11:35:42.062071 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003caddress type\u003d\"pci\" domain\u003d\"0x0000\" bus\u003d\"0x00\" slot\u003d\"0x07\" function\u003d\"0x0\"/\u003e\n49238 May 29 11:35:42.062071 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: \u003c/disk\u003e\n\n\n652942 2020-05-29 11:35:42.063+0000: 56770: debug : qemuMonitorDelDevice:2850 : mon:0x7fe768114960 vm:0x7fe754006020 fd:38\n652943 2020-05-29 11:35:42.063+0000: 56770: info : qemuMonitorSend:993 : QEMU_MONITOR_SEND_MSG: mon\u003d0x7fe768114960 msg\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-369\"}^M\n652944  fd\u003d-1                                                                          \n652945 2020-05-29 11:35:42.063+0000: 56766: info : qemuMonitorIOWrite:450 : QEMU_MONITOR_IO_WRITE: mon\u003d0x7fe768114960 buf\u003d{\"execute\":\"device_del\",\"arguments\":{\"id\":\"virtio-disk1\"},\"id\":\"libvirt-369\"}^M\n652946  len\u003d79 ret\u003d79 errno\u003d0                                                          \n652947 2020-05-29 11:35:42.064+0000: 56766: debug : qemuMonitorJSONIOProcessLine:220 : Line [{\"return\": {}, \"id\": \"libvirt-369\"}]\n\n# Device vdb no longer attached but n-cpu still retrying?\n\n49269 May 29 11:35:49.074772 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: DEBUG oslo.service.loopingcall [-] Invoking nova.virt.libvirt.guest.Guest.detach_device_with_retry.\u003clocals\u003e._do_wait_and_retry_detach; retry count is 1. {{(pid\u003d82434) _func /usr/local/lib/python3.8/dist-packages/oslo_service/      loopingcall.py:403}}\n49270 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: DEBUG nova.virt.libvirt.guest [-] detach device xml: \u003cdisk type\u003d\"block\" device\u003d\"disk\"\u003e\n49271 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003cdriver name\u003d\"qemu\" type\u003d\"raw\" cache\u003d\"none\" io\u003d\"native\"/\u003e\n49272 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003csource dev\u003d\"/dev/sdc\"/\u003e\n49273 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003ctarget dev\u003d\"vdb\" bus\u003d\"virtio\"/\u003e\n49274 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003cserial\u003eb31f7064-104e-4472-9c7c-f3c8d7fb9ff9\u003c/serial\u003e\n49275 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:   \u003caddress type\u003d\"pci\" domain\u003d\"0x0000\" bus\u003d\"0x00\" slot\u003d\"0x07\" function\u003d\"0x0\"/\u003e\n49276 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]: \u003c/disk\u003e\n49277 May 29 11:35:49.077234 ubuntu-focal-limestone-regionone-0016857542 nova-compute[82434]:  {{(pid\u003d82434) detach_device /opt/stack/nova/nova/virt/libvirt/guest.py:476}}\n\n\n653026 2020-05-29 11:35:49.077+0000: 56767: debug : qemuMonitorDelDevice:2850 : mon:0x7fe768114960 vm:0x7fe754006020 fd:38\n653027 2020-05-29 11:35:49.077+0000: 56767: info : qemuMonitorSend:993 : QEMU_MONITOR_SEND_MSG: mon\u003d0x7fe768114960 msg\u003d{\"execute\":\"device_del\",\"arguments\":{\"id       \":\"virtio-disk1\"},\"id\":\"libvirt-371\"}^M\n653028  fd\u003d-1                                                                          \n653029 2020-05-29 11:35:49.077+0000: 56766: info : qemuMonitorIOWrite:450 : QEMU_MONITOR_IO_WRITE: mon\u003d0x7fe768114960 buf\u003d{\"execute\":\"device_del\",\"arguments\":{       \"id\":\"virtio-disk1\"},\"id\":\"libvirt-371\"}^M\n653030  len\u003d79 ret\u003d79 errno\u003d0                                                          \n653031 2020-05-29 11:35:49.080+0000: 56766: debug : qemuMonitorJSONIOProcessLine:220 : Line [{\"id\": \"libvirt-371\", \"error\": {\"class\": \"DeviceNotFound\", \"desc\": \"Device \u0027virtio-disk1\u0027 not found\"}}]\n\n# This then repeats AFAICT\n\n656685 2020-05-29 11:35:58.092+0000: 56768: debug : qemuMonitorDelDevice:2850 : mon:0x7fe768114960 vm:0x7fe754006020 fd:38\n656686 2020-05-29 11:35:58.092+0000: 56768: info : qemuMonitorSend:993 : QEMU_MONITOR_SEND_MSG: mon\u003d0x7fe768114960 msg\u003d{\"execute\":\"device_del\",\"arguments\":{\"id       \":\"virtio-disk1\"},\"id\":\"libvirt-373\"}^M\n656687  fd\u003d-1                                                                          \n656688 2020-05-29 11:35:58.092+0000: 56766: info : qemuMonitorIOWrite:450 : QEMU_MONITOR_IO_WRITE: mon\u003d0x7fe768114960 buf\u003d{\"execute\":\"device_del\",\"arguments\":{       \"id\":\"virtio-disk1\"},\"id\":\"libvirt-373\"}^M \n656689  len\u003d79 ret\u003d79 errno\u003d0                                                           \n656690 2020-05-29 11:35:58.093+0000: 56766: debug : qemuMonitorJSONIOProcessLine:220 : Line [{\"id\": \"libvirt-373\", \"error\": {\"class\": \"DeviceNotFound\", \"desc\":        \"Device \u0027virtio-disk1\u0027 not found\"}}]","accounts_in_message":[],"_revision_number":1},{"id":"d1b3d3ef2e79b476328c91adba086577980fde35","author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"date":"2020-05-29 13:24:46.000000000","message":"Patch Set 1:\n\nrecheck","accounts_in_message":[],"_revision_number":1},{"id":"c352bdff364a0f380cdff9b56bfdf4e3fa15b3dc","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-05-29 15:11:42.000000000","message":"Patch Set 1:\n\nBuild succeeded (check pipeline).\n\n- grenade https://zuul.opendev.org/t/openstack/build/4e395bb8794a48248f550b38665b9f19 : SUCCESS in 1h 05m 22s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/554a67e1220545908038d4537953e105 : SUCCESS in 1h 32m 06s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/d07777c9870b419f8a7c912ad9ecd10d : SUCCESS in 5m 12s\n- devstack https://zuul.opendev.org/t/openstack/build/78a4a6809af94281b0eeb661db8f6a0e : SUCCESS in 28m 49s\n- devstack-ipv6 https://zuul.opendev.org/t/openstack/build/a08691c7833042628f0daf4d391d94ab : SUCCESS in 29m 56s\n- devstack-platform-opensuse-15 https://zuul.opendev.org/t/openstack/build/07119b80dc4e4be08d7e1b60a32ce0f6 : SUCCESS in 1h 37m 18s\n- devstack-platform-fedora-latest https://zuul.opendev.org/t/openstack/build/7518ea27af6e41da86ec25f9cf66c636 : SUCCESS in 1h 40m 35s (non-voting)\n- devstack-platform-centos-8 https://zuul.opendev.org/t/openstack/build/0c8f411a09e14372875b20f61ab58d7a : SUCCESS in 1h 31m 40s (non-voting)\n- devstack-platform-focal https://zuul.opendev.org/t/openstack/build/fbed1b323a3943a7989d5a3f9bf39321 : FAILURE in 1h 14m 35s (non-voting)\n- devstack-multinode https://zuul.opendev.org/t/openstack/build/4de85ace113242c0b8c07f43fabed1ad : SUCCESS in 39m 26s\n- devstack-unit-tests https://zuul.opendev.org/t/openstack/build/8df8e5ca0ad343f2a6c3dc81d4e2a4fc : SUCCESS in 4m 00s\n- openstack-tox-bashate https://zuul.opendev.org/t/openstack/build/d22980243e3f4be99fdae629e0224299 : SUCCESS in 4m 55s\n- ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa https://zuul.opendev.org/t/openstack/build/e8fd57abc9d345db824021bf32afe554 : SUCCESS in 57m 30s (non-voting)\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/a1021218c3254b4b821340c61d6f204f : SUCCESS in 40m 26s (non-voting)\n- neutron-grenade-multinode https://zuul.opendev.org/t/openstack/build/56331a8cc7b240ecb3e8877cfd48dc9f : SUCCESS in 1h 12m 17s\n- neutron-tempest-linuxbridge https://zuul.opendev.org/t/openstack/build/342ea608eb2f4a989d3db6521ebfc291 : SUCCESS in 1h 13m 24s\n- tempest-multinode-full-py3 https://zuul.opendev.org/t/openstack/build/9b1b0a4d6496461b8ccf88319f44d20b : SUCCESS in 1h 36m 00s (non-voting)\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/787d69fdd6264b68b8adf13ec83447ba : SUCCESS in 54m 41s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/d14dce988aee49fdbcfc656871d1627f : SUCCESS in 57m 17s\n- nova-live-migration https://zuul.opendev.org/t/openstack/build/aa791df8765e4dc583f8bcd3ca5369ca : SUCCESS in 42m 07s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"98e9f23f9f53fd0f815a0306af8b43aa86abd7c9","author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"date":"2020-07-16 11:17:32.000000000","message":"Abandoned","accounts_in_message":[],"_revision_number":1}],"current_revision_number":1,"current_revision":"6366665bbbde21a437ae9dbad499047ef738ebee","revisions":{"6366665bbbde21a437ae9dbad499047ef738ebee":{"kind":"REWORK","_number":1,"created":"2020-05-29 10:36:07.000000000","uploader":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"ref":"refs/changes/23/731723/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/devstack","ref":"refs/changes/23/731723/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/devstack refs/changes/23/731723/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/devstack refs/changes/23/731723/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/devstack refs/changes/23/731723/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/devstack refs/changes/23/731723/1"}}},"commit":{"parents":[{"commit":"e6ded2f04c87c5a00aeb35046d78b2c4fa143811","subject":"Make devstack run on focal (Ubuntu LTS 20.04)","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/devstack/commit/e6ded2f04c87c5a00aeb35046d78b2c4fa143811"}]}],"author":{"name":"Lee Yarwood","email":"lyarwood@redhat.com","date":"2020-05-29 10:34:57.000000000","tz":60},"committer":{"name":"Lee Yarwood","email":"lyarwood@redhat.com","date":"2020-05-29 10:34:57.000000000","tz":60},"subject":"DNM: Add Nova debug logging when detaching devices for focal failures","message":"DNM: Add Nova debug logging when detaching devices for focal failures\n\nChange-Id: Ie55de13b34e7df546bf5e2fa0f1fce5a143a72ad\nDepends-On: https://review.opendev.org/731721\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/devstack/commit/6366665bbbde21a437ae9dbad499047ef738ebee"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/devstack/commit/6366665bbbde21a437ae9dbad499047ef738ebee"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}