)]}'
{"nova/virt/libvirt/migration.py":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"2f7e7455fe07b54b9cecad32ed30a47b37b7b5ff","unresolved":false,"context_lines":[{"line_number":125,"context_line":"        vcpusched.set(\u0027vcpus\u0027, hardware.format_cpu_spec(info.sched_vcpus))"},{"line_number":126,"context_line":"        vcpusched.set(\u0027priority\u0027, str(info.sched_priority))"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    LOG.debug(\u0027Outgoing XML: %s\u0027, etree.tostring(xml_doc, encoding\u003d\u0027unicode\u0027))"},{"line_number":129,"context_line":"    return xml_doc"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_6435dfa7","line":128,"range":{"start_line":128,"start_character":3,"end_line":128,"end_character":78},"updated":"2019-09-02 21:25:35.000000000","message":":)\n\nthis is exactly the logging we need to debug why hugepage migration is broken.\n\nalso you as bad as me for working on your day off\n\nthere are at least two erros\n\nwhen we have multiple numa nodes we get\n\nERROR nova.virt.libvirt.driver [None req-d522d343-98f1-4959-b605-e7ac9818a641 None None] [instance: 9ffe2cab-1c8a-4696-a2f9-ea9d4f05b83b] Live Migration failure: nodeset attribute of hugepages of sizes 2048 and 2048 intersect: libvirt.libvirtError: nodeset attribute of hugepages of sizes 2048 a│\nnd 2048 intersect\n\nif the numa node on the dest is not avaibale on the source we get\n\n ERROR nova.virt.libvirt.driver [None req-2b8be57a-3026-4c97-a8a5-af62ccf257aa None None] [instance: 8ab37278-774a-468c-a5cd-55b4b9c97ec1] Live Migration failure: hugepages: node 1 not found: libvirt.libvirtError: hugepages: node 1 not found","commit_id":"45191cb97d2ecbfbea1eadb29f79077407ba4654"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"157d3e36158705c60ec87eafe209d816ceee8547","unresolved":false,"context_lines":[{"line_number":125,"context_line":"        vcpusched.set(\u0027vcpus\u0027, hardware.format_cpu_spec(info.sched_vcpus))"},{"line_number":126,"context_line":"        vcpusched.set(\u0027priority\u0027, str(info.sched_priority))"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    LOG.debug(\u0027Outgoing XML: %s\u0027, etree.tostring(xml_doc, encoding\u003d\u0027unicode\u0027))"},{"line_number":129,"context_line":"    return xml_doc"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_e4222fba","line":128,"range":{"start_line":128,"start_character":3,"end_line":128,"end_character":78},"in_reply_to":"7faddb67_440ea3e4","updated":"2019-09-02 23:07:59.000000000","message":"Looking at how we generate the pages XML [1], I suspect we end up with something like:\n\n      \u003cpage size\u003d\"2\" unit\u003d\"M\" nodeset\u003d\"1\"/\u003e\n      \u003cpage size\u003d\"2\" unit\u003d\"M\" nodeset\u003d\"2\"/\u003e\n      \u003cpage size\u003d\"2\" unit\u003d\"M\" nodeset\u003d\"3\"/\u003e\n\nand so on. So when we update them all on L121, they intersect and cause libvirt to complain.\n\n[1] https://github.com/openstack/nova/blob/master/nova/virt/libvirt/driver.py#L4956-L4967","commit_id":"45191cb97d2ecbfbea1eadb29f79077407ba4654"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"b46a8c1f17a80fec41aca61456c1b3e749d3fea5","unresolved":false,"context_lines":[{"line_number":125,"context_line":"        vcpusched.set(\u0027vcpus\u0027, hardware.format_cpu_spec(info.sched_vcpus))"},{"line_number":126,"context_line":"        vcpusched.set(\u0027priority\u0027, str(info.sched_priority))"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    LOG.debug(\u0027Outgoing XML: %s\u0027, etree.tostring(xml_doc, encoding\u003d\u0027unicode\u0027))"},{"line_number":129,"context_line":"    return xml_doc"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"7faddb67_440ea3e4","line":128,"range":{"start_line":128,"start_character":3,"end_line":128,"end_character":78},"in_reply_to":"7faddb67_6435dfa7","updated":"2019-09-02 21:30:35.000000000","message":"\u003e \n \u003e this is exactly the logging we need to debug why hugepage migration\n \u003e is broken.\n \u003e \n \u003e also you as bad as me for working on your day off\n\nI normally wouldn\u0027t, but given the pressure on this I\u0027ll make an exception :)\n\n \u003e there are at least two erros\n \u003e \n \u003e when we have multiple numa nodes we get\n \u003e \n \u003e ERROR nova.virt.libvirt.driver [None req-d522d343-98f1-4959-b605-e7ac9818a641\n \u003e None None] [instance: 9ffe2cab-1c8a-4696-a2f9-ea9d4f05b83b] Live\n \u003e Migration failure: nodeset attribute of hugepages of sizes 2048 and\n \u003e 2048 intersect: libvirt.libvirtError: nodeset attribute of\n \u003e hugepages of sizes 2048 a│\n \u003e nd 2048 intersect\n\nThis is weird - I\u0027m not adding any XML, just modifying existing attributes, so I\u0027m not sure what\u0027s up. Let\u0027s see what the new logging turns up.\n\n \u003e if the numa node on the dest is not avaibale on the source we get\n \u003e \n \u003e ERROR nova.virt.libvirt.driver [None req-2b8be57a-3026-4c97-a8a5-af62ccf257aa\n \u003e None None] [instance: 8ab37278-774a-468c-a5cd-55b4b9c97ec1] Live\n \u003e Migration failure: hugepages: node 1 not found: libvirt.libvirtError:\n \u003e hugepages: node 1 not found\n\nYou mean, in the above example, node 1 doesn\u0027t exist on the dest, right? IOW, the source is failing to update something.","commit_id":"45191cb97d2ecbfbea1eadb29f79077407ba4654"}]}