)]}'
{"specs/rocky/approved/libvirt-qemu-numa-support.rst":[{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"d5176e158ee1db0d742b951a5c0a2bd4a8f5f4da","unresolved":false,"context_lines":[{"line_number":23,"context_line":"KVM backend."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Libvirt driver currently doesn\u0027t support those NUMA features for QEMU"},{"line_number":26,"context_line":"backend, because it doesn’t support vcpu pinning to indivisual physical"},{"line_number":27,"context_line":"cpus and currently nova libvirt driver uses that cpu pinning implicitly"},{"line_number":28,"context_line":"if NUMA related features are specified by users."},{"line_number":29,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_d339efd6","line":26,"range":{"start_line":26,"start_character":52,"end_line":26,"end_character":62},"updated":"2018-02-01 11:35:29.000000000","message":"individual","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"d5176e158ee1db0d742b951a5c0a2bd4a8f5f4da","unresolved":false,"context_lines":[{"line_number":23,"context_line":"KVM backend."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Libvirt driver currently doesn\u0027t support those NUMA features for QEMU"},{"line_number":26,"context_line":"backend, because it doesn’t support vcpu pinning to indivisual physical"},{"line_number":27,"context_line":"cpus and currently nova libvirt driver uses that cpu pinning implicitly"},{"line_number":28,"context_line":"if NUMA related features are specified by users."},{"line_number":29,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_73349bde","line":26,"range":{"start_line":26,"start_character":36,"end_line":26,"end_character":40},"updated":"2018-02-01 11:35:29.000000000","message":"vCPU","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"d5176e158ee1db0d742b951a5c0a2bd4a8f5f4da","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Libvirt driver currently doesn\u0027t support those NUMA features for QEMU"},{"line_number":26,"context_line":"backend, because it doesn’t support vcpu pinning to indivisual physical"},{"line_number":27,"context_line":"cpus and currently nova libvirt driver uses that cpu pinning implicitly"},{"line_number":28,"context_line":"if NUMA related features are specified by users."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"However, QEMU backend potentially has capability to support NUMA"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_53539f17","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":4},"updated":"2018-02-01 11:35:29.000000000","message":"CPUs","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"d5176e158ee1db0d742b951a5c0a2bd4a8f5f4da","unresolved":false,"context_lines":[{"line_number":29,"context_line":""},{"line_number":30,"context_line":"However, QEMU backend potentially has capability to support NUMA"},{"line_number":31,"context_line":"related features such as Hugepages and NUMA Topology. This spec enables"},{"line_number":32,"context_line":"libvirt/QEMU driver to support these 2 features without pinning cpus."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"FYI, the list of Openstack NUMA related features;"},{"line_number":35,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_13599733","line":32,"range":{"start_line":32,"start_character":37,"end_line":32,"end_character":38},"updated":"2018-02-01 11:35:29.000000000","message":"two","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"d5176e158ee1db0d742b951a5c0a2bd4a8f5f4da","unresolved":false,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"- Hugepages:"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"  - `hw:mem_page_size\u003d[small|large|any|\u003cpage_size\u003e]`"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"- NUMA topology:"},{"line_number":41,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_d33ecfb6","line":38,"range":{"start_line":38,"start_character":4,"end_line":38,"end_character":52},"updated":"2018-02-01 11:35:29.000000000","message":"You need double backticks (``literal``). Same for all the below","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"d5176e158ee1db0d742b951a5c0a2bd4a8f5f4da","unresolved":false,"context_lines":[{"line_number":58,"context_line":"  - `hw:emulator_threads_policy\u003d[share|isolate]`"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Since CPU pinning policy, CPU real-time policy, and emulator threads"},{"line_number":61,"context_line":"policy all depend on indivisual cpu pinning, we can\u0027t support them"},{"line_number":62,"context_line":"with libvirt/QEMU driver in principle."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_f36e6bc7","line":61,"range":{"start_line":61,"start_character":21,"end_line":61,"end_character":31},"updated":"2018-02-01 11:35:29.000000000","message":"individual","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"d5176e158ee1db0d742b951a5c0a2bd4a8f5f4da","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Use Cases"},{"line_number":76,"context_line":"---------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"NFV cloud developers and operators, who can\u0027t prepare enviroment with"},{"line_number":79,"context_line":"KVM backend (e.g. VirtualBox on the laptop) uses this spec when they"},{"line_number":80,"context_line":"try NFV products (e.g. DPDK vswitches) and see how they behave via"},{"line_number":81,"context_line":"OpenStack."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":84,"context_line":"cpu pinning, NUMA topology for instances becomes nothing to do with"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5f93b717_13ce77b3","line":81,"range":{"start_line":78,"start_character":0,"end_line":81,"end_character":10},"updated":"2018-02-01 11:35:29.000000000","message":"Hmm, this is a soft argument. Nested virt is a possibility on any Intel processor with vt-x and any AMD CPU with AMD-V (which is pretty much all of them, afaik). Given that these a features that are intended for high performance use cases, \"validating\" them on hardware that doesn\u0027t provide the necessary capabilities seems a little odd, don\u0027t you think?\n\nA more sensible use case, which sean-k-mooney has raised at least once, is to allow us to test this functionality in the upstream gate and reduce the need for the Intel NFV CI. It\u0027s not a perfect solution, but being able to validate that we\u0027re not breaking the nova-libvirt interaction at least would be a good thing, IMO. Even with that though, hmm...","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"eff30e2d25e3592dc742504e5f6e993d97433b00","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Use Cases"},{"line_number":76,"context_line":"---------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"NFV cloud developers and operators, who can\u0027t prepare enviroment with"},{"line_number":79,"context_line":"KVM backend (e.g. VirtualBox on the laptop) uses this spec when they"},{"line_number":80,"context_line":"try NFV products (e.g. DPDK vswitches) and see how they behave via"},{"line_number":81,"context_line":"OpenStack."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":84,"context_line":"cpu pinning, NUMA topology for instances becomes nothing to do with"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_ce7a6587","line":81,"range":{"start_line":78,"start_character":0,"end_line":81,"end_character":10},"in_reply_to":"3fa0c359_b61bb871","updated":"2018-02-05 12:02:41.000000000","message":"actully anecdotally for stablity reasons, i have heard that using nested virt without setting the cpu model to host-passthough is ill advised. i have never had issues with nested virt but again i always use host-passthrough so my sample size of one does not help with there assertion however i know its diabled in the gate because of issues that were encourterd. \n\ni do think there are other usecase beyond ci for this support. for example openstack allows the use of guest vms that do not match the host architecture. you simply have to set the right metadata on the image. so if you are on x86 and are virtualising arm or power you cannont use kvm as an acclerator as are result you have to fall back to qemu but that does not mean you can optimize its deployment with numa topology or hugepage request.","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"5ce9eddfe1226d713a11e8e4e6cbea2ba7684d23","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Use Cases"},{"line_number":76,"context_line":"---------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"NFV cloud developers and operators, who can\u0027t prepare enviroment with"},{"line_number":79,"context_line":"KVM backend (e.g. VirtualBox on the laptop) uses this spec when they"},{"line_number":80,"context_line":"try NFV products (e.g. DPDK vswitches) and see how they behave via"},{"line_number":81,"context_line":"OpenStack."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":84,"context_line":"cpu pinning, NUMA topology for instances becomes nothing to do with"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_b61bb871","line":81,"range":{"start_line":78,"start_character":0,"end_line":81,"end_character":10},"in_reply_to":"5f93b717_13ce77b3","updated":"2018-02-05 03:07:56.000000000","message":"\u003e \"validating\" them on hardware that doesn\u0027t provide the necessary capabilities seems a little odd, don\u0027t you think?\n\nI think so too. I didn\u0027t mean \"validate\" to use. I meant \"try and see\" for simple evaluation or development. Whatever the reason may be software or hardware, there are environments that can\u0027t enable nested virt. For those environments, we can just say \"try another environment.\" But I don\u0027t think that is OSSwise. Since nova is downstream of hypervisors, we should make effort to support as many functions as long as the hypervisor support that function. And it is not us (developers) but users who choose what to use.\n\nAnyway, I updated this part. And if you put your \"hmmm\" points into words, I\u0027m okay to write them down on impact chapters.","commit_id":"fa686328df2d780ac4bfdac96490fcf762bb7d64"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"81016c0ff26bc7b6780296a331556aa036bf5ad3","unresolved":false,"context_lines":[{"line_number":40,"context_line":"- NUMA topology:"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of numa nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_55dc9973","line":43,"range":{"start_line":43,"start_character":22,"end_line":43,"end_character":35},"updated":"2018-02-05 16:58:11.000000000","message":"number of pCPUs for NUMA node N","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"eb47da3c89e933ae4ff695eb5c8def8125e1fc28","unresolved":false,"context_lines":[{"line_number":40,"context_line":"- NUMA topology:"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of numa nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_8c957a89","line":43,"range":{"start_line":43,"start_character":22,"end_line":43,"end_character":35},"in_reply_to":"3fa0c359_55dc9973","updated":"2018-02-13 11:59:36.000000000","message":"Done","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"81016c0ff26bc7b6780296a331556aa036bf5ad3","unresolved":false,"context_lines":[{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of numa nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_f5d60550","line":44,"range":{"start_line":44,"start_character":20,"end_line":44,"end_character":30},"updated":"2018-02-05 16:58:11.000000000","message":"amount of memory for NUMA node N","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"eb47da3c89e933ae4ff695eb5c8def8125e1fc28","unresolved":false,"context_lines":[{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of numa nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003ccpu number of numa node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_2c90e694","line":44,"range":{"start_line":44,"start_character":20,"end_line":44,"end_character":30},"in_reply_to":"3fa0c359_f5d60550","updated":"2018-02-13 11:59:36.000000000","message":"Done","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"81016c0ff26bc7b6780296a331556aa036bf5ad3","unresolved":false,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":".. note::"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"    Because CPU topology feature doesn\u0027t depend on NUMA topology,"},{"line_number":67,"context_line":"    it already works with libvirt/QEMU driver."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"    - CPU topology:"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"      - ``hw:cpu_sockets\u003d\u003cnumber of sockets for the guest\u003e``"},{"line_number":72,"context_line":"      - ``hw:cpu_cores\u003d\u003cnumber of cores per socket for the guest\u003e``"},{"line_number":73,"context_line":"      - ``hw:cpu_threads\u003d\u003cnumber of threads per core for the guest\u003e``"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Use Cases"},{"line_number":76,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_d52d0947","line":73,"range":{"start_line":66,"start_character":0,"end_line":73,"end_character":69},"updated":"2018-02-05 16:58:11.000000000","message":"Frankly, I have always found it incongruent that we support a \"CPU topology\" that looks very NUMA-like but isn\u0027t actually related to NUMA topologies. I mean, I understand that NUMA is primarily about the closeness of a particular socket/core to a bank of RAM, the fact that we have a cpu_sockets, cpu_cores and cpu_threads configuration/metadata option is super confusing.","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"db88e16c2f4c00223282bb967aec5cd61eb0fcbc","unresolved":false,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":".. note::"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"    Because CPU topology feature doesn\u0027t depend on NUMA topology,"},{"line_number":67,"context_line":"    it already works with libvirt/QEMU driver."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"    - CPU topology:"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"      - ``hw:cpu_sockets\u003d\u003cnumber of sockets for the guest\u003e``"},{"line_number":72,"context_line":"      - ``hw:cpu_cores\u003d\u003cnumber of cores per socket for the guest\u003e``"},{"line_number":73,"context_line":"      - ``hw:cpu_threads\u003d\u003cnumber of threads per core for the guest\u003e``"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Use Cases"},{"line_number":76,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_cefeda04","line":73,"range":{"start_line":66,"start_character":0,"end_line":73,"end_character":69},"in_reply_to":"3fa0c359_d52d0947","updated":"2018-02-05 18:30:40.000000000","message":"you are correct that numa affinity and cpu pinning have noting to do with each other. originally when intel proposed our version they were completely decoupled but in icehouse the choice was made to go with redhats proposal that combind that deeply coupled numa,hugepagnes and cpuinning together.\nredhats propsoal also move it from a host configuration option to a flavor extra spec which has pros and cons.\n\nnuma is purely related to memory and nothing else.\nmost modern intel xeons have 2 numa nodes per socket.\n(haswell+ with more then 10 cores.)\nsimilarly anything before sandybridge likely had a singel numa node  regardless of how many socket you have.\n\ncpu toplolgy, numa affintiy(of memory or hostdevices), memory backing and cpu pinning should be usable entirely independently but in the libvirt direver they are not.\n\nif you look at the hyperv driver they have a flavor extra spec which allow you to turn off stict numa efficency for the memory to cpus. \n\nopenstack flavor set \u003cFLAVOR\u003e --property hw:numa_mempolicy\u003dpreferred\n\nin the libvirt driver this is hardcoded to strict, libvirt dose not require stict affinity for the memory but we over optimise performance at the expense of wasting capacity.\n\nthe other side effect of this is we also are overly tightly coupling virtual numa topology and host topology.\n\noriginally we had envisioned allowing you to spawn a guest with N numa node on a host with Y numa nodes. in the libvirt driver we require N\u003cY however there is no reason to do this.\nthe actually requirement should have been that a vm with N virtual numa nodes will be on at most N host numa nodes if hw:numa_mempolicy\u003dstrict.\n  e.g.a vm with  2 virtual numa nodes should work on a host with 1.\n\nwe similarly over fit for cpu thread policies. we disable the require policy if the virtual cpu hw:cpu_threads\u003d1. the virtual cpu topoloy was never meant to interact with the host cpu thread policy. \n\nthe result of all this additional work we do is all the conditionals and complexity related to the numa features in the code base. if they were independent then the code would be a lot simpler.","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"81016c0ff26bc7b6780296a331556aa036bf5ad3","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"This spec also enables us to execute automatic tests for dpdk"},{"line_number":83,"context_line":"networking projects on qemu backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, NUMA topology for instances becomes nothing to do with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_751e750d","line":84,"range":{"start_line":84,"start_character":0,"end_line":84,"end_character":40},"updated":"2018-02-05 16:58:11.000000000","message":"is this still true? I thought DPDK had remove this restriction?","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"db88e16c2f4c00223282bb967aec5cd61eb0fcbc","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"This spec also enables us to execute automatic tests for dpdk"},{"line_number":83,"context_line":"networking projects on qemu backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, NUMA topology for instances becomes nothing to do with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_0b1b541b","line":84,"range":{"start_line":84,"start_character":0,"end_line":84,"end_character":40},"in_reply_to":"3fa0c359_751e750d","updated":"2018-02-05 18:30:40.000000000","message":"actully dpdk never required this but libvirt could not provide what it actully requries unless you enabled hugepages.\n\ne.g. file descriptor backed preallocated locked shared memory.\n\nyou can now request that specifically via libvirt.\nhugepages meet all of those requirement but from a pure dpdk requirements perspective hugepages were just a perfromace boost over 4k pages that are preallocated and locked and mmapped shared with an open file descriptor to the root of the memory.\n\nso we just used hugepages as it was the only way to get this to work in libvirt back in icehouse. support was added to libvirt about 6-12 months ago to request the actural minimal memory backing that would work for dpdk. https://libvirt.org/formatdomain.html#elementsMemoryBacking\n\ni had a poc patch up to use it but abandoned it when we discovered this cpu pinning limitation with qemu. we originally added the libvirt support to be able to use ovs-dpdk with out kvm but we would need to fix the other aspects adress in this spec to enable it in openstack fully.","commit_id":"be258c7cb3084ba25f14e654ab8f861d21f30815"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"c4e263e21f58def1bb63ce6e45f346c241cbace2","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Use Cases"},{"line_number":76,"context_line":"---------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"This spec allows us to test the NFV functionalities in the upstream"},{"line_number":79,"context_line":"gate that we\u0027re not breaking the nova-libvirt interaction. This reduces"},{"line_number":80,"context_line":"the need for third party CIs such as the Intel NFV CI."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"This spec also enables us to execute automatic tests for dpdk"},{"line_number":83,"context_line":"networking projects on qemu backend environments since vhost-user IFs"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_cfc27b53","line":80,"range":{"start_line":78,"start_character":0,"end_line":80,"end_character":54},"updated":"2018-02-14 10:21:31.000000000","message":"Aye, this is a much stronger use case IMO","commit_id":"f6b1b663d0a549c2329d6855ff081db6568b7cb0"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"c4e263e21f58def1bb63ce6e45f346c241cbace2","unresolved":false,"context_lines":[{"line_number":80,"context_line":"the need for third party CIs such as the Intel NFV CI."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"This spec also enables us to execute automatic tests for dpdk"},{"line_number":83,"context_line":"networking projects on qemu backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_8fa6b3c0","line":83,"range":{"start_line":83,"start_character":23,"end_line":83,"end_character":27},"updated":"2018-02-14 10:21:31.000000000","message":"QEMU","commit_id":"f6b1b663d0a549c2329d6855ff081db6568b7cb0"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"0cc4a3d8060893dc8ffd5dd8ff5bf6d88bdef493","unresolved":false,"context_lines":[{"line_number":80,"context_line":"the need for third party CIs such as the Intel NFV CI."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"This spec also enables us to execute automatic tests for dpdk"},{"line_number":83,"context_line":"networking projects on qemu backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_2820a0b0","line":83,"range":{"start_line":83,"start_character":23,"end_line":83,"end_character":27},"in_reply_to":"3fa0c359_8fa6b3c0","updated":"2018-02-15 01:25:29.000000000","message":"Done","commit_id":"f6b1b663d0a549c2329d6855ff081db6568b7cb0"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"c4e263e21f58def1bb63ce6e45f346c241cbace2","unresolved":false,"context_lines":[{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, NUMA topology for instances becomes nothing to do with"},{"line_number":88,"context_line":"the host\u0027s NUMA topology, and so of the hugepages. They are not tuned"},{"line_number":89,"context_line":"for performance at all."},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_afaa8f6d","line":88,"range":{"start_line":87,"start_character":13,"end_line":88,"end_character":50},"updated":"2018-02-14 10:21:31.000000000","message":"a host\u0027s NUMA topology has no reflection on the NUMA topology of an instance.","commit_id":"f6b1b663d0a549c2329d6855ff081db6568b7cb0"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"0cc4a3d8060893dc8ffd5dd8ff5bf6d88bdef493","unresolved":false,"context_lines":[{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, NUMA topology for instances becomes nothing to do with"},{"line_number":88,"context_line":"the host\u0027s NUMA topology, and so of the hugepages. They are not tuned"},{"line_number":89,"context_line":"for performance at all."},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_c82c9cae","line":88,"range":{"start_line":87,"start_character":13,"end_line":88,"end_character":50},"in_reply_to":"3fa0c359_afaa8f6d","updated":"2018-02-15 01:25:29.000000000","message":"Done","commit_id":"f6b1b663d0a549c2329d6855ff081db6568b7cb0"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"c4e263e21f58def1bb63ce6e45f346c241cbace2","unresolved":false,"context_lines":[{"line_number":91,"context_line":"Proposed change"},{"line_number":92,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"- Change libvirt driver to report non-None value for numa_topology for"},{"line_number":95,"context_line":"  hosts with QEMU backend so that NUMATopologyFilter doesn\u0027t"},{"line_number":96,"context_line":"  exclude the hosts with QEMU backend."},{"line_number":97,"context_line":"- Change libvirt driver not to build a CPU pinning configuration if"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_8fbd53b8","line":94,"range":{"start_line":94,"start_character":53,"end_line":94,"end_character":66},"updated":"2018-02-14 10:21:31.000000000","message":"``numa_topology``","commit_id":"f6b1b663d0a549c2329d6855ff081db6568b7cb0"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"0cc4a3d8060893dc8ffd5dd8ff5bf6d88bdef493","unresolved":false,"context_lines":[{"line_number":91,"context_line":"Proposed change"},{"line_number":92,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"- Change libvirt driver to report non-None value for numa_topology for"},{"line_number":95,"context_line":"  hosts with QEMU backend so that NUMATopologyFilter doesn\u0027t"},{"line_number":96,"context_line":"  exclude the hosts with QEMU backend."},{"line_number":97,"context_line":"- Change libvirt driver not to build a CPU pinning configuration if"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_e829589b","line":94,"range":{"start_line":94,"start_character":53,"end_line":94,"end_character":66},"in_reply_to":"3fa0c359_8fbd53b8","updated":"2018-02-15 01:25:29.000000000","message":"Done","commit_id":"f6b1b663d0a549c2329d6855ff081db6568b7cb0"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"5e01634b2b065b2b4a29ced16ea76d537a9c7248","unresolved":false,"context_lines":[{"line_number":37,"context_line":""},{"line_number":38,"context_line":"  - ``hw:mem_page_size\u003d[small|large|any|\u003cpage_size\u003e]``"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"- NUMA topology:"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of NUMA nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_8c0a7e1d","line":40,"updated":"2018-02-22 15:27:55.000000000","message":"This does not really make sense, all the vCPUs are running in one thread in TCG.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"8b6287ca9ed9f48bd24675a563cdb359abf76d87","unresolved":false,"context_lines":[{"line_number":37,"context_line":""},{"line_number":38,"context_line":"  - ``hw:mem_page_size\u003d[small|large|any|\u003cpage_size\u003e]``"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"- NUMA topology:"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of NUMA nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ff6b8bd7_525fc340","line":40,"in_reply_to":"3fa0c359_8c0a7e1d","updated":"2018-03-06 11:26:50.000000000","message":"Although all the vCPUs are running in one thread in TCG, we can expose virtual NUMA topology to users in a VM.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d4d90c845f74ba1afc2621a1074bb332a6f44914","unresolved":false,"context_lines":[{"line_number":37,"context_line":""},{"line_number":38,"context_line":"  - ``hw:mem_page_size\u003d[small|large|any|\u003cpage_size\u003e]``"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"- NUMA topology:"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of NUMA nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_ccd88654","line":40,"in_reply_to":"3fa0c359_8c0a7e1d","updated":"2018-03-22 21:42:14.000000000","message":"first of all numa has very little to do with the vCPUS but second TCG support multithread backend too if you add --accel tcg,thread\u003dmulti\n\nsee https://wiki.qemu.org/Features/tcg-multithread\n\nby default tcg does run a singel thread yes.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"c62422dd90b2e0ba0c754d1aadc4b733d96abf0a","unresolved":false,"context_lines":[{"line_number":37,"context_line":""},{"line_number":38,"context_line":"  - ``hw:mem_page_size\u003d[small|large|any|\u003cpage_size\u003e]``"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"- NUMA topology:"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of NUMA nodes for the guest\u003e``"},{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_6eef29b0","line":40,"in_reply_to":"ff6b8bd7_525fc340","updated":"2018-03-12 14:00:54.000000000","message":"Yes probably, I\u0027m just thinking about the use-cases of having it.\n\nI might have not well understand something but in your use-case your are saying it\u0027s related to the tests but our CI are using KVM acceleration.\n\nI think that nobody are going to enable hugepages or NUMA without KVM accel and we are not going to improve our CI. So my worry is about to introduce more code that we will have to maintain which are not going to be used.\n\n\nI think a spec related on making better our functional tests to effectively test all of these features would make more sense.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"5e01634b2b065b2b4a29ced16ea76d537a9c7248","unresolved":false,"context_lines":[{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003camount of memory for NUMA node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":49,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_8c235ea7","line":46,"updated":"2018-02-22 15:27:55.000000000","message":"Not really sure about that... please consider the previous comment.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"8b6287ca9ed9f48bd24675a563cdb359abf76d87","unresolved":false,"context_lines":[{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003camount of memory for NUMA node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":49,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ff6b8bd7_720a0727","line":46,"in_reply_to":"3fa0c359_8c235ea7","updated":"2018-03-06 11:26:50.000000000","message":"I\u0027m not going to support CPU pinning in this spec. This list is a reference to sort out which functionalities can be supported with libvirt/QEMU driver and which can not in this spec. We only support hugepages and numa topology and it is mentioned in @L30-32 amd @L60-62","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d4d90c845f74ba1afc2621a1074bb332a6f44914","unresolved":false,"context_lines":[{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003camount of memory for NUMA node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":49,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_0cc3ce58","line":46,"in_reply_to":"3fa0c359_8c235ea7","updated":"2018-03-22 21:42:14.000000000","message":"qemu allows all vcpus of a vm to be pinned to a cpu set but not per vcpu pinning.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"5e01634b2b065b2b4a29ced16ea76d537a9c7248","unresolved":false,"context_lines":[{"line_number":48,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":49,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"- CPU real-time policy:"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"  - ``hw:cpu_realtime\u003d[yes|no]``"},{"line_number":54,"context_line":"  - ``hw:cpu_realtime_mask\u003d\u003cmask for cpus for non real-time\u003e``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_6910445a","line":51,"updated":"2018-02-22 15:27:55.000000000","message":"This is not supported in TCG.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b72c209310dc77f1816d6f1a0fb8f703afe6154","unresolved":false,"context_lines":[{"line_number":43,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"},{"line_number":44,"context_line":"  - ``hw:num_mem.N\u003d\u003camount of memory for NUMA node N for the guest\u003e``"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"- CPU pinning policy:"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":49,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"- CPU real-time policy:"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"  - ``hw:cpu_realtime\u003d[yes|no]``"},{"line_number":54,"context_line":"  - ``hw:cpu_realtime_mask\u003d\u003cmask for cpus for non real-time\u003e``"},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"- Emulator threads policy:"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"  - ``hw:emulator_threads_policy\u003d[share|isolate]``"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Since CPU pinning policy, CPU real-time policy, and emulator threads"},{"line_number":61,"context_line":"policy all depend on individual CPU pinning, we can\u0027t support them"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_a011e706","line":58,"range":{"start_line":46,"start_character":0,"end_line":58,"end_character":50},"updated":"2018-02-18 01:26:16.000000000","message":"nit technically these are not numa related featurs as they have nothing to do with memory access times/bandwidth.\n\na general term that can aggreate these types of request is enhanced plathform awareness or EPA. so these would be examples of epa features however they are not the only epa features. epa features are any features that use detailed knolages of the plathform (compute,network or storage) to more efficently place a workload and assign resouce to that workload. e.g. nic/cpu capablities, network bandwith, cpu or numa affintiy or even storage latency/locality.\n\nno need to resping for this but if you do EPA related features would be more correct then numa related.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"5e01634b2b065b2b4a29ced16ea76d537a9c7248","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"This spec also enables us to execute automatic tests for dpdk"},{"line_number":83,"context_line":"networking projects on QEMU backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, a host\u0027s NUMA topology has no reflection on the NUMA"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_ac9bbac6","line":84,"updated":"2018-02-22 15:27:55.000000000","message":"Is vhost supported in TCG?","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"8b6287ca9ed9f48bd24675a563cdb359abf76d87","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"This spec also enables us to execute automatic tests for dpdk"},{"line_number":83,"context_line":"networking projects on QEMU backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, a host\u0027s NUMA topology has no reflection on the NUMA"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ff6b8bd7_051ca368","line":84,"in_reply_to":"3fa0c359_ac9bbac6","updated":"2018-03-06 11:26:50.000000000","message":"Looks like not [1]. Deleted this part. Thanks.\n\n[1] https://bugzilla.redhat.com/show_bug.cgi?id\u003d1410716","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b72c209310dc77f1816d6f1a0fb8f703afe6154","unresolved":false,"context_lines":[{"line_number":83,"context_line":"networking projects on QEMU backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, a host\u0027s NUMA topology has no reflection on the NUMA"},{"line_number":88,"context_line":"topology of an instance. They are not tuned for performance at all."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Proposed change"},{"line_number":91,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_e0304fa1","line":88,"range":{"start_line":86,"start_character":0,"end_line":88,"end_character":67},"updated":"2018-02-18 01:26:16.000000000","message":"nit i would remove this if you respin.\nthis is not very precises and does not really add much.\n\nqemu with the tci backend can do pining of all cores to a mask of cores via libvirt, you just cant use the vcpuin element which is only requred for 1:1 vCPU:pCPU pinning so you have to do it via the vcpu element.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"8b6287ca9ed9f48bd24675a563cdb359abf76d87","unresolved":false,"context_lines":[{"line_number":83,"context_line":"networking projects on QEMU backend environments since vhost-user IFs"},{"line_number":84,"context_line":"require that VMs have hugepages backend."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Note that this spec can\u0027t be brought in production use cases. Without"},{"line_number":87,"context_line":"CPU pinning, a host\u0027s NUMA topology has no reflection on the NUMA"},{"line_number":88,"context_line":"topology of an instance. They are not tuned for performance at all."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Proposed change"},{"line_number":91,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ff6b8bd7_e5516f10","line":88,"range":{"start_line":86,"start_character":0,"end_line":88,"end_character":67},"in_reply_to":"3fa0c359_e0304fa1","updated":"2018-03-06 11:26:50.000000000","message":"Done","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b72c209310dc77f1816d6f1a0fb8f703afe6154","unresolved":false,"context_lines":[{"line_number":93,"context_line":"- Change libvirt driver to report non-None value for ``numa_topology``"},{"line_number":94,"context_line":"  for hosts with QEMU backend so that NUMATopologyFilter doesn\u0027t"},{"line_number":95,"context_line":"  exclude the hosts with QEMU backend."},{"line_number":96,"context_line":"- Change libvirt driver not to build a CPU pinning configuration if"},{"line_number":97,"context_line":"  the backend is QEMU."},{"line_number":98,"context_line":"- Change NUMATopologyFilter to exclude the hosts with QEMU backend if"},{"line_number":99,"context_line":"  CPU pinning policy is set to dedicated expicitly by users."},{"line_number":100,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"3fa0c359_c0d2f334","line":97,"range":{"start_line":96,"start_character":2,"end_line":97,"end_character":22},"updated":"2018-02-18 01:26:16.000000000","message":"an alternive would be to pin all vcpus via the vcpu element but allow them too float over the set of cores. this would have the same effect as the semi pinned cores with current numa support if the guest has a singel numa node. for guest with multiple numa nodes they could still float across numa but would be confined to only the number of host numa nodes as guest numa nodes requested.","commit_id":"b2413a65af5926335b1ffc5866e19cfe6fd07be2"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"522158d7c248a999bd787af20c2826af539a70eb","unresolved":false,"context_lines":[{"line_number":77,"context_line":""},{"line_number":78,"context_line":"This spec allows us to test the NFV functionalities in the upstream"},{"line_number":79,"context_line":"gate that we\u0027re not breaking the nova-libvirt interaction. This reduces"},{"line_number":80,"context_line":"the need for third party CIs such as the Intel NFV CI."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Proposed change"},{"line_number":83,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_e7d62928","line":80,"updated":"2018-03-27 08:18:56.000000000","message":"While I agree that it is important because it allows us to do testing in the gate, I don\u0027t think we should assert that it reduces the need for third party CI. We\u0027d still like the Intel NFV CI to be doing its thing, wouldn\u0027t we?","commit_id":"50c43186d7ea4e16afc8c8895a5650f9dece2a5a"},{"author":{"_account_id":7634,"name":"Takashi Natsume","email":"takanattie@gmail.com","username":"natsumet"},"change_message_id":"cd433c5455164faf0b1bf3b54eaf19a09a9ea866","unresolved":false,"context_lines":[{"line_number":172,"context_line":""},{"line_number":173,"context_line":"Most of the focus will be on unit tests for construction of the guest"},{"line_number":174,"context_line":"instance xml configuration in libvirt driver, and on unit tests for"},{"line_number":175,"context_line":"NUMATopologyFilter."},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"Documentation Impact"},{"line_number":178,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_5bf7d0d5","line":175,"updated":"2018-04-06 06:20:03.000000000","message":"Functional tests?\nWill the tests be added or modified in the following file?\n\n* nova/tests/functional/libvirt/test_numa_servers.py","commit_id":"48b6abf1b508a15313d9d962a9889f3885e985c0"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":32,"context_line":""},{"line_number":33,"context_line":"However, QEMU backend potentially has capability to support NUMA"},{"line_number":34,"context_line":"related features such as Hugepages and NUMA Topology. Furthermore,"},{"line_number":35,"context_line":"CPU pinning policy is also possible by running the single thread of"},{"line_number":36,"context_line":"QEMU TCG to on the dedicated cores. This might not be practical,"},{"line_number":37,"context_line":"but it is useful for testing the pinning feature in the CI gate,"},{"line_number":38,"context_line":"and we can at least ensure that the dedicated cores are not used by"},{"line_number":39,"context_line":"another instance."}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_bee6b9ae","line":36,"range":{"start_line":35,"start_character":58,"end_line":36,"end_character":34},"updated":"2018-10-23 19:54:58.000000000","message":"the TCG backend support multiple thread by the way.\n\ni think libvirt will enable the multithread support if the vm has mupltipls cpus.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"- Hugepages:"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"  - ``hw:mem_page_size\u003d[small|large|any|\u003cpage_size\u003e]``"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"- NUMA topology:"},{"line_number":48,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_7e37c14c","line":45,"range":{"start_line":45,"start_character":2,"end_line":45,"end_character":54},"updated":"2018-10-23 19:54:58.000000000","message":"this works if we skipp the 1:x vCPU to host CPU pinning we do when you request a numa toplogy.\n\npining all cpus without using the vcpuin element work.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":46,"context_line":""},{"line_number":47,"context_line":"- NUMA topology:"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"  - ``hw:numa_nodes\u003d\u003cnumber of NUMA nodes for the guest\u003e``"},{"line_number":50,"context_line":"  - ``hw:numa_cpus.N\u003d\u003cnumber of pCPUs for NUMA node N for the guest\u003e``"},{"line_number":51,"context_line":"  - ``hw:num_mem.N\u003d\u003camount of memory for NUMA node N for the guest\u003e``"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"- CPU pinning policy:"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_3e3dc928","line":51,"range":{"start_line":49,"start_character":3,"end_line":51,"end_character":69},"updated":"2018-10-23 19:54:58.000000000","message":"this all works module the cpu pinning with the vcpu pin set elment also.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":52,"context_line":""},{"line_number":53,"context_line":"- CPU pinning policy:"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":56,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"- CPU real-time policy:"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_9e7addec","line":55,"range":{"start_line":55,"start_character":3,"end_line":55,"end_character":40},"updated":"2018-10-23 19:54:58.000000000","message":"you can pin all the vms cores to a set of host cores\nbut you cant do per VCPU core  pinning so you cant\nimplement the thread policies but you can support both shared and dedicated policy.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":53,"context_line":"- CPU pinning policy:"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":56,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"- CPU real-time policy:"},{"line_number":59,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_fe1431a1","line":56,"range":{"start_line":56,"start_character":2,"end_line":56,"end_character":53},"updated":"2018-10-23 19:54:58.000000000","message":"these cant be supported. really but i would argure the prefer policy is technically supportable see previous comment.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":55,"context_line":"  - ``hw:cpu_policy\u003d[shared|dedicated]``"},{"line_number":56,"context_line":"  - ``hw:cpu_thread_policy\u003d[prefer|isolate|require]``"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"- CPU real-time policy:"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"  - ``hw:cpu_realtime\u003d[yes|no]``"},{"line_number":61,"context_line":"  - ``hw:cpu_realtime_mask\u003d\u003cmask for cpus for non real-time\u003e``"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"- Emulator threads policy:"},{"line_number":64,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_1e6ecd2c","line":61,"range":{"start_line":58,"start_character":0,"end_line":61,"end_character":62},"updated":"2018-10-23 19:54:58.000000000","message":"this i think need kvm","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":60,"context_line":"  - ``hw:cpu_realtime\u003d[yes|no]``"},{"line_number":61,"context_line":"  - ``hw:cpu_realtime_mask\u003d\u003cmask for cpus for non real-time\u003e``"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"- Emulator threads policy:"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"  - ``hw:emulator_threads_policy\u003d[share|isolate]``"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Supporting CPU real-time policy and emulator threads policy for QEMU"},{"line_number":68,"context_line":"backend is out of the scope of this spec."}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_5e51a562","line":65,"range":{"start_line":63,"start_character":0,"end_line":65,"end_character":50},"updated":"2018-10-23 19:54:58.000000000","message":"i belive you could support this too as its a seperate element in the libvirt xml/qemu commandline","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":69,"context_line":""},{"line_number":70,"context_line":".. note::"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"    Because CPU topology feature doesn\u0027t depend on NUMA topology,"},{"line_number":73,"context_line":"    it already works with libvirt/QEMU driver."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"    - CPU topology:"},{"line_number":76,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_dec29502","line":73,"range":{"start_line":72,"start_character":3,"end_line":73,"end_character":46},"updated":"2018-10-23 19:54:58.000000000","message":"yep this all works fine.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":81,"context_line":"Use Cases"},{"line_number":82,"context_line":"---------"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"This spec allows us to test the NFV functionalities in the upstream"},{"line_number":85,"context_line":"CI gate that we\u0027re not breaking the nova-libvirt interaction."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Proposed change"},{"line_number":88,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_7e582156","line":85,"range":{"start_line":84,"start_character":0,"end_line":85,"end_character":61},"updated":"2018-10-23 19:54:58.000000000","message":"yes and it will allow you to run vms with non native cpu instruction sets in the cloud.\n\ne.g. as a develop of a arm applciation i may want to use vm to host my dev\\test enviornment ....\n\nkvm does not allow emulation of other architecture so if you want to do architecure emulation and still want semi\nok perfromnce then enabling cpu pinning + hugepages + the multi thread tcg backend will likly still compiling or developing on a low end arm device directly.\n\ne.g. think iot application development.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Change libvirt driver not to build individual vCPU pinning configuration"},{"line_number":91,"context_line":"if the backend is QEMU. Instead, we support VM wide pinning for the"},{"line_number":92,"context_line":"single thread of QEMU TCG."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"Alternatives"},{"line_number":95,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_f94d2b71","line":92,"range":{"start_line":92,"start_character":1,"end_line":92,"end_character":25},"updated":"2018-10-23 19:54:58.000000000","message":"TCG backend.\n\ndrop single threaded \nhttps://wiki.qemu.org/Features/tcg-multithread\nwhen i tested this in the past libvirt seamed to do the right hing when you enabled the multi threaded backed.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"f3ffefe239bcabd092ece2f69e4cde64eb3eb258","unresolved":false,"context_lines":[{"line_number":167,"context_line":""},{"line_number":168,"context_line":"None"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"Testing"},{"line_number":171,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"Most of the focus will be on unit tests for construction of the guest"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_b3a112f9","line":170,"updated":"2018-10-23 18:31:54.000000000","message":"If the main point of this spec is just to emulate this with the qemu virt_type so we can test it in the gate without dedicated hardware, that should probably be explained in this section along with mentioning the need for some job configuration to run this and any tests we could add (maybe we could re-use some of the existing 3rd party CI tempest plugins for testing in that job?).","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"8a3bafc1963e6983ac8c5c9936052bcb4c7ba0cb","unresolved":false,"context_lines":[{"line_number":167,"context_line":""},{"line_number":168,"context_line":"None"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"Testing"},{"line_number":171,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"Most of the focus will be on unit tests for construction of the guest"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3f79a3b5_79a7db0a","line":170,"in_reply_to":"3f79a3b5_b3a112f9","updated":"2018-10-23 19:54:58.000000000","message":"the intel ones are alittl invaise in that they ssh into the host and actully read the xml generated and make sure it did  the right thing.\n\nbut some of them might be resonable to reuse.\n\ni do not think this is the only usecase however.\ni really do think cross architecture emulation is a thing that woudl be nice to leaverage hugepages with.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"f3ffefe239bcabd092ece2f69e4cde64eb3eb258","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"3f79a3b5_d37fee66","line":188,"updated":"2018-10-23 18:31:54.000000000","message":"Missing the History section.","commit_id":"aaf98f36e5919ce78cf536b6578f924c3711c751"}]}