)]}'
{"specs/rocky/approved/numa-aware-vswitches.rst":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae9e380e09600c93831c875e60ba06e7e549993f","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-vswitches"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"vSwitches such as OVS-DPDK allow binding to physical NICs, usually connected"},{"line_number":14,"context_line":"via PCIe. These NICs have NUMA affinity which is not currently taken into"},{"line_number":15,"context_line":"account when creating an instance. This can result in up to a 50% performance"},{"line_number":16,"context_line":"drop."}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_462e1844","line":13,"range":{"start_line":13,"start_character":18,"end_line":13,"end_character":26},"updated":"2018-02-06 15:57:57.000000000","message":"actully ovs-dpdk does not support this currently.\n\nkernel ovs can with melonox nics, intel nics will be supporte in the future. or you can use netronomes fork agilo ovs with there nics.\n\nin both case if nova is provisioning the vf for the hardware offload then the existing pci passthrough filter and the virt driver should already enforce numa affinity.","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"706d0b4db63f22f9013c97117ca26ea870edcf73","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-vswitches"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"vSwitches such as OVS-DPDK allow binding to physical NICs, usually connected"},{"line_number":14,"context_line":"via PCIe. These NICs have NUMA affinity which is not currently taken into"},{"line_number":15,"context_line":"account when creating an instance. This can result in up to a 50% performance"},{"line_number":16,"context_line":"drop."}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_4e43066d","line":13,"range":{"start_line":13,"start_character":18,"end_line":13,"end_character":26},"in_reply_to":"3fa0c359_462e1844","updated":"2018-02-13 11:33:57.000000000","message":"Correct. I\u0027ve updated this to reflect what was actually intended (the natural affinity due to use of dpdk ports)","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae9e380e09600c93831c875e60ba06e7e549993f","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-vswitches"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"vSwitches such as OVS-DPDK allow binding to physical NICs, usually connected"},{"line_number":14,"context_line":"via PCIe. These NICs have NUMA affinity which is not currently taken into"},{"line_number":15,"context_line":"account when creating an instance. This can result in up to a 50% performance"},{"line_number":16,"context_line":"drop."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_bb04d518","line":16,"range":{"start_line":13,"start_character":1,"end_line":16,"end_character":5},"updated":"2018-02-06 15:57:57.000000000","message":"well if dpdk is built against libnumactl and CONFIG_RTE_LIBRTE_VHOST_NUMA\u003dy then ovs-dpdk will allocate the vm iterface to be processed by a pmd core on the same numa node so the packets per second impact will be much less e.g. \u003c10% if i recall correctly.\n\nyou should not see a 50% performance hit unless both the pmd and vm are on diffent numa nodes and the pmd is also on a different numa node form the nic.","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"706d0b4db63f22f9013c97117ca26ea870edcf73","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-vswitches"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"vSwitches such as OVS-DPDK allow binding to physical NICs, usually connected"},{"line_number":14,"context_line":"via PCIe. These NICs have NUMA affinity which is not currently taken into"},{"line_number":15,"context_line":"account when creating an instance. This can result in up to a 50% performance"},{"line_number":16,"context_line":"drop."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_9153516b","line":16,"range":{"start_line":13,"start_character":1,"end_line":16,"end_character":5},"in_reply_to":"3fa0c359_bb04d518","updated":"2018-02-13 11:33:57.000000000","message":"I\u0027ve updated this to clarify things a little. Hopefully it\u0027s not complete nonsense :)","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae9e380e09600c93831c875e60ba06e7e549993f","unresolved":false,"context_lines":[{"line_number":23,"context_line":"driver. This was done by utilizing the NUMA information for these PCIe devices"},{"line_number":24,"context_line":"that was provided by libvirt. However, the use of vSwitch solutions complicates"},{"line_number":25,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":26,"context_line":"functions but rather a VIF object. The issues raised by NUMA affinity are still"},{"line_number":27,"context_line":"present though and still need to be addressed."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Use Cases"},{"line_number":30,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_86b8a02a","line":27,"range":{"start_line":26,"start_character":35,"end_line":27,"end_character":46},"updated":"2018-02-06 15:57:57.000000000","message":"well there are numa effect but i had hoped that openstack would never have to adress them and specificlaly if it did i hoped that neutron would adress this not nova.\n\nthe 50% perfromace hit above would lead me to susspect that\nCONFIG_RTE_LIBRTE_VHOST_NUMA\u003dy was not enabled when dpdk was compiled(it is disabled by default as libnuma is an optional dependecy) there for ovs was not able optimis the numa affinity internally.","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae9e380e09600c93831c875e60ba06e7e549993f","unresolved":false,"context_lines":[{"line_number":39,"context_line":"Proposed change"},{"line_number":40,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"We add a new configuration option, ``[pci] neutron_bridge_mappings``, which"},{"line_number":43,"context_line":"define a mapping for neutron bridges to the physical PCI devices that are used."},{"line_number":44,"context_line":"For example::"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"   [pci]"},{"line_number":47,"context_line":"   neutron_bridge_mappings \u003d [(\u0027mgmt\u0027, \u0027MyNIC\u0027), ...]"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"where ``MyNIC`` is the name of an alias defined in ``[pci] alias``.  This"},{"line_number":50,"context_line":"allows us to harness the functionality we already have in-place for PCI-NUMA"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_26898c22","line":47,"range":{"start_line":42,"start_character":0,"end_line":47,"end_character":53},"updated":"2018-02-06 15:57:57.000000000","message":"no this should be in neutron or you will have to replicate this for all neutron backends.\n\nalso unfortunetly this will not work.\n\nif i use a bond for 2 nics as my uplink port in neutron for physnet A then each member of the bond could have different numa affinity. for tunneled networks select there uplink are selected via the kernel routing tables. if you are doing multipath routing then you cant know ahead of time which numa node it will take. \n\nincidentally this is why i always advise that you allocate the local tunnel enpoint ip to the ovs bridge with the phyical nic or bond attached to avoid that.","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":24777,"name":"Steven Webster","email":"steven.webster@windriver.com","username":"swebster"},"change_message_id":"8aa00db57eaffa589203fc10326f2d084f69e756","unresolved":false,"context_lines":[{"line_number":52,"context_line":":doc:`/specs/queens/approved/share-pci-between-numa-nodes`, with minimal"},{"line_number":53,"context_line":"changes to nova and no changes to neutron. The configuration of this attribute"},{"line_number":54,"context_line":"would be left to the user or, more realistically, the deployment tool."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"Alternatives"},{"line_number":57,"context_line":"------------"},{"line_number":58,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_c7b1c291","line":55,"updated":"2018-02-06 18:00:09.000000000","message":"Would the PciPassthroughFilter need to be configured?  Or would another (perhaps more name appropriate) filter be created?","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae9e380e09600c93831c875e60ba06e7e549993f","unresolved":false,"context_lines":[{"line_number":78,"context_line":"information as deployment time, and discarding that in favour of dynamic"},{"line_number":79,"context_line":"generation introduces unnecessary magic to the formula."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"Finally, we could take variants of both of the above and report this NUMA"},{"line_number":82,"context_line":"information to placement itself. This hasn\u0027t been fully evaluated but at first"},{"line_number":83,"context_line":"glance it would not solve any of the problems with either solution and would"},{"line_number":84,"context_line":"appear to offer no improvement over simply passing this information as part of"},{"line_number":85,"context_line":"the response from neutron."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Data model impact"},{"line_number":88,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_c1c53290","line":85,"range":{"start_line":81,"start_character":0,"end_line":85,"end_character":26},"updated":"2018-02-06 15:57:57.000000000","message":"this is the aproch that i wanted to pursue in denver by starting to model network backend in placement.\n\nin denver i was proposing that neutron would be respocible for creating a placement resouce provider useing the neutron agent uuid as the RP uuid per agent per host. \n\nthe l2 agents would have a child RP per uplink bridge an a optional addtion RP for tunneled traffic. where a RP is used for tunneled traffic or not would be modeled as a Trait.\n\nthese child resouce providers would have an inventory of bandwith,and optionally vifs.the placement aspects of this overall structure are covered in https://review.openstack.org/#/c/502306/\n\nsince that was written however i would make on change based on how placement has evolved.\n\ninstead of having a RP for the agent we should have placement aggregate per neutron agent per host. then the child resouce providers of the agent would be \"parented\" form the numa nodes of the compute node. \n\nthe original spec was assuming we would either A.) allow multi pareent with a primay parent but not allow any tree to have multiple roots or loops, or B.) we would have a \"week reference\" style constuct in placement to allow the the RP to be pareneted to the numa nodes and weakly childeren of the agent RP. aggreagtes can be used for that fuction.","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"cadc8664cbd18ca745cb80be4412341dad06a698","unresolved":false,"context_lines":[{"line_number":78,"context_line":"information as deployment time, and discarding that in favour of dynamic"},{"line_number":79,"context_line":"generation introduces unnecessary magic to the formula."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"Finally, we could take variants of both of the above and report this NUMA"},{"line_number":82,"context_line":"information to placement itself. This hasn\u0027t been fully evaluated but at first"},{"line_number":83,"context_line":"glance it would not solve any of the problems with either solution and would"},{"line_number":84,"context_line":"appear to offer no improvement over simply passing this information as part of"},{"line_number":85,"context_line":"the response from neutron."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Data model impact"},{"line_number":88,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_4f7e28e5","line":85,"range":{"start_line":81,"start_character":0,"end_line":85,"end_character":26},"in_reply_to":"3fa0c359_c1c53290","updated":"2018-02-06 20:24:56.000000000","message":"Sean\u0027s comment is really the only part of this I can weigh in on.  I agree it would be nice to do this via placement modeling of NUMA.  That\u0027s the long-term goal anyway.  But aside from NRP (which I\u0027m pretty confident will be finished up early in Rocky), we would need to solve the NUMA affinity problem as well.  One way is via the \"aggregate distance\" thing Jay has been talking about, which I don\u0027t fully understand yet.  There are other possibilities.  But we should put together some ideas for discussion in Dublin.  Because the only thing I see in the main proposal here is \"aaargh, not more stuff bolted onto the PCI subsystem!\"","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":24777,"name":"Steven Webster","email":"steven.webster@windriver.com","username":"swebster"},"change_message_id":"8aa00db57eaffa589203fc10326f2d084f69e756","unresolved":false,"context_lines":[{"line_number":113,"context_line":"Performance Impact"},{"line_number":114,"context_line":"------------------"},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"There will be a negligible increase in the time take to schedule an instance on"},{"line_number":117,"context_line":"account of the additional NUMA affinity checks. However, network performance of"},{"line_number":118,"context_line":"some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":119,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_4763721b","line":116,"updated":"2018-02-06 18:00:09.000000000","message":"taken","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"706d0b4db63f22f9013c97117ca26ea870edcf73","unresolved":false,"context_lines":[{"line_number":113,"context_line":"Performance Impact"},{"line_number":114,"context_line":"------------------"},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"There will be a negligible increase in the time take to schedule an instance on"},{"line_number":117,"context_line":"account of the additional NUMA affinity checks. However, network performance of"},{"line_number":118,"context_line":"some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":119,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_d18479d5","line":116,"in_reply_to":"3fa0c359_4763721b","updated":"2018-02-13 11:33:57.000000000","message":"Done","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae9e380e09600c93831c875e60ba06e7e549993f","unresolved":false,"context_lines":[{"line_number":115,"context_line":""},{"line_number":116,"context_line":"There will be a negligible increase in the time take to schedule an instance on"},{"line_number":117,"context_line":"account of the additional NUMA affinity checks. However, network performance of"},{"line_number":118,"context_line":"some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Other deployer impact"},{"line_number":121,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_c173f27f","line":118,"range":{"start_line":118,"start_character":0,"end_line":118,"end_character":75},"updated":"2018-02-06 15:57:57.000000000","message":"am no it should increae by at most 10-20% if do not have a pmd on all numa nodes. in a default unoptimsed ovs-dpdk deployment you should always have 2 pmd per numa node that has an nic and preferbaly at least one pmd on each numa node.","commit_id":"9eafc78b9e5d66dadcf454bec11d1cc5e6a2d96e"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":29,"context_line":"driver. This was done by utilizing the NUMA information for these PCIe devices"},{"line_number":30,"context_line":"that was provided by libvirt. However, the use of vSwitch solutions complicates"},{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Because these VIFs have a NUMA affinity"},{"line_number":33,"context_line":"which is not accounted for."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Consider this from the perspective of a configuration using Open vSwitch with"},{"line_number":36,"context_line":"the DPDK datapath, commonly referred to as OVS-DPDK::"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_5d6ab319","line":33,"range":{"start_line":32,"start_character":35,"end_line":33,"end_character":27},"updated":"2018-02-13 18:27:45.000000000","message":"I think there was something else intended to be in this sentence.","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":29,"context_line":"driver. This was done by utilizing the NUMA information for these PCIe devices"},{"line_number":30,"context_line":"that was provided by libvirt. However, the use of vSwitch solutions complicates"},{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Because these VIFs have a NUMA affinity"},{"line_number":33,"context_line":"which is not accounted for."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Consider this from the perspective of a configuration using Open vSwitch with"},{"line_number":36,"context_line":"the DPDK datapath, commonly referred to as OVS-DPDK::"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_4f018b69","line":33,"range":{"start_line":32,"start_character":35,"end_line":33,"end_character":27},"in_reply_to":"3fa0c359_5d6ab319","updated":"2018-03-07 15:48:36.000000000","message":"Um, yeah :)","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"    port0 --rxq--\u003e vswitch --txq--\u003e port1"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"where ``rxq`` is one or more receive (Rx) queues and ``txq`` is one of more"},{"line_number":60,"context_line":"transmit (Tx) queues. In OVS-DPDK, the ``rxq``\\s will be placed on the same"},{"line_number":61,"context_line":"NUMA node as ``port0``, assuming DPDK was compiled with ``libnumactl`` and"},{"line_number":62,"context_line":"``CONFIG_RTE_LIBRTE_VHOST_NUMA\u003dy`` [3]_. However, if the NIC associated with"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_9d5d9b7a","line":59,"range":{"start_line":59,"start_character":68,"end_line":59,"end_character":70},"updated":"2018-02-13 18:27:45.000000000","message":"\"or\"","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"    port0 --rxq--\u003e vswitch --txq--\u003e port1"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"where ``rxq`` is one or more receive (Rx) queues and ``txq`` is one of more"},{"line_number":60,"context_line":"transmit (Tx) queues. In OVS-DPDK, the ``rxq``\\s will be placed on the same"},{"line_number":61,"context_line":"NUMA node as ``port0``, assuming DPDK was compiled with ``libnumactl`` and"},{"line_number":62,"context_line":"``CONFIG_RTE_LIBRTE_VHOST_NUMA\u003dy`` [3]_. However, if the NIC associated with"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_8f9a3311","line":59,"range":{"start_line":59,"start_character":68,"end_line":59,"end_character":70},"in_reply_to":"3fa0c359_9d5d9b7a","updated":"2018-03-07 15:48:36.000000000","message":"Done","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Once again, traffic egressing from *Guest 1* would take a flow like so::"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"    port0 --rxq--\u003e vswitch --txq--\u003e port1"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"Because ``port1`` is once again on a different NUMA node (because that\u0027s where"},{"line_number":85,"context_line":"the guest is), we will see cross-NUMA traffic."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_5dd873ed","line":82,"range":{"start_line":82,"start_character":4,"end_line":82,"end_character":9},"updated":"2018-02-13 18:27:45.000000000","message":"Do these correspond to guest1 and guest2?","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Once again, traffic egressing from *Guest 1* would take a flow like so::"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"    port0 --rxq--\u003e vswitch --txq--\u003e port1"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"Because ``port1`` is once again on a different NUMA node (because that\u0027s where"},{"line_number":85,"context_line":"the guest is), we will see cross-NUMA traffic."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_6186db68","line":82,"range":{"start_line":82,"start_character":4,"end_line":82,"end_character":9},"in_reply_to":"3fa0c359_5dd873ed","updated":"2018-03-07 15:48:36.000000000","message":"They do. This is clarified","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":113,"context_line":"affinity, including things like"},{"line_number":114,"context_line":":doc:`/specs/queens/approved/share-pci-between-numa-nodes`, with minimal"},{"line_number":115,"context_line":"changes to nova and no changes to neutron. The configuration of this attribute"},{"line_number":116,"context_line":"would be left to the user or, more realistically, the deployment tool."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Alternatives"},{"line_number":119,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_5d015337","line":116,"updated":"2018-02-13 18:27:45.000000000","message":"Are we not able to determine the mapping ourselves? I\u0027ve never understood why we need so much of this sort of manual configuration for such things.","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":113,"context_line":"affinity, including things like"},{"line_number":114,"context_line":":doc:`/specs/queens/approved/share-pci-between-numa-nodes`, with minimal"},{"line_number":115,"context_line":"changes to nova and no changes to neutron. The configuration of this attribute"},{"line_number":116,"context_line":"would be left to the user or, more realistically, the deployment tool."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Alternatives"},{"line_number":119,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_c3d55645","line":116,"in_reply_to":"3fa0c359_5d015337","updated":"2018-03-07 15:48:36.000000000","message":"It\u0027s really f****** difficult :D _Lots_ more info added about this (including shiny pictures!)","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":114,"context_line":":doc:`/specs/queens/approved/share-pci-between-numa-nodes`, with minimal"},{"line_number":115,"context_line":"changes to nova and no changes to neutron. The configuration of this attribute"},{"line_number":116,"context_line":"would be left to the user or, more realistically, the deployment tool."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Alternatives"},{"line_number":119,"context_line":"------------"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_5daf1329","line":117,"updated":"2018-02-13 18:27:45.000000000","message":"I think there\u0027s detail missing here. So far, the \"proposed change\" is just \"add a configuration element\". Presumably we need a new filter that will try to pick proper numa topologies for these constraints? Maybe it\u0027s the same filter? Can you describe the thing that\u0027s changing and not just the new bit of data you\u0027ll need?","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":114,"context_line":":doc:`/specs/queens/approved/share-pci-between-numa-nodes`, with minimal"},{"line_number":115,"context_line":"changes to nova and no changes to neutron. The configuration of this attribute"},{"line_number":116,"context_line":"would be left to the user or, more realistically, the deployment tool."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Alternatives"},{"line_number":119,"context_line":"------------"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_43d8667e","line":117,"in_reply_to":"3fa0c359_5daf1329","updated":"2018-03-07 15:48:36.000000000","message":"I think I\u0027ve clarified this now, but tl;dr: we use NUMATopologyFilter but users don\u0027t need to register devices. Let me know if it\u0027s still not clear","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":121,"context_line":"There are multiple possible solutions to this issue of various levels of"},{"line_number":122,"context_line":"complexity."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"We could store the static bridge-PCI device mapping in neutron instead and pass"},{"line_number":125,"context_line":"this over the wire as part of the VIF request. As neutron is responsible for"},{"line_number":126,"context_line":"all things network\u0027y, storing this information in nova might appear to break"},{"line_number":127,"context_line":"the separation of concerns between the two services. However, this is only"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_7d9d2f08","line":124,"range":{"start_line":124,"start_character":9,"end_line":124,"end_character":62},"updated":"2018-02-13 18:27:45.000000000","message":"I guess this addresses my question above? It seems unfortunate to have to provide nova with these details manually.","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":121,"context_line":"There are multiple possible solutions to this issue of various levels of"},{"line_number":122,"context_line":"complexity."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"We could store the static bridge-PCI device mapping in neutron instead and pass"},{"line_number":125,"context_line":"this over the wire as part of the VIF request. As neutron is responsible for"},{"line_number":126,"context_line":"all things network\u0027y, storing this information in nova might appear to break"},{"line_number":127,"context_line":"the separation of concerns between the two services. However, this is only"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_43c14680","line":124,"range":{"start_line":124,"start_character":9,"end_line":124,"end_character":62},"in_reply_to":"3fa0c359_7d9d2f08","updated":"2018-03-07 15:48:36.000000000","message":"Rather :( I honestly can\u0027t think of any better ideas yet though, for the reasons noted below","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"e492a4e43686dd964a1d138a6ae6c30bec3b9730","unresolved":false,"context_lines":[{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Deployment tooling will need to be enhanced to configure the bridge-PCI mapping"},{"line_number":173,"context_line":"configuration option at deployment time."},{"line_number":174,"context_line":""},{"line_number":175,"context_line":"Performance Impact"},{"line_number":176,"context_line":"------------------"},{"line_number":177,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_bdaacd6d","line":174,"updated":"2018-02-14 02:51:01.000000000","message":"Given that this spec is going to use the same scheme as PCI-NUMA affinity and use existing NUMATopologyFilter, do this spec force users to have \"port creation first operation\" like SR-IOV port? \n\nIf true, that sounds the end user impact to me; at least that should be mentioned. If false, how do we schedule?","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Deployment tooling will need to be enhanced to configure the bridge-PCI mapping"},{"line_number":173,"context_line":"configuration option at deployment time."},{"line_number":174,"context_line":""},{"line_number":175,"context_line":"Performance Impact"},{"line_number":176,"context_line":"------------------"},{"line_number":177,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_e3197222","line":174,"in_reply_to":"3fa0c359_bdaacd6d","updated":"2018-03-07 15:48:36.000000000","message":"You\u0027re right. I\u0027ve expanded this here.","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":213,"context_line":""},{"line_number":214,"context_line":"#. Add support for the new configuration option"},{"line_number":215,"context_line":""},{"line_number":216,"context_line":"#. Parse the configuration option and use it at scheduling time"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"Dependencies"},{"line_number":219,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_dd51e337","line":216,"updated":"2018-02-13 18:27:45.000000000","message":"Presumably this confirms my suspicion above that more work will be done than just adding a config variable :)","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c6af9f65d31a4e6b38200c790fb0bf5c4b0df422","unresolved":false,"context_lines":[{"line_number":225,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"This will need to be tested in a third-party CI due to the reliance on"},{"line_number":228,"context_line":"hardware."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_7d444f74","line":228,"updated":"2018-02-13 18:27:45.000000000","message":"It will need to be, but will it? Or are we likely expecting more \"well it works on my NUMA test box\" ?","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":225,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"This will need to be tested in a third-party CI due to the reliance on"},{"line_number":228,"context_line":"hardware."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3fa0c359_c3ca3619","line":228,"in_reply_to":"3fa0c359_7d444f74","updated":"2018-03-07 15:48:36.000000000","message":"I\u0027d love to say it will, but I really don\u0027t know yet. The Intel PCI CIs are down for ages now and no one else is stepping up to provide this infrastructure. The best I can do is validate locally and provide functional tests to cover it, even if they are mocked up to the eyeballs.\n\nI\u0027ve clarified this here.","commit_id":"597a09d2242828f768d57c0aa054301aa9c26d46"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"55dcedf4d51a43584ead73801b19abf0b29e7c79","unresolved":false,"context_lines":[{"line_number":29,"context_line":"driver. This was done by utilizing the NUMA information for these PCIe devices"},{"line_number":30,"context_line":"that was provided by libvirt. However, the use of vSwitch solutions complicates"},{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Depending on how these VIFs related to"},{"line_number":33,"context_line":"underlying virtual switching processes and actual hardware, these VIFs can have"},{"line_number":34,"context_line":"an inherent NUMA topology. This NUMA affinity is not currently accounted for,"},{"line_number":35,"context_line":"which can result in cross-NUMA traffic and significant packet processing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_b894ad45","line":32,"range":{"start_line":32,"start_character":63,"end_line":32,"end_character":70},"updated":"2018-02-19 21:55:12.000000000","message":"\"relate\", or \"are related\"","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":29,"context_line":"driver. This was done by utilizing the NUMA information for these PCIe devices"},{"line_number":30,"context_line":"that was provided by libvirt. However, the use of vSwitch solutions complicates"},{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Depending on how these VIFs related to"},{"line_number":33,"context_line":"underlying virtual switching processes and actual hardware, these VIFs can have"},{"line_number":34,"context_line":"an inherent NUMA topology. This NUMA affinity is not currently accounted for,"},{"line_number":35,"context_line":"which can result in cross-NUMA traffic and significant packet processing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1f9dbf25_1ba41b08","line":32,"range":{"start_line":32,"start_character":63,"end_line":32,"end_character":70},"in_reply_to":"3fa0c359_b894ad45","updated":"2018-03-07 15:48:36.000000000","message":"Done","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"55dcedf4d51a43584ead73801b19abf0b29e7c79","unresolved":false,"context_lines":[{"line_number":38,"context_line":"Let\u0027s use an example using one possible vSwitch solution, OVS-DPDK. Consider"},{"line_number":39,"context_line":"the following network topology:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":".. figure:: /_images/numa-aware-vswitches-1.png"},{"line_number":42,"context_line":"   :width: 100%"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"In the above, we have two guests, each with a number of interfaces, along with"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_7877053c","line":41,"range":{"start_line":41,"start_character":12,"end_line":41,"end_character":47},"updated":"2018-02-19 21:55:12.000000000","message":"This is cool, and you should totally do it... and we should probably remove/soften the template verbiage that insists on ascii diagrams.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":38,"context_line":"Let\u0027s use an example using one possible vSwitch solution, OVS-DPDK. Consider"},{"line_number":39,"context_line":"the following network topology:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":".. figure:: /_images/numa-aware-vswitches-1.png"},{"line_number":42,"context_line":"   :width: 100%"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"In the above, we have two guests, each with a number of interfaces, along with"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1f9dbf25_9bb7cb3b","line":41,"range":{"start_line":41,"start_character":12,"end_line":41,"end_character":47},"in_reply_to":"3fa0c359_7877053c","updated":"2018-03-07 15:48:36.000000000","message":"Heh, good point. I tried ASCII but I couldn\u0027t illustrate this as well as I wanted. Unless someone wants to be a stickler for this, I\u0027ll submit a separate change to remove said verbiage","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"e328235665087c064ccc3c1881deb87532b44ed4","unresolved":false,"context_lines":[{"line_number":39,"context_line":"the following network topology:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":".. figure:: /_images/numa-aware-vswitches-1.png"},{"line_number":42,"context_line":"   :width: 100%"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"In the above, we have two guests, each with a number of interfaces, along with"},{"line_number":45,"context_line":"a number of host interfaces. When using OVS-DPDK, the guest interfaces would"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_08d576b9","line":42,"updated":"2018-02-16 17:23:11.000000000","message":"It\u0027s not a huge deal, the meaning is still clear, but the leftmost entity label in the image has its first letter cut off in most or all of the images.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":39,"context_line":"the following network topology:"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":".. figure:: /_images/numa-aware-vswitches-1.png"},{"line_number":42,"context_line":"   :width: 100%"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"In the above, we have two guests, each with a number of interfaces, along with"},{"line_number":45,"context_line":"a number of host interfaces. When using OVS-DPDK, the guest interfaces would"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1f9dbf25_db96b395","line":42,"in_reply_to":"3fa0c359_08d576b9","updated":"2018-03-07 15:48:36.000000000","message":"Done (I think)","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":111,"context_line":"used. For example::"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"   [pci]"},{"line_number":114,"context_line":"   neutron_bridge_mapping \u003d {"},{"line_number":115,"context_line":"     \"bridge\": \"brq\","},{"line_number":116,"context_line":"     \"address\": \"0000:07:00.0\","},{"line_number":117,"context_line":"   }"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"where:"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_52e9c7b4","line":117,"range":{"start_line":114,"start_character":4,"end_line":117,"end_character":4},"updated":"2018-02-19 13:46:50.000000000","message":"this will be problematic for non bridge based neutron backends.\n\nif we must do this via nova config options can i suggest the following.\n\n[numa]\nneutron_backend_mapping \u003d [{\n     \"physnet\": \"physnet0\",\n     \"tunnel_provider\": \"no\",\n     \"numa_nodes\": [\"0\"],\n     \"name\": \"tenant_data0\"\n   },{\n     \"physnet\": \"physnet1\",\n     \"tunnel_provider\": \"no\",\n     \"numa_nodes\": [\"1\"],\n     \"name\": \"tenant_data1\"\n   },{\n     \"physnet\": \"\",\n     \"tunnel_provider\": \"yes\",\n     \"numa_nodes\": [0,1],\n     \"name\": \"tenant_tunneled_data\"\n   }]\n\nhere we are saying we hat 2 phynets for provider networks\nphysnet0 is on numa node 0\nphysnet1 is on numa node 1\nneither of which will be used for tunneled traffic.\nfinally we have a dedicated backent provider of tunneled\ntraffic that is assocated with 2 numa nodes for example because of the use of multipath routinging or bonds form two\ndifferent numa nodes.\n\nor another example where on interface is used for both provider and tunneled traffic.\n\n[numa]\nneutron_backend_mapping \u003d [{\n     \"physnet\": \"physnet0\",\n     \"tunnel_provider\": \"yes\",\n     \"numa_nodes\": [\"0\"],\n   }]\n\nthis would be more common and is how we generally deploy ovs-dpdk.\n\ni have removed address as not all architrues support pci e.g. some arm systems all nova does not need this info you are including it simply to reuse the exisiting pci code.\n\nfeilds definitions:\n\nphysnet:\nthis is the neutron phyical network associated with the neutron network the port is attached too. this could be added to port binding details if not already present.\nit is only relevent for provider networks(vlan/flat).\n\ntunnel_provider: \nindicate if this backend is used for tunneled traffic e.g. not vlan/flat such as vxlan/gre/geneve...\n\n\"numa_nodes\":\na list of numa nodes associated with this backend. it is a list to cater for cross numa bonds and multipath routing\nNOTE: a smart enough vswitch with a active active cross numa bond could use the numa affinity of the vm interface as an input to the has to enaure no cross numa traffic for inter host traffic. or a dumb one could have os-vif hardcode it using a openflow multipath action and some other ovs fancyness to set the mac affinity to numa local bond peer.\n\nname:\nan optional arbitrary name for easier use in debuging.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"55dcedf4d51a43584ead73801b19abf0b29e7c79","unresolved":false,"context_lines":[{"line_number":111,"context_line":"used. For example::"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"   [pci]"},{"line_number":114,"context_line":"   neutron_bridge_mapping \u003d {"},{"line_number":115,"context_line":"     \"bridge\": \"brq\","},{"line_number":116,"context_line":"     \"address\": \"0000:07:00.0\","},{"line_number":117,"context_line":"   }"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"where:"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_9873e680","line":117,"range":{"start_line":114,"start_character":4,"end_line":117,"end_character":4},"in_reply_to":"3fa0c359_52e9c7b4","updated":"2018-02-19 21:55:12.000000000","message":"+1 to avoiding the [domain:]bus:device.function address.  What\u0027s needed is an identifier that the specific virt knows how to recognize, and nova compute outside of the specific virt needs to leave it the f alone.\n\n(@sean, I don\u0027t see how just physnet+tunnel_provider+numa_nodes is enough for the driver to identify which device should be used.  E.g. if different NUMA nodes own ports on the same physnet.)","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"054c16768c3dbcac0790a0ed44e435c6b1664b82","unresolved":false,"context_lines":[{"line_number":111,"context_line":"used. For example::"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"   [pci]"},{"line_number":114,"context_line":"   neutron_bridge_mapping \u003d {"},{"line_number":115,"context_line":"     \"bridge\": \"brq\","},{"line_number":116,"context_line":"     \"address\": \"0000:07:00.0\","},{"line_number":117,"context_line":"   }"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"where:"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_8c765306","line":117,"range":{"start_line":114,"start_character":4,"end_line":117,"end_character":4},"in_reply_to":"3fa0c359_52e9c7b4","updated":"2018-02-19 20:15:07.000000000","message":"Do we have any other dict/json based configuration options today, other than the universally-hated pci whitelist? IMHO, it would be really nice if we could avoid adding more of these.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"5ba02c79418cae720c4acf8c72f6f780518f9d38","unresolved":false,"context_lines":[{"line_number":111,"context_line":"used. For example::"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"   [pci]"},{"line_number":114,"context_line":"   neutron_bridge_mapping \u003d {"},{"line_number":115,"context_line":"     \"bridge\": \"brq\","},{"line_number":116,"context_line":"     \"address\": \"0000:07:00.0\","},{"line_number":117,"context_line":"   }"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"where:"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_ee2104a1","line":117,"range":{"start_line":114,"start_character":4,"end_line":117,"end_character":4},"in_reply_to":"3fa0c359_78fdbaaf","updated":"2018-02-19 22:28:14.000000000","message":"The accepted alternative is \"don\u0027t do that evil thing\" of asking people to write/maintain json (or equivalent) in config files. If we need a standalone yaml file for this and we can commit to a format for the long term then that\u0027s cool. I still feel like we should be able to detect this stuff and write it into the database though right? Maybe we just need a conf value list of bridges that we look up the address for and write in?","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"55dcedf4d51a43584ead73801b19abf0b29e7c79","unresolved":false,"context_lines":[{"line_number":111,"context_line":"used. For example::"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"   [pci]"},{"line_number":114,"context_line":"   neutron_bridge_mapping \u003d {"},{"line_number":115,"context_line":"     \"bridge\": \"brq\","},{"line_number":116,"context_line":"     \"address\": \"0000:07:00.0\","},{"line_number":117,"context_line":"   }"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"where:"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_78fdbaaf","line":117,"range":{"start_line":114,"start_character":4,"end_line":117,"end_character":4},"in_reply_to":"3fa0c359_8c765306","updated":"2018-02-19 21:55:12.000000000","message":"@dan, are there known/accepted alternatives?  Separate YAML config file?  I\u0027d be on board with starting to standardize this with an eye toward generic device management.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"aef316e3ab0952209f3e397d068a4e40a85a538e","unresolved":false,"context_lines":[{"line_number":111,"context_line":"used. For example::"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"   [pci]"},{"line_number":114,"context_line":"   neutron_bridge_mapping \u003d {"},{"line_number":115,"context_line":"     \"bridge\": \"brq\","},{"line_number":116,"context_line":"     \"address\": \"0000:07:00.0\","},{"line_number":117,"context_line":"   }"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"where:"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_eedca447","line":117,"range":{"start_line":114,"start_character":4,"end_line":117,"end_character":4},"in_reply_to":"3fa0c359_ee2104a1","updated":"2018-02-19 22:43:26.000000000","message":"As long as \"detect\"/\"look up the address for\" means \"ask virt\" and not \"trawl sysfs\".","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"0a22ea374dbfb430ac10f6a9b1dd2a776f3447f9","unresolved":false,"context_lines":[{"line_number":111,"context_line":"used. For example::"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"   [pci]"},{"line_number":114,"context_line":"   neutron_bridge_mapping \u003d {"},{"line_number":115,"context_line":"     \"bridge\": \"brq\","},{"line_number":116,"context_line":"     \"address\": \"0000:07:00.0\","},{"line_number":117,"context_line":"   }"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"where:"},{"line_number":120,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_47fd6faf","line":117,"range":{"start_line":114,"start_character":4,"end_line":117,"end_character":4},"in_reply_to":"3fa0c359_eedca447","updated":"2018-02-20 13:21:02.000000000","message":"the point that i am trying to make is we dont care about nics or bridges as not all neutron backend track both. e.g. calico has no bridges, ovs dose not track nics it ahs a bridge per phynet and its up to the operator to decide how to connect it to the realy network. \n\nFrom a neutron point of view it dose not know if the interface it is using to connect to a tunnel or physnet is a bond or is using multi path routing so makeing nova have to care when neutron does not seams dumb hence why i wanted to remove all references to the uplink nic.\n\nthe reason i chose physnet+tunnel_provider+numa_nodes \nwas that all neutron networks are either l3 only/tunneled networks which are routed or they are provider networks which have l2 connectivity.\n\nso from a nova boot point of view we check the segmentation:type of the neutron network associated with the port then if its flat/vlan we check the segmentation:provider to get the physnet name. in the tunneled case if the segmentation:type is not flat/vlan we know its a tunneled network. we then use that info to look at the reported  numa nodes physnet or tunnel backend.\n\npersonally i think neutron should be providing this info via placement you cannont discover physnet affinity locally on the server via any mechanisium other then a config. It is not discoverable by the virt dirver or sysfs or any other means. \n\nits a completely arbitrary mapping that is defined in the neutron configs so you would have to query the neutron api for the agent report for the local system and parse the physnet mappings from the configuration section. once you have the physnet mapplins you could find the bridge/interface and iterate over all interfaces found to collect the numa nodes.\n\nsimilarly the tunnel local ip is specifed in the agent report but you would then have to check the routing table in the network namespace where the network backend is runing to determin the interface and then lookup the numa affinity.\nfor containeriesed deployments the name space nova is in may not be the same that vms network backend is in hence why i brougt it up.\n\nso this could be automatically discovered with alot of effort and queries to the neutron api but it seams much simpler to modify opentack installer to template out a config or config section.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"054c16768c3dbcac0790a0ed44e435c6b1664b82","unresolved":false,"context_lines":[{"line_number":125,"context_line":"  .. note::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"     This identifier may change if PCI device tracking moves to resource"},{"line_number":128,"context_line":"     providers in this cycle."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"``bridge``"},{"line_number":131,"context_line":"  The neutron bridge that this physical PCI device is attached to and to which"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_bffecc35","line":128,"updated":"2018-02-19 20:15:07.000000000","message":"Hmm, why? Presumably the compute node will need to us this same bit of information to report those things as resources no?","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"55dcedf4d51a43584ead73801b19abf0b29e7c79","unresolved":false,"context_lines":[{"line_number":125,"context_line":"  .. note::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"     This identifier may change if PCI device tracking moves to resource"},{"line_number":128,"context_line":"     providers in this cycle."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"``bridge``"},{"line_number":131,"context_line":"  The neutron bridge that this physical PCI device is attached to and to which"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_b8f4a288","line":128,"in_reply_to":"3fa0c359_bffecc35","updated":"2018-02-19 21:55:12.000000000","message":"Agree.\n\nWe need an identifier, and it would be really neat if that identifier didn\u0027t have to be changed when device tracking moves to placement.\n\nThe only entity that needs to understand the format of that identifier, and be able to map it to specific hardware, is the virt driver.  So whereas libvirt and others may decide to use the [domain:]bus:device.function format for their \"address\", we should not be restricting it to that format in spec or in code.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":141,"context_line":""},{"line_number":142,"context_line":"    $ neutron port-create \u003cnet-id\u003e \\"},{"line_number":143,"context_line":"        --name my_port \\"},{"line_number":144,"context_line":"        --binding:profile type\u003ddict numa_affinity\u003dtrue"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"NUMA affinity can only be specified for a single interface. Attempting to"},{"line_number":147,"context_line":"specify it for multiple interfaces will result in a build failure. The reasons"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_f23a53cb","line":144,"range":{"start_line":144,"start_character":36,"end_line":144,"end_character":54},"updated":"2018-02-19 13:46:50.000000000","message":"if we do this i would also like this to take effect for sriov.\ne.g. allow the pci numa affinity policy based on the value of numa_affinity in the neutron port. so instead of a boolean why not make this numa_policy and reuse the policies defiend for pci passthrough.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":143,"context_line":"        --name my_port \\"},{"line_number":144,"context_line":"        --binding:profile type\u003ddict numa_affinity\u003dtrue"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"NUMA affinity can only be specified for a single interface. Attempting to"},{"line_number":147,"context_line":"specify it for multiple interfaces will result in a build failure. The reasons"},{"line_number":148,"context_line":"for this are outlined later."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The PCI devices identified by ``address`` will be tracked by the PCI manager."},{"line_number":151,"context_line":"When creating a new instance, the PCI device associated with the given"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_12649fe4","line":148,"range":{"start_line":146,"start_character":0,"end_line":148,"end_character":28},"updated":"2018-02-19 13:46:50.000000000","message":"i think this is overly constratided for multi numa guests. if you request nics on multiple numa nodes but have a multi numa guest then we should be able to support that.\n\ni would change this to may result in a build failure.\nin a lot of nfv deployments you will want at least 2 interfaces.\n\nis your resonaing the fact that qemu only create a single pcie root inside the guest as it does not support multi root io virtualisation.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":6962,"name":"Kashyap Chamarthy","email":"kchamart@redhat.com","username":"kashyapc"},"change_message_id":"2269cbf56e6350ca6c7a01efb49c54ead536fe10","unresolved":false,"context_lines":[{"line_number":143,"context_line":"        --name my_port \\"},{"line_number":144,"context_line":"        --binding:profile type\u003ddict numa_affinity\u003dtrue"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"NUMA affinity can only be specified for a single interface. Attempting to"},{"line_number":147,"context_line":"specify it for multiple interfaces will result in a build failure. The reasons"},{"line_number":148,"context_line":"for this are outlined later."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The PCI devices identified by ``address`` will be tracked by the PCI manager."},{"line_number":151,"context_line":"When creating a new instance, the PCI device associated with the given"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_d8ac1937","line":148,"range":{"start_line":146,"start_character":0,"end_line":148,"end_character":28},"in_reply_to":"3fa0c359_12649fe4","updated":"2018-02-19 14:00:23.000000000","message":"On your last point about multiple PCIe root ports, maybe this currently in-progress change helps? -- https://review.openstack.org/#/c/545034/ (\"Allow to configure amount of PCIe ports\")","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e4fe8d625b95ce50f9daced515332074550edb6d","unresolved":false,"context_lines":[{"line_number":143,"context_line":"        --name my_port \\"},{"line_number":144,"context_line":"        --binding:profile type\u003ddict numa_affinity\u003dtrue"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"NUMA affinity can only be specified for a single interface. Attempting to"},{"line_number":147,"context_line":"specify it for multiple interfaces will result in a build failure. The reasons"},{"line_number":148,"context_line":"for this are outlined later."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The PCI devices identified by ``address`` will be tracked by the PCI manager."},{"line_number":151,"context_line":"When creating a new instance, the PCI device associated with the given"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_de6e4195","line":148,"range":{"start_line":146,"start_character":0,"end_line":148,"end_character":28},"in_reply_to":"3fa0c359_d8ac1937","updated":"2018-02-19 15:02:38.000000000","message":"am no that is adresssing the maxium number of slots by using addtionaly root ports  per pcie root complex but does not allow multipel pcie root complexes.\n\nspecifically to emulate real multi numa hardware we would need to create a pcie root complex per socket/numa node.\nNOTE: real hardware tends to have 1 route complex per socket and 1+ numa nodes per socket. in a vm world 1 root complex per numa node makes more sense.\n\nqemu does allow us to create addtional root complexs\n\nhttps://github.com/qemu/qemu/blob/72f1094b6cd8191a5ed842bc57cffeefae95ac1e/docs/pcie.txt#L52-L53\n\nso we would have to create a root complex per numa nodes\n\n-device pxb-pcie,id\u003dpcie.1,bus_nr\u003dx[,numa_node\u003dy][,addr\u003dz]\n\nin libvirt parlance i belive this equates to createing the following per numa node.\n\n\u003ccontroller type\u003d\u0027pci\u0027 index\u003d[(255/number of numanodes*guest virtual numa node)+1] model\u003d\u0027pcie-expander-bus\u0027 node\u003d[guest virtual numa node] /\u003e\n\nif we did that we could correctly model the numa affinity of pci devices assigned to a guest.\n\nthis is rather new and need libvirt 1.3.6 and a failrly recent qemu also but its deffinetly a nice feature to make use of if available.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"054c16768c3dbcac0790a0ed44e435c6b1664b82","unresolved":false,"context_lines":[{"line_number":143,"context_line":"        --name my_port \\"},{"line_number":144,"context_line":"        --binding:profile type\u003ddict numa_affinity\u003dtrue"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"NUMA affinity can only be specified for a single interface. Attempting to"},{"line_number":147,"context_line":"specify it for multiple interfaces will result in a build failure. The reasons"},{"line_number":148,"context_line":"for this are outlined later."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The PCI devices identified by ``address`` will be tracked by the PCI manager."},{"line_number":151,"context_line":"When creating a new instance, the PCI device associated with the given"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_e2dcbb47","line":148,"range":{"start_line":146,"start_character":0,"end_line":148,"end_character":28},"in_reply_to":"3fa0c359_de6e4195","updated":"2018-02-19 20:15:07.000000000","message":"So just to be clear, the reason we let them specify numa_affinity\u003dtrue is so they can choose which one they need to be high-performance, right? If they only have one such nic, wouldn\u0027t we want to assume/default to require affinity for that one? Like, isn\u0027t it better for everyone on the system if you\u0027re affined to your resources?\n\nI ask just because more do_obvious_thing\u003dtrue flags aren\u0027t really helpful, so I want to make sure it\u0027s clear that there\u0027s some reason why this should be an option.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"0a22ea374dbfb430ac10f6a9b1dd2a776f3447f9","unresolved":false,"context_lines":[{"line_number":143,"context_line":"        --name my_port \\"},{"line_number":144,"context_line":"        --binding:profile type\u003ddict numa_affinity\u003dtrue"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"NUMA affinity can only be specified for a single interface. Attempting to"},{"line_number":147,"context_line":"specify it for multiple interfaces will result in a build failure. The reasons"},{"line_number":148,"context_line":"for this are outlined later."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The PCI devices identified by ``address`` will be tracked by the PCI manager."},{"line_number":151,"context_line":"When creating a new instance, the PCI device associated with the given"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_07bf67d0","line":148,"range":{"start_line":146,"start_character":0,"end_line":148,"end_character":28},"in_reply_to":"3fa0c359_e2dcbb47","updated":"2018-02-20 13:21:02.000000000","message":"well it would be a change in behavior. if you defaulted to true but only had a nic on one of the 2 numa nodes of a system the second numa node would never be used for vms unless you specifically created a neutron port with numa_affinity\u003dfalse.\n\nsince the ports in this case are created by tenants i would not want to rely on them to do that so we would have to default to not numa affinitiesing.  you are correct however that if you have nics on all numa nodes it would be better to default numa affinities the interfaces.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":147,"context_line":"specify it for multiple interfaces will result in a build failure. The reasons"},{"line_number":148,"context_line":"for this are outlined later."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The PCI devices identified by ``address`` will be tracked by the PCI manager."},{"line_number":151,"context_line":"When creating a new instance, the PCI device associated with the given"},{"line_number":152,"context_line":"NUMA-affined port will be used. This allows us to utilize the"},{"line_number":153,"context_line":"``NUMATopologyFilter`` to exclude hosts who won\u0027t be able to provide our"},{"line_number":154,"context_line":"requested NUMA affinity."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":".. note::"},{"line_number":157,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_f249f331","line":154,"range":{"start_line":150,"start_character":0,"end_line":154,"end_character":24},"updated":"2018-02-19 13:46:50.000000000","message":"you are assuming all nics will be connected to the pcie bus.\narm for example often does not have a pcie bus and instead uses AMBA bus instead.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"054c16768c3dbcac0790a0ed44e435c6b1664b82","unresolved":false,"context_lines":[{"line_number":195,"context_line":"to have NUMA affinity (via the bridge it is attached to) will be used."},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"The third and final situation is VM to VM traffic. NFV deployments do not care"},{"line_number":198,"context_line":"about VM to VM traffic as it\u0027s highly unlike that two related VNFs would ever"},{"line_number":199,"context_line":"be intentionally collocated on a given host. As such, all traffic is expected"},{"line_number":200,"context_line":"to be physical to virtual traffic."},{"line_number":201,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_9f88b0c6","line":198,"range":{"start_line":198,"start_character":38,"end_line":198,"end_character":44},"updated":"2018-02-19 20:15:07.000000000","message":"unlikely","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":227,"context_line":"that in favour of dynamic generation introduces unnecessary magic to the"},{"line_number":228,"context_line":"formula."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"We could take variants of both of the above and report this NUMA information to"},{"line_number":231,"context_line":"placement itself. This hasn\u0027t been fully evaluated but at first glance it would"},{"line_number":232,"context_line":"not solve any of the problems with either solution and would appear to offer no"},{"line_number":233,"context_line":"improvement over simply passing this information as part of the response from"},{"line_number":234,"context_line":"neutron."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"Finally, and this is the nuclear option, we could simply embrace the fact that"},{"line_number":237,"context_line":"scheduling decisions in the NFV space are tough to do automatically and it"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_586329b2","line":234,"range":{"start_line":230,"start_character":0,"end_line":234,"end_character":8},"updated":"2018-02-19 13:46:50.000000000","message":"haveing neutron model network backends includeing numa affinity of uplinks in placement the passing a request for placement resouces via the neutron port would still be my preference as any other soltion that is neutron drivven has to work for all possible neutron backends equally that includes not bridge based backends like calico and plathforms with out pcie buses. i have given some examples of how that can be done above.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"e328235665087c064ccc3c1881deb87532b44ed4","unresolved":false,"context_lines":[{"line_number":238,"context_line":"might be better to delegate this to another system. To this end, we could"},{"line_number":239,"context_line":"simply expose a \"deploy on this **host** NUMA node\" knob and let users at it."},{"line_number":240,"context_line":"This is distinctly uncloudy but it would simplify our lives and satisfy the"},{"line_number":241,"context_line":"requests of these users."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"Data model impact"},{"line_number":244,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_48f4de5b","line":241,"updated":"2018-02-16 17:23:11.000000000","message":"I\u0027m glad you included this one. It certainly has a lot of appeal some days.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"054c16768c3dbcac0790a0ed44e435c6b1664b82","unresolved":false,"context_lines":[{"line_number":238,"context_line":"might be better to delegate this to another system. To this end, we could"},{"line_number":239,"context_line":"simply expose a \"deploy on this **host** NUMA node\" knob and let users at it."},{"line_number":240,"context_line":"This is distinctly uncloudy but it would simplify our lives and satisfy the"},{"line_number":241,"context_line":"requests of these users."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"Data model impact"},{"line_number":244,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_5f80e88b","line":241,"in_reply_to":"3fa0c359_38d2dd5d","updated":"2018-02-19 20:15:07.000000000","message":"I think we\u0027ve previously said that this is never going to be a thing, so I think we could probably leave it out of this section or at least mention the out-of-scope nature as the reason for excluding it.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":238,"context_line":"might be better to delegate this to another system. To this end, we could"},{"line_number":239,"context_line":"simply expose a \"deploy on this **host** NUMA node\" knob and let users at it."},{"line_number":240,"context_line":"This is distinctly uncloudy but it would simplify our lives and satisfy the"},{"line_number":241,"context_line":"requests of these users."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"Data model impact"},{"line_number":244,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_38d2dd5d","line":241,"in_reply_to":"3fa0c359_48f4de5b","updated":"2018-02-19 13:46:50.000000000","message":"this makes my sad in side but yes it would definetly fix the issue. for some definition of fix... at that point however i think we would also want to let you say what cores to use and where to allocate vm memory from and just have the nova compute agent validate that there is no conflicting request and be done with it. this would totally break live migration and shelving however.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":263,"context_line":"Other end user impact"},{"line_number":264,"context_line":"---------------------"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"Users will need to create ports ahead of time if they wish to define NUMA"},{"line_number":267,"context_line":"affinity. This is similar to the requirements for SR-IOV devices [6]_."},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"Performance Impact"},{"line_number":270,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_9868d187","line":267,"range":{"start_line":266,"start_character":0,"end_line":267,"end_character":70},"updated":"2018-02-19 13:46:50.000000000","message":"i dont think this is contoversial.\nanything beyond a port connected to a subnet or network requires this already.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3940542738e0da9e48c8554cce3ef62bbcb66042","unresolved":false,"context_lines":[{"line_number":269,"context_line":"Performance Impact"},{"line_number":270,"context_line":"------------------"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"There will be a negligible increase in the time taken to schedule an instance"},{"line_number":273,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":274,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_58d6c929","line":273,"range":{"start_line":272,"start_character":0,"end_line":273,"end_character":49},"updated":"2018-02-19 13:46:50.000000000","message":"this will impact resource claims also as we will need to validate these constraints in the conductor before calling to the compute node otherwise we will have increased retries which i a problem for cells.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"0a22ea374dbfb430ac10f6a9b1dd2a776f3447f9","unresolved":false,"context_lines":[{"line_number":269,"context_line":"Performance Impact"},{"line_number":270,"context_line":"------------------"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"There will be a negligible increase in the time taken to schedule an instance"},{"line_number":273,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":274,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_e7611b3b","line":273,"range":{"start_line":272,"start_character":0,"end_line":273,"end_character":49},"in_reply_to":"3fa0c359_422b2f81","updated":"2018-02-20 13:21:02.000000000","message":"to be able to know what interface on a host will be used to handle the traffic for a given guest interfaces a scheduling time we need two things. \n\nFirst wethere we auto discover or just statically configer the numa affinity for the backends  that needs to be storein in the nova db and made availele in the host state obejct so it can be consumed via a filter.\n\nsecond we need to move the creationg/retrival of the neutron port details form the compute node to the super conductor so that we can determing if the interface is connected to a tunneled network or a specific physnet before we call the filter scheduler. \n\nperferable we would do this before placement is called so that we can window the result set pasted to the filter scheduler to only host that are connected to the relevent phynet.\n\nthis would require neutron network backends to be modeled in placement with phynets and tunnel provider atribute modeled as traits on the backend resouce provider.\n\ntechnically the placement piece could be dont later as its just an optimisation but we would still have to implemnt moving the retrival fo the neutorn port to the supper conductor to provided the required info to the filter scheduler.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"054c16768c3dbcac0790a0ed44e435c6b1664b82","unresolved":false,"context_lines":[{"line_number":269,"context_line":"Performance Impact"},{"line_number":270,"context_line":"------------------"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"There will be a negligible increase in the time taken to schedule an instance"},{"line_number":273,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":274,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_422b2f81","line":273,"range":{"start_line":272,"start_character":0,"end_line":273,"end_character":49},"in_reply_to":"3fa0c359_58d6c929","updated":"2018-02-19 20:15:07.000000000","message":"Sean, can you elaborate on this? I\u0027m not sure I follow.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"55dcedf4d51a43584ead73801b19abf0b29e7c79","unresolved":false,"context_lines":[{"line_number":285,"context_line":"This change will only target the libvirt driver. Additional drivers do not"},{"line_number":286,"context_line":"currently provide sufficient awareness of host NUMA information for this change"},{"line_number":287,"context_line":"to be incorporated in a meaningful manner. If this changes in the future, the"},{"line_number":288,"context_line":"feature may need to be extended to support other types of PCI device."},{"line_number":289,"context_line":""},{"line_number":290,"context_line":"Upgrade impact"},{"line_number":291,"context_line":"--------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_1831160c","line":288,"updated":"2018-02-19 21:55:12.000000000","message":"Cool.  Nevertheless, it should be implemented such that libvirt-isms are limited to the libvirt driver.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"054c16768c3dbcac0790a0ed44e435c6b1664b82","unresolved":false,"context_lines":[{"line_number":307,"context_line":""},{"line_number":308,"context_line":"#. Add support for the new configuration option"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"#. Parse the configuration option and use it at scheduling time"},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Dependencies"},{"line_number":313,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_e2cb5ba2","line":310,"updated":"2018-02-19 20:15:07.000000000","message":"Last time around I asked for more detail on what was going to be done here, and while there are words, I don\u0027t feel like I get a better idea of what the change is.\n\nI get that we have a config option and that we need to do stuff with it. However, presumably there is compute-side work to report those new devices you say will be tracked by the PCI manager, is that right? And, the NUMA filter will need to consider those things. Will simply reporting those things cause it to do the right thing, or will it need changes too?\n\nMaybe I\u0027m just lacking enough context to make sense of this, but it still seems really opaque to me. If other spec reviewers are satisfied with the level of detail, then they should push it through.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"0a22ea374dbfb430ac10f6a9b1dd2a776f3447f9","unresolved":false,"context_lines":[{"line_number":307,"context_line":""},{"line_number":308,"context_line":"#. Add support for the new configuration option"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"#. Parse the configuration option and use it at scheduling time"},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Dependencies"},{"line_number":313,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_a7e9d380","line":310,"in_reply_to":"3fa0c359_93bae707","updated":"2018-02-20 13:21:02.000000000","message":"i know that Stephen is advocating reusing the use of the pci manager to do the tracking but in the case of a bound(has not pci address) or multipath routed networks we can assssume we will have pci address so we need something new to track network backend numa affinity unless we are going to limit the scope fo this to ovs like things with a single interface per physnet and 1 interface for tunneling per server.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"55dcedf4d51a43584ead73801b19abf0b29e7c79","unresolved":false,"context_lines":[{"line_number":307,"context_line":""},{"line_number":308,"context_line":"#. Add support for the new configuration option"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"#. Parse the configuration option and use it at scheduling time"},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Dependencies"},{"line_number":313,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3fa0c359_93bae707","line":310,"in_reply_to":"3fa0c359_e2cb5ba2","updated":"2018-02-19 21:55:12.000000000","message":"I agree with Dan.  I thought it was _my_ lack of context, but if Dan doesn\u0027t have enough to grok this, that\u0027s a decent sign it should be explained further.","commit_id":"b382dedf2baf5b421f3b580670cae7020f930c21"},{"author":{"_account_id":17280,"name":"Tim Rozet","email":"trozet@redhat.com","username":"trozet"},"change_message_id":"1ce9503ab1f9b7c07da047982da1071a266a4644","unresolved":false,"context_lines":[{"line_number":125,"context_line":"    backend_mapping \u003d {"},{"line_number":126,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":127,"context_line":"      \"physnet\": None,"},{"line_number":128,"context_line":"      \"tunnel_provider\": True,"},{"line_number":129,"context_line":"      \"numa_nodes\": [0,1],"},{"line_number":130,"context_line":"    }"},{"line_number":131,"context_line":"    backend_mapping \u003d {"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ff6b8bd7_4b54e1ca","line":128,"range":{"start_line":128,"start_character":5,"end_line":128,"end_character":30},"updated":"2018-03-06 19:52:42.000000000","message":"how does nova intend on finding which L3 interface is being used for the tunnel?  For example, in OVS we could be setting tunnel source IP on the vxlan port, so you could look that up to determine which interface is being used for the tunnel.  But what if the OVS is configured to use flow based tunnels (possibility in the future with ODL), where the IP may be set via OpenFlow rules?\n\nMaybe it makes more sense here to specify an IP address of the interface used for the the tunnel?  Then I guess you would need to look up the br-phy bridge with the IP address and determine which DPDK port is on it.  Alternatively for tunnel_provider you could provide a the bridge being used to host the DPDK port, like \"br-phy\" and get the port that way.","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6d9d6a0c7e8bf6a0924b4e3dda01ebbfaadcdbb3","unresolved":false,"context_lines":[{"line_number":125,"context_line":"    backend_mapping \u003d {"},{"line_number":126,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":127,"context_line":"      \"physnet\": None,"},{"line_number":128,"context_line":"      \"tunnel_provider\": True,"},{"line_number":129,"context_line":"      \"numa_nodes\": [0,1],"},{"line_number":130,"context_line":"    }"},{"line_number":131,"context_line":"    backend_mapping \u003d {"}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_2da44508","line":128,"in_reply_to":"","updated":"2018-03-07 03:39:45.000000000","message":"The intent is that nova will not. The operator or preferably the deployment tool that was used to deploy openstack will tell nova what the numa affinity of the tunnel provider iterface is by setting this config option.  Nova cannot easily discover this info itself so it will not try as in a containerized deployment nova-compute may not even share the same network namespace as the network backend so we can reference port or ips as we cannot guarantee the nova compute agent will be able to see either frim its network namespace.","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6d9d6a0c7e8bf6a0924b4e3dda01ebbfaadcdbb3","unresolved":false,"context_lines":[{"line_number":128,"context_line":"      \"tunnel_provider\": True,"},{"line_number":129,"context_line":"      \"numa_nodes\": [0,1],"},{"line_number":130,"context_line":"    }"},{"line_number":131,"context_line":"    backend_mapping \u003d {"},{"line_number":132,"context_line":"      \"name\": \"tenant_tunneled_data_1\","},{"line_number":133,"context_line":"      \"physnet\": \"physnet0\","},{"line_number":134,"context_line":"      \"tunnel_provider\": True,"}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_edf72dd4","line":131,"updated":"2018-03-07 03:39:45.000000000","message":"Currently in neutron you can only have 1 tunnel provider per host to simplfy the parily conneted tunnel mesh required for unicast tunnel protocol. As a result i think this last example can be removed or left as a separate example to show that a backend be both a tunnel and provider network backend","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":128,"context_line":"      \"tunnel_provider\": True,"},{"line_number":129,"context_line":"      \"numa_nodes\": [0,1],"},{"line_number":130,"context_line":"    }"},{"line_number":131,"context_line":"    backend_mapping \u003d {"},{"line_number":132,"context_line":"      \"name\": \"tenant_tunneled_data_1\","},{"line_number":133,"context_line":"      \"physnet\": \"physnet0\","},{"line_number":134,"context_line":"      \"tunnel_provider\": True,"}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_1bcdf65c","line":131,"in_reply_to":"df7087c5_edf72dd4","updated":"2018-03-07 15:48:36.000000000","message":"Good point. Done","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6d9d6a0c7e8bf6a0924b4e3dda01ebbfaadcdbb3","unresolved":false,"context_lines":[{"line_number":168,"context_line":".. note::"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"   A smart enough vSwitch with a active-active cross NUMA bond could use the"},{"line_number":171,"context_line":"   NUMA affinity of the VM interface as an input to the has to ensure no"},{"line_number":172,"context_line":"   cross-NUMA traffic for inter-host traffic. Alternatively, a dumb one could"},{"line_number":173,"context_line":"   have *os-vif* hardcode it using a OpenFlow multipath action and some other"},{"line_number":174,"context_line":"   Open vSwitch fanciness to set the MAC affinity to NUMA local bond peer. This"}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_0dcce187","line":171,"in_reply_to":"","updated":"2018-03-07 03:39:45.000000000","message":"As an input to the hash for selecting the bond peer","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":168,"context_line":".. note::"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"   A smart enough vSwitch with a active-active cross NUMA bond could use the"},{"line_number":171,"context_line":"   NUMA affinity of the VM interface as an input to the has to ensure no"},{"line_number":172,"context_line":"   cross-NUMA traffic for inter-host traffic. Alternatively, a dumb one could"},{"line_number":173,"context_line":"   have *os-vif* hardcode it using a OpenFlow multipath action and some other"},{"line_number":174,"context_line":"   Open vSwitch fanciness to set the MAC affinity to NUMA local bond peer. This"}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_f83a88cb","line":171,"in_reply_to":"df7087c5_0dcce187","updated":"2018-03-07 15:48:36.000000000","message":"Whoops, done","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":17280,"name":"Tim Rozet","email":"trozet@redhat.com","username":"trozet"},"change_message_id":"1ce9503ab1f9b7c07da047982da1071a266a4644","unresolved":false,"context_lines":[{"line_number":227,"context_line":"There are multiple possible solutions to this issue of various levels of"},{"line_number":228,"context_line":"complexity."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"We could store the static backend-NUMA mapping in neutron instead and pass this"},{"line_number":231,"context_line":"over the wire as part of the VIF request. As neutron is responsible for all"},{"line_number":232,"context_line":"things network\u0027y, storing this information in nova might appear to break the"},{"line_number":233,"context_line":"separation of concerns between the two services. However, this is only"},{"line_number":234,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":235,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":236,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":237,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":238,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":239,"context_line":"matters for little to no gain."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"An even more elaborate alternative would be dynamically generate this"},{"line_number":242,"context_line":"information in neutron, rather than hardcoding it in either nova or neutron."}],"source_content_type":"text/x-rst","patch_set":5,"id":"ff6b8bd7_8bae79e5","line":239,"range":{"start_line":230,"start_character":0,"end_line":239,"end_character":30},"updated":"2018-03-06 19:52:42.000000000","message":"Just as a comment inline with this paragraph:  OpenDaylight uses per host configuration stored in OVS (like bridge mappings), which are then read by ODL and populated into Neutron DB as pseudo agents.  An alternative approach would be to store the numa:physnet affinity per network on a per-host basis in OVS, eventually propagating this info to the VIF request for Nova.","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6d9d6a0c7e8bf6a0924b4e3dda01ebbfaadcdbb3","unresolved":false,"context_lines":[{"line_number":236,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":237,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":238,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":239,"context_line":"matters for little to no gain."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"An even more elaborate alternative would be dynamically generate this"},{"line_number":242,"context_line":"information in neutron, rather than hardcoding it in either nova or neutron."}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_4d45f916","line":239,"in_reply_to":"","updated":"2018-03-07 03:39:45.000000000","message":"Network are tenant creatable and the numa topology of the host is not visible to a tenant so i dont see how this is visible in a public cloud context. For private cloud this could work but i think a network level affinity is too granular as it expose too much of the nfvi configuration via the neutron api. Parsing the agent report to read the configuration section for the numa:physnet mapping could work however it requires that neutron must be deployed and upgraded before the nova-compute agent is deployed/upgraded which is a dependency i think we want to avoid.","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"38efe7800e604063566fa57a55d579b442b997d9","unresolved":false,"context_lines":[{"line_number":236,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":237,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":238,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":239,"context_line":"matters for little to no gain."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"An even more elaborate alternative would be dynamically generate this"},{"line_number":242,"context_line":"information in neutron, rather than hardcoding it in either nova or neutron."}],"source_content_type":"text/x-rst","patch_set":5,"id":"df7087c5_7bc67219","line":239,"in_reply_to":"df7087c5_4d45f916","updated":"2018-03-07 15:48:36.000000000","message":"Added this as another alternative, along with the reasons why it won\u0027t work. Let me know if I\u0027ve captured anything incorrectly.","commit_id":"ef272ff56c9c8e78d48557ee4de604d61e67ac17"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Depending on how these VIFs relate to the"},{"line_number":33,"context_line":"underlying virtual switching processes and actual hardware, they can have an"},{"line_number":34,"context_line":"inherent NUMA topology. This NUMA affinity is not currently accounted for,"},{"line_number":35,"context_line":"which can result in cross-NUMA traffic and significant packet processing"},{"line_number":36,"context_line":"performance penalties."},{"line_number":37,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_34e20931","line":34,"range":{"start_line":34,"start_character":0,"end_line":34,"end_character":22},"updated":"2018-03-09 19:06:58.000000000","message":"I don\u0027t think there is any such thing as a vSwitch having an \"inherent NUMA topology\". I think what you mean to say is that \"a vSwitch can have an affinity to a specific NUMA node based on the physical networks the vSwitch is responsible for managing access to\"","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Depending on how these VIFs relate to the"},{"line_number":33,"context_line":"underlying virtual switching processes and actual hardware, they can have an"},{"line_number":34,"context_line":"inherent NUMA topology. This NUMA affinity is not currently accounted for,"},{"line_number":35,"context_line":"which can result in cross-NUMA traffic and significant packet processing"},{"line_number":36,"context_line":"performance penalties."},{"line_number":37,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_19ed708a","line":34,"range":{"start_line":34,"start_character":0,"end_line":34,"end_character":22},"in_reply_to":"df7087c5_34e20931","updated":"2018-03-14 17:14:14.000000000","message":"Yup, this is clearer though still not entirely correct. It\u0027s not so much the entire vSwitch that will have NUMA affinity but rather the networks it manages. If you\u0027ve got two PCI devices and they\u0027re attached to different networks, then each network will gain its NUMA affinity from the PCI device that it\u0027s using","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":58,"context_line":"- One of the NUMA nodes associated with the guest, for a ``dpdkvhostuser`` or"},{"line_number":59,"context_line":"  ``dpdkvhostuserclient`` interface"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"If multiple queues exist (i.e. \"multi-queue), assigning is is done per queue,"},{"line_number":62,"context_line":"ensuring that an individual PMD does not become a bottleneck. Where multiple"},{"line_number":63,"context_line":"PMDs exist for a given NUMA node, as is often the case, assigning is done in a"},{"line_number":64,"context_line":"round-robin fashion. If there is no local (in the sense of NUMA locality) PMD"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_37d78ff0","line":61,"updated":"2018-03-08 15:41:30.000000000","message":"nit: missing \"","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":58,"context_line":"- One of the NUMA nodes associated with the guest, for a ``dpdkvhostuser`` or"},{"line_number":59,"context_line":"  ``dpdkvhostuserclient`` interface"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"If multiple queues exist (i.e. \"multi-queue), assigning is is done per queue,"},{"line_number":62,"context_line":"ensuring that an individual PMD does not become a bottleneck. Where multiple"},{"line_number":63,"context_line":"PMDs exist for a given NUMA node, as is often the case, assigning is done in a"},{"line_number":64,"context_line":"round-robin fashion. If there is no local (in the sense of NUMA locality) PMD"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_59c528f0","line":61,"in_reply_to":"df7087c5_37d78ff0","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":83,"context_line":"    In OVS-DPDK, egress traffic (VM -\u003e Phy) is not particularly costly as the"},{"line_number":84,"context_line":"    PMD dequeuing packets from the guest queue will be located on the same NUMA"},{"line_number":85,"context_line":"    node as the guest queue itself. Meanwhile, the enqueue onto the physical"},{"line_number":86,"context_line":"    network interface\u0027s queue is efficent thanks to DMA. However, the return"},{"line_number":87,"context_line":"    path is much more costly as the enqueue to the guests queue cannot benefit"},{"line_number":88,"context_line":"    from DMA."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"This is the root cause of the issue: clearly the guest should have been put on"},{"line_number":91,"context_line":"the same NUMA node as the NIC it\u0027s using, as seen below. This is what we wish"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_17fdab39","line":88,"range":{"start_line":86,"start_character":57,"end_line":88,"end_character":13},"updated":"2018-03-08 15:41:30.000000000","message":"I thought that OVS-DPDK guests needs to be on shared huge pages so that OVS-DPDK could transfer the packet efficiently into the guest memory.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":83,"context_line":"    In OVS-DPDK, egress traffic (VM -\u003e Phy) is not particularly costly as the"},{"line_number":84,"context_line":"    PMD dequeuing packets from the guest queue will be located on the same NUMA"},{"line_number":85,"context_line":"    node as the guest queue itself. Meanwhile, the enqueue onto the physical"},{"line_number":86,"context_line":"    network interface\u0027s queue is efficent thanks to DMA. However, the return"},{"line_number":87,"context_line":"    path is much more costly as the enqueue to the guests queue cannot benefit"},{"line_number":88,"context_line":"    from DMA."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"This is the root cause of the issue: clearly the guest should have been put on"},{"line_number":91,"context_line":"the same NUMA node as the NIC it\u0027s using, as seen below. This is what we wish"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_f0da6615","line":88,"range":{"start_line":86,"start_character":57,"end_line":88,"end_character":13},"in_reply_to":"df7087c5_17fdab39","updated":"2018-03-14 17:14:14.000000000","message":"Right, but that doesn\u0027t matter if those huge pages are on another node - you\u0027re still going to have cross-NUMA traffic. FYI, hugepages aren\u0027t necessary. As Sean says, \n\n  for vhost user to work the vhost ring buffers and command\n  queues must be located in prealloced,locked,mmapped shared\n  memory with an open file descrptor that can be passed from\n  the vhost front end(in qemu) to the vhost backend in the\n  vswitch/kernel. Hugepages are just the simplest way to do\n  this in OpenStack","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"eda8de4279a3941a21cd429baa83c9ebf243a0d0","unresolved":false,"context_lines":[{"line_number":83,"context_line":"    In OVS-DPDK, egress traffic (VM -\u003e Phy) is not particularly costly as the"},{"line_number":84,"context_line":"    PMD dequeuing packets from the guest queue will be located on the same NUMA"},{"line_number":85,"context_line":"    node as the guest queue itself. Meanwhile, the enqueue onto the physical"},{"line_number":86,"context_line":"    network interface\u0027s queue is efficent thanks to DMA. However, the return"},{"line_number":87,"context_line":"    path is much more costly as the enqueue to the guests queue cannot benefit"},{"line_number":88,"context_line":"    from DMA."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"This is the root cause of the issue: clearly the guest should have been put on"},{"line_number":91,"context_line":"the same NUMA node as the NIC it\u0027s using, as seen below. This is what we wish"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_7bf06968","line":88,"range":{"start_line":86,"start_character":57,"end_line":88,"end_character":13},"in_reply_to":"df7087c5_f0da6615","updated":"2018-03-28 12:41:28.000000000","message":"I learned something new today, thanks. :)","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":88,"context_line":"    from DMA."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"This is the root cause of the issue: clearly the guest should have been put on"},{"line_number":91,"context_line":"the same NUMA node as the NIC it\u0027s using, as seen below. This is what we wish"},{"line_number":92,"context_line":"to resolve with this spec."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":".. figure:: /_images/numa-aware-vswitches-4.png"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_775c274b","line":91,"range":{"start_line":91,"start_character":42,"end_line":91,"end_character":56},"updated":"2018-03-08 15:41:30.000000000","message":"On the 4th picture the guest are not put on different NUMA compared to the 3rd picture but the traffic of the guest was rerouted to use the physical nic from the same NUMA the guest was originally booted on. However I understand the spec that we really want to place the VNF0 guest to NUMA1 and VNF1 guet to NUMA0 with the proposed feature. Do I understand the intention correctly?","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":88,"context_line":"    from DMA."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"This is the root cause of the issue: clearly the guest should have been put on"},{"line_number":91,"context_line":"the same NUMA node as the NIC it\u0027s using, as seen below. This is what we wish"},{"line_number":92,"context_line":"to resolve with this spec."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":".. figure:: /_images/numa-aware-vswitches-4.png"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_997bc0bc","line":91,"range":{"start_line":91,"start_character":42,"end_line":91,"end_character":56},"in_reply_to":"df7087c5_775c274b","updated":"2018-03-14 17:14:14.000000000","message":"Correct. The guest names are incorrect in that image and I\u0027ll update them now.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":121,"context_line":"physical network and, in the case of Open vSwitch, these physical networks are"},{"line_number":122,"context_line":"mapped to a given bridge containing the external interfaces. To this end, we"},{"line_number":123,"context_line":"propose adding a new configuration option, ``[neutron] backend_mapping``, which"},{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_14f105ce","line":124,"range":{"start_line":124,"start_character":49,"end_line":124,"end_character":55},"updated":"2018-03-09 19:06:58.000000000","message":"physnet","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":121,"context_line":"physical network and, in the case of Open vSwitch, these physical networks are"},{"line_number":122,"context_line":"mapped to a given bridge containing the external interfaces. To this end, we"},{"line_number":123,"context_line":"propose adding a new configuration option, ``[neutron] backend_mapping``, which"},{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b4424b3d","line":124,"range":{"start_line":124,"start_character":49,"end_line":124,"end_character":55},"in_reply_to":"df7087c5_14f105ce","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":122,"context_line":"mapped to a given bridge containing the external interfaces. To this end, we"},{"line_number":123,"context_line":"propose adding a new configuration option, ``[neutron] backend_mapping``, which"},{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"},{"line_number":128,"context_line":"    backend_mapping \u003d {"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_f24cc577","line":125,"updated":"2018-03-08 15:41:30.000000000","message":"It feels like this new nova.conf option will contain information that neutron is responsible for.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1d7212379c0dbbc345806eb62a6715012e4798d4","unresolved":false,"context_lines":[{"line_number":122,"context_line":"mapped to a given bridge containing the external interfaces. To this end, we"},{"line_number":123,"context_line":"propose adding a new configuration option, ``[neutron] backend_mapping``, which"},{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"},{"line_number":128,"context_line":"    backend_mapping \u003d {"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_5669771e","line":125,"in_reply_to":"df7087c5_f24cc577","updated":"2018-03-13 16:37:30.000000000","message":"On the hangouts with Jay and Stephen we figured out a nice separation.\n* Physnet, Networking backend - Neutron\n* PCI devices, NUMA - Nova\n\nAnd then the pci or numa filter needs to resolve the nova instance\u0027s networks to physnet and then that to numa to place the instance optimally. The network-\u003e physnet mapping can be done via the neutron API and the nova need to do the physnet -\u003e numa mapping somehow. This is where the config come into the picture. This config maps a neutron concept to a nova concept so there is no perfect place for it. As the scheduler filter is now in nova I\u0027m OK with adding this config to nova.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"},{"line_number":128,"context_line":"    backend_mapping \u003d {"},{"line_number":129,"context_line":"      \"name\": \"tenant_data_0\","},{"line_number":130,"context_line":"      \"physnet\": \"physnet0\","},{"line_number":131,"context_line":"      \"tunnel_provider\": False,"},{"line_number":132,"context_line":"      \"numa_nodes\": [0],"},{"line_number":133,"context_line":"    }"},{"line_number":134,"context_line":"    backend_mapping \u003d {"},{"line_number":135,"context_line":"      \"name\": \"tenant_data_1\","},{"line_number":136,"context_line":"      \"physnet\": \"physnet1\","},{"line_number":137,"context_line":"      \"tunnel_provider\": False,"},{"line_number":138,"context_line":"      \"numa_nodes\": [1],"},{"line_number":139,"context_line":"    }"},{"line_number":140,"context_line":"    backend_mapping \u003d {"},{"line_number":141,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":142,"context_line":"      \"physnet\": None,"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_34c5295c","line":139,"range":{"start_line":127,"start_character":1,"end_line":139,"end_character":5},"updated":"2018-03-09 19:06:58.000000000","message":"In general, I\u0027m opposed to adding yet more oslo.cfg CONF options for stuff that isn\u0027t configuration information. This is more inventory/traits information than configuration information, IMHO.\n\nThis kind of configuration option is similar to the pci_passthrough_alias and pci_passthrough_whitelist, and I think we\u0027ve been clear that we do not wish to add yet more options like that which are just data about hardware that is masquerading as a configuration option.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"77df8bddacd03cf3a9df95978a2f599c78df82a1","unresolved":false,"context_lines":[{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"},{"line_number":128,"context_line":"    backend_mapping \u003d {"},{"line_number":129,"context_line":"      \"name\": \"tenant_data_0\","},{"line_number":130,"context_line":"      \"physnet\": \"physnet0\","},{"line_number":131,"context_line":"      \"tunnel_provider\": False,"},{"line_number":132,"context_line":"      \"numa_nodes\": [0],"},{"line_number":133,"context_line":"    }"},{"line_number":134,"context_line":"    backend_mapping \u003d {"},{"line_number":135,"context_line":"      \"name\": \"tenant_data_1\","},{"line_number":136,"context_line":"      \"physnet\": \"physnet1\","},{"line_number":137,"context_line":"      \"tunnel_provider\": False,"},{"line_number":138,"context_line":"      \"numa_nodes\": [1],"},{"line_number":139,"context_line":"    }"},{"line_number":140,"context_line":"    backend_mapping \u003d {"},{"line_number":141,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":142,"context_line":"      \"physnet\": None,"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_0feafeaa","line":139,"range":{"start_line":127,"start_character":1,"end_line":139,"end_character":5},"in_reply_to":"df7087c5_0f2bde2f","updated":"2018-03-13 18:09:24.000000000","message":"Dan threw this together as an example:\n\nhttps://pastebin.com/zP7eyAZw\n\nI could go for that.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"c9020c94db804c482a0916e7ce4f5e7418f503c0","unresolved":false,"context_lines":[{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"},{"line_number":128,"context_line":"    backend_mapping \u003d {"},{"line_number":129,"context_line":"      \"name\": \"tenant_data_0\","},{"line_number":130,"context_line":"      \"physnet\": \"physnet0\","},{"line_number":131,"context_line":"      \"tunnel_provider\": False,"},{"line_number":132,"context_line":"      \"numa_nodes\": [0],"},{"line_number":133,"context_line":"    }"},{"line_number":134,"context_line":"    backend_mapping \u003d {"},{"line_number":135,"context_line":"      \"name\": \"tenant_data_1\","},{"line_number":136,"context_line":"      \"physnet\": \"physnet1\","},{"line_number":137,"context_line":"      \"tunnel_provider\": False,"},{"line_number":138,"context_line":"      \"numa_nodes\": [1],"},{"line_number":139,"context_line":"    }"},{"line_number":140,"context_line":"    backend_mapping \u003d {"},{"line_number":141,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":142,"context_line":"      \"physnet\": None,"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_562f378d","line":139,"range":{"start_line":127,"start_character":1,"end_line":139,"end_character":5},"in_reply_to":"df7087c5_34c5295c","updated":"2018-03-13 16:04:31.000000000","message":"After a video call, the following configuration option was suggested instead of the above:\n\n [neutron]\n physnet_mapping \u003d {\n    \"phynet0\": {\n        \"name\": \"tenant_data_0\",\n        \"numa_nodes\": [0],\n    },\n    \"phynet1\": {\n        \"name\": \"tenant_data_1\",\n        \"numa_nodes\": [1],\n    },\n    None: {\n        \"name\": \"tenant_tunneled_data\",\n        \"tunneled\": true,\n        \"numa_nodes\": [0, 1],\n    }\n }\n\nThe idea here is that the physnet_mapping CONF option would be a single CONF option instead of multiple backend_mapping CONF option lines. In addition, it more clearly delineates that nova is responsible for the *physical* infrastructure on the host and the Neutron agent is responsible for the *virtual* infrastructure -- i.e. the network backend.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"75e2ab3441b08e59e9ceec8311624c5c7d126c41","unresolved":false,"context_lines":[{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"},{"line_number":128,"context_line":"    backend_mapping \u003d {"},{"line_number":129,"context_line":"      \"name\": \"tenant_data_0\","},{"line_number":130,"context_line":"      \"physnet\": \"physnet0\","},{"line_number":131,"context_line":"      \"tunnel_provider\": False,"},{"line_number":132,"context_line":"      \"numa_nodes\": [0],"},{"line_number":133,"context_line":"    }"},{"line_number":134,"context_line":"    backend_mapping \u003d {"},{"line_number":135,"context_line":"      \"name\": \"tenant_data_1\","},{"line_number":136,"context_line":"      \"physnet\": \"physnet1\","},{"line_number":137,"context_line":"      \"tunnel_provider\": False,"},{"line_number":138,"context_line":"      \"numa_nodes\": [1],"},{"line_number":139,"context_line":"    }"},{"line_number":140,"context_line":"    backend_mapping \u003d {"},{"line_number":141,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":142,"context_line":"      \"physnet\": None,"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d4681fba","line":139,"range":{"start_line":127,"start_character":1,"end_line":139,"end_character":5},"in_reply_to":"df7087c5_562f378d","updated":"2018-03-13 17:17:19.000000000","message":"I thought at the PTG we said this would be cleaner to do as a dynamic section like cinder does for backends, instead of encoding JSON in our config file and requiring admins to edit it by hand?","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4a88f33cc4f6b20fd0a583a9dbb1c71d506683e8","unresolved":false,"context_lines":[{"line_number":124,"context_line":"defines a mapping for a given physical network (*phynet*) to a NUMA node. For"},{"line_number":125,"context_line":"example::"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    [neutron]"},{"line_number":128,"context_line":"    backend_mapping \u003d {"},{"line_number":129,"context_line":"      \"name\": \"tenant_data_0\","},{"line_number":130,"context_line":"      \"physnet\": \"physnet0\","},{"line_number":131,"context_line":"      \"tunnel_provider\": False,"},{"line_number":132,"context_line":"      \"numa_nodes\": [0],"},{"line_number":133,"context_line":"    }"},{"line_number":134,"context_line":"    backend_mapping \u003d {"},{"line_number":135,"context_line":"      \"name\": \"tenant_data_1\","},{"line_number":136,"context_line":"      \"physnet\": \"physnet1\","},{"line_number":137,"context_line":"      \"tunnel_provider\": False,"},{"line_number":138,"context_line":"      \"numa_nodes\": [1],"},{"line_number":139,"context_line":"    }"},{"line_number":140,"context_line":"    backend_mapping \u003d {"},{"line_number":141,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":142,"context_line":"      \"physnet\": None,"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_0f2bde2f","line":139,"range":{"start_line":127,"start_character":1,"end_line":139,"end_character":5},"in_reply_to":"df7087c5_d4681fba","updated":"2018-03-13 18:00:47.000000000","message":"Either way the deployer needs to set this information. It\u0027s not something that is administrative, really. This is information that the deployment tooling / configuration management system will need to inject.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":139,"context_line":"    }"},{"line_number":140,"context_line":"    backend_mapping \u003d {"},{"line_number":141,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":142,"context_line":"      \"physnet\": None,"},{"line_number":143,"context_line":"      \"tunnel_provider\": True,"},{"line_number":144,"context_line":"      \"numa_nodes\": [0,1],"},{"line_number":145,"context_line":"    }"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_741de10d","line":142,"range":{"start_line":142,"start_character":6,"end_line":142,"end_character":22},"updated":"2018-03-09 19:06:58.000000000","message":"Above, you write \"To this end, we propose adding a new configuration option, ``[neutron] backend_mapping``, which defines a mapping for a given physical network (*phynet*) to a NUMA node\". However, this block has no physnet value. So what precisely does this block indicate?","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":139,"context_line":"    }"},{"line_number":140,"context_line":"    backend_mapping \u003d {"},{"line_number":141,"context_line":"      \"name\": \"tenant_tunneled_data_0\","},{"line_number":142,"context_line":"      \"physnet\": None,"},{"line_number":143,"context_line":"      \"tunnel_provider\": True,"},{"line_number":144,"context_line":"      \"numa_nodes\": [0,1],"},{"line_number":145,"context_line":"    }"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_12268bcf","line":142,"range":{"start_line":142,"start_character":6,"end_line":142,"end_character":22},"in_reply_to":"df7087c5_741de10d","updated":"2018-03-14 17:14:14.000000000","message":"This should be clearer now. Let me know if not.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":154,"context_line":"  or flat) to which one or more physical NICs are connected. This may be added"},{"line_number":155,"context_line":"  to port binding details if not already present."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"``tunnel_provider``"},{"line_number":158,"context_line":"  A boolean that indicates if this backend is used for tunneled traffic. It is"},{"line_number":159,"context_line":"  possible for a backend to be both a tunnel and provider network backend. This"},{"line_number":160,"context_line":"  will be true if the backend is used for tunnelled traffic (VXLAN, GRE,"},{"line_number":161,"context_line":"  GENEVE, ...)."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"``numa_nodes``"},{"line_number":164,"context_line":"  An optional list of integers of NUMA nodes associated with this backend. It"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d4a14dab","line":161,"range":{"start_line":157,"start_character":0,"end_line":161,"end_character":15},"updated":"2018-03-09 19:06:58.000000000","message":"what does this attribute have to do with NUMA affinity?","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":154,"context_line":"  or flat) to which one or more physical NICs are connected. This may be added"},{"line_number":155,"context_line":"  to port binding details if not already present."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"``tunnel_provider``"},{"line_number":158,"context_line":"  A boolean that indicates if this backend is used for tunneled traffic. It is"},{"line_number":159,"context_line":"  possible for a backend to be both a tunnel and provider network backend. This"},{"line_number":160,"context_line":"  will be true if the backend is used for tunnelled traffic (VXLAN, GRE,"},{"line_number":161,"context_line":"  GENEVE, ...)."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"``numa_nodes``"},{"line_number":164,"context_line":"  An optional list of integers of NUMA nodes associated with this backend. It"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_32238fbf","line":161,"range":{"start_line":157,"start_character":0,"end_line":161,"end_character":15},"in_reply_to":"df7087c5_d4a14dab","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":166,"context_line":""},{"line_number":167,"context_line":".. note::"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"   In the future, we may wish to enhance this with additional fields, such as"},{"line_number":170,"context_line":"   ``numa_policy``. This is out of scope for this spec, however."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":".. note::"},{"line_number":173,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_7490814f","line":170,"range":{"start_line":169,"start_character":0,"end_line":170,"end_character":64},"updated":"2018-03-09 19:06:58.000000000","message":"even more evidence that this stuff isn\u0027t configuration information but rather inventory/traits information...","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":166,"context_line":""},{"line_number":167,"context_line":".. note::"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"   In the future, we may wish to enhance this with additional fields, such as"},{"line_number":170,"context_line":"   ``numa_policy``. This is out of scope for this spec, however."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":".. note::"},{"line_number":173,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b220ffad","line":170,"range":{"start_line":169,"start_character":0,"end_line":170,"end_character":64},"in_reply_to":"df7087c5_7490814f","updated":"2018-03-14 17:14:14.000000000","message":"Totally agree. I\u0027ve dropped this here. Hopefully by time we need this, we\u0027ll be in a position to most of this feature into placement.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":178,"context_line":"   multipath action and some other Open vSwitch fanciness to set the MAC"},{"line_number":179,"context_line":"   affinity to NUMA local bond peer. This is outside of the scope of this spec."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"This configuration will be generated by an orchestration tool at deployment"},{"line_number":182,"context_line":"time. NUMA affinity will be provided for all interfaces attached to networks"},{"line_number":183,"context_line":"defined in a ``backend_mapping`` option. If a given network is not defined in"},{"line_number":184,"context_line":"any such option, no NUMA affinity will be provided. As with other devices that"},{"line_number":185,"context_line":"provide NUMA affinity, namely PCI or SR-IOV [6]_ devices, attempting to boot an"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d48f0d21","line":182,"range":{"start_line":181,"start_character":0,"end_line":182,"end_character":5},"updated":"2018-03-09 19:06:58.000000000","message":"even more evidence here that this isn\u0027t configuration information but actually is inventory/trait information for resource providers.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":186,"context_line":"instance may fail in a number of cases. These are described later."},{"line_number":187,"context_line":""},{"line_number":188,"context_line":"There are a number of advantages to this approach, chief among them being that"},{"line_number":189,"context_line":"this is actually doable. Identifying which NUMA topology is a difficult problem"},{"line_number":190,"context_line":"that grows in complexity with the design of the networks and VNFs. This is"},{"line_number":191,"context_line":"discussed in further detail below. Using a static configuration option that is"},{"line_number":192,"context_line":"generated at deployment time ensure we have the full amount of information"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d22661b8","line":189,"range":{"start_line":189,"start_character":27,"end_line":189,"end_character":79},"updated":"2018-03-08 15:41:30.000000000","message":"I think a word is missing here.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":186,"context_line":"instance may fail in a number of cases. These are described later."},{"line_number":187,"context_line":""},{"line_number":188,"context_line":"There are a number of advantages to this approach, chief among them being that"},{"line_number":189,"context_line":"this is actually doable. Identifying which NUMA topology is a difficult problem"},{"line_number":190,"context_line":"that grows in complexity with the design of the networks and VNFs. This is"},{"line_number":191,"context_line":"discussed in further detail below. Using a static configuration option that is"},{"line_number":192,"context_line":"generated at deployment time ensure we have the full amount of information"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d58ebd77","line":189,"range":{"start_line":189,"start_character":27,"end_line":189,"end_character":79},"in_reply_to":"df7087c5_d22661b8","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":196,"context_line":"result, there are a number of situations that need special workarounds. The"},{"line_number":197,"context_line":"first arises where physical NICs from different NUMA nodes are bonded. This"},{"line_number":198,"context_line":"will not break the world but is not intended to have anything but poor"},{"line_number":199,"context_line":"performance. This is exact situation can arise in SR-IOV too and generally"},{"line_number":200,"context_line":"occurs because:"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"- The user has misconfigured something"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_54a3fd9e","line":199,"range":{"start_line":199,"start_character":17,"end_line":199,"end_character":26},"updated":"2018-03-09 19:06:58.000000000","message":"s/is exact/exact/","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":196,"context_line":"result, there are a number of situations that need special workarounds. The"},{"line_number":197,"context_line":"first arises where physical NICs from different NUMA nodes are bonded. This"},{"line_number":198,"context_line":"will not break the world but is not intended to have anything but poor"},{"line_number":199,"context_line":"performance. This is exact situation can arise in SR-IOV too and generally"},{"line_number":200,"context_line":"occurs because:"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"- The user has misconfigured something"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_75a7b10c","line":199,"range":{"start_line":199,"start_character":17,"end_line":199,"end_character":26},"in_reply_to":"df7087c5_54a3fd9e","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":203,"context_line":"- The user does not care about optimal performance"},{"line_number":204,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"To work around this, we have defined ``numa_nodes`` as a list, rather than a"},{"line_number":207,"context_line":"single integer."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"The second situation is guests with complex, multi-NIC configuration. Most VNFs"},{"line_number":210,"context_line":"utilize multiple interfaces, some of which will be used for VM management"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_f4bb31c1","line":207,"range":{"start_line":206,"start_character":0,"end_line":207,"end_character":15},"updated":"2018-03-09 19:06:58.000000000","message":"I don\u0027t see how this \"works around this problem\"...","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":203,"context_line":"- The user does not care about optimal performance"},{"line_number":204,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"To work around this, we have defined ``numa_nodes`` as a list, rather than a"},{"line_number":207,"context_line":"single integer."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"The second situation is guests with complex, multi-NIC configuration. Most VNFs"},{"line_number":210,"context_line":"utilize multiple interfaces, some of which will be used for VM management"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_35c4d990","line":207,"range":{"start_line":206,"start_character":0,"end_line":207,"end_character":15},"in_reply_to":"df7087c5_f4bb31c1","updated":"2018-03-14 17:14:14.000000000","message":"This should be clearer now. Let me know if not.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":206,"context_line":"To work around this, we have defined ``numa_nodes`` as a list, rather than a"},{"line_number":207,"context_line":"single integer."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"The second situation is guests with complex, multi-NIC configuration. Most VNFs"},{"line_number":210,"context_line":"utilize multiple interfaces, some of which will be used for VM management"},{"line_number":211,"context_line":"interfaces (SSH, logging, SNMP, ...) while the remainder will be used for"},{"line_number":212,"context_line":"actual data plane interfaces. NUMA affinity doesn\u0027t matter for the former so we"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b4b5b9cf","line":209,"range":{"start_line":209,"start_character":55,"end_line":209,"end_character":68},"updated":"2018-03-09 19:06:58.000000000","message":"configurations","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":206,"context_line":"To work around this, we have defined ``numa_nodes`` as a list, rather than a"},{"line_number":207,"context_line":"single integer."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"The second situation is guests with complex, multi-NIC configuration. Most VNFs"},{"line_number":210,"context_line":"utilize multiple interfaces, some of which will be used for VM management"},{"line_number":211,"context_line":"interfaces (SSH, logging, SNMP, ...) while the remainder will be used for"},{"line_number":212,"context_line":"actual data plane interfaces. NUMA affinity doesn\u0027t matter for the former so we"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_f5b98114","line":209,"range":{"start_line":209,"start_character":55,"end_line":209,"end_character":68},"in_reply_to":"df7087c5_b4b5b9cf","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":218,"context_line":""},{"line_number":219,"context_line":"This exact situation occurs with PCI and SR-IOV devices and here, as with those"},{"line_number":220,"context_line":"devices, we expect users to define a multi-node guest topology or to avoid"},{"line_number":221,"context_line":"specifying NUMA affinity for the non-data plane interfaces,"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"The third and final situation is VM to VM traffic. NFV deployments do not care"},{"line_number":224,"context_line":"about VM to VM traffic as it\u0027s highly unlike that two related VNFs would ever"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_14c6e549","line":221,"range":{"start_line":221,"start_character":58,"end_line":221,"end_character":59},"updated":"2018-03-09 19:06:58.000000000","message":". not ,","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":218,"context_line":""},{"line_number":219,"context_line":"This exact situation occurs with PCI and SR-IOV devices and here, as with those"},{"line_number":220,"context_line":"devices, we expect users to define a multi-node guest topology or to avoid"},{"line_number":221,"context_line":"specifying NUMA affinity for the non-data plane interfaces,"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"The third and final situation is VM to VM traffic. NFV deployments do not care"},{"line_number":224,"context_line":"about VM to VM traffic as it\u0027s highly unlike that two related VNFs would ever"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_75433114","line":221,"range":{"start_line":221,"start_character":58,"end_line":221,"end_character":59},"in_reply_to":"df7087c5_14c6e549","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":221,"context_line":"specifying NUMA affinity for the non-data plane interfaces,"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"The third and final situation is VM to VM traffic. NFV deployments do not care"},{"line_number":224,"context_line":"about VM to VM traffic as it\u0027s highly unlike that two related VNFs would ever"},{"line_number":225,"context_line":"be intentionally collocated on a given host. As such, all traffic is expected"},{"line_number":226,"context_line":"to be physical to virtual traffic."},{"line_number":227,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_34c1a92f","line":224,"range":{"start_line":224,"start_character":38,"end_line":224,"end_character":44},"updated":"2018-03-09 19:06:58.000000000","message":"unlikely","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":221,"context_line":"specifying NUMA affinity for the non-data plane interfaces,"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"The third and final situation is VM to VM traffic. NFV deployments do not care"},{"line_number":224,"context_line":"about VM to VM traffic as it\u0027s highly unlike that two related VNFs would ever"},{"line_number":225,"context_line":"be intentionally collocated on a given host. As such, all traffic is expected"},{"line_number":226,"context_line":"to be physical to virtual traffic."},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"Alternatives"},{"line_number":229,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_34d689f3","line":226,"range":{"start_line":224,"start_character":62,"end_line":226,"end_character":34},"updated":"2018-03-09 19:06:58.000000000","message":"I\u0027m not sold on this statement. A service function chain would actually likely be composed of multiple VMs on the same host and would naturally pass traffic between the VMs on that host as traffic was shaped through the service function chain.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":25733,"name":"Jan Gutter","email":"github@jangutter.com","username":"jangutter"},"change_message_id":"a951a97ba412f8ecae80f9a3685db0c475e29fb4","unresolved":false,"context_lines":[{"line_number":221,"context_line":"specifying NUMA affinity for the non-data plane interfaces,"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"The third and final situation is VM to VM traffic. NFV deployments do not care"},{"line_number":224,"context_line":"about VM to VM traffic as it\u0027s highly unlike that two related VNFs would ever"},{"line_number":225,"context_line":"be intentionally collocated on a given host. As such, all traffic is expected"},{"line_number":226,"context_line":"to be physical to virtual traffic."},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"Alternatives"},{"line_number":229,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_080353c0","line":226,"range":{"start_line":224,"start_character":62,"end_line":226,"end_character":34},"in_reply_to":"df7087c5_34d689f3","updated":"2018-06-05 11:02:10.000000000","message":"I agree. It all depends on how the control plane/traffic mix looks whether it\u0027s better to maintain affinity between VM\u0027s in an SFC or to spread them out. You would have to think of some really odd edge cases for it to be more efficient cross-host.\n\nSpecifying that an instance should have Numa affinity to another instance is orthogonal to this spec, though.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":233,"context_line":""},{"line_number":234,"context_line":"We could store the static backend-NUMA mapping in neutron instead and pass this"},{"line_number":235,"context_line":"over the wire as part of the VIF request. As neutron is responsible for all"},{"line_number":236,"context_line":"things network\u0027y, storing this information in nova might appear to break the"},{"line_number":237,"context_line":"separation of concerns between the two services. However, this is only"},{"line_number":238,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":239,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":240,"context_line":"instances. This does not get nova into the business of actually creating the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_12a059f4","line":237,"range":{"start_line":236,"start_character":18,"end_line":237,"end_character":47},"updated":"2018-03-08 15:41:30.000000000","message":"exactly how I feel about this so far.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1d7212379c0dbbc345806eb62a6715012e4798d4","unresolved":false,"context_lines":[{"line_number":233,"context_line":""},{"line_number":234,"context_line":"We could store the static backend-NUMA mapping in neutron instead and pass this"},{"line_number":235,"context_line":"over the wire as part of the VIF request. As neutron is responsible for all"},{"line_number":236,"context_line":"things network\u0027y, storing this information in nova might appear to break the"},{"line_number":237,"context_line":"separation of concerns between the two services. However, this is only"},{"line_number":238,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":239,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":240,"context_line":"instances. This does not get nova into the business of actually creating the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b6dc93e4","line":237,"range":{"start_line":236,"start_character":18,"end_line":237,"end_character":47},"in_reply_to":"df7087c5_12a059f4","updated":"2018-03-13 16:37:30.000000000","message":"See my comment at L125. I\u0027m convinced that the proposed solution is necessary compromise.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":238,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":239,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":240,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":241,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":242,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":243,"context_line":"matters for little to no gain."},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"An even more elaborate alternative would be dynamically generate this"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_526a5180","line":242,"range":{"start_line":241,"start_character":29,"end_line":242,"end_character":20},"updated":"2018-03-08 15:41:30.000000000","message":"As far as I understand nova will store this data to do a proper placement decision of the guest to NUMA nodes. I agree that placing guest to NUMA nodes are the responsibility of nova but it does not necessary means that nova has to store the data that used for such placement decision. For example in the bandwidth case neutron will create the bandwidth providers, neutron will store the requested bandwidth for a port but nova will be able to do a proper placement decision based on the bandwidth request. \n\nI know that numa affinity is not a solved thing in placement so neutron today cannot provide a placement compatible resource and trait request in the neutron port (as we don\u0027t know how that request will look like). But neutron could provide something today in the port that can be used by the NUMATopologyFilter in nova to do the proper NUMA placement of the guest.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1d7212379c0dbbc345806eb62a6715012e4798d4","unresolved":false,"context_lines":[{"line_number":238,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":239,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":240,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":241,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":242,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":243,"context_line":"matters for little to no gain."},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"An even more elaborate alternative would be dynamically generate this"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_36f2c371","line":242,"range":{"start_line":241,"start_character":29,"end_line":242,"end_character":20},"in_reply_to":"df7087c5_526a5180","updated":"2018-03-13 16:37:30.000000000","message":"I\u0027m convinced during our discussion that the other way around solution would be equally awkward.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":254,"context_line":"dynamic generation introduces unnecessary magic to the formula."},{"line_number":255,"context_line":""},{"line_number":256,"context_line":"We could go all in on placement from a neutron side and start storing"},{"line_number":257,"context_line":"information about the various backends from neutron. nova would then be able to"},{"line_number":258,"context_line":"use this information in order to make its scheduling decisions. This is"},{"line_number":259,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":260,"context_line":"other issues, such as discovering the types of networks available on a compute"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b20e4de3","line":257,"range":{"start_line":257,"start_character":53,"end_line":257,"end_character":54},"updated":"2018-03-08 15:41:30.000000000","message":"nit:N","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":254,"context_line":"dynamic generation introduces unnecessary magic to the formula."},{"line_number":255,"context_line":""},{"line_number":256,"context_line":"We could go all in on placement from a neutron side and start storing"},{"line_number":257,"context_line":"information about the various backends from neutron. nova would then be able to"},{"line_number":258,"context_line":"use this information in order to make its scheduling decisions. This is"},{"line_number":259,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":260,"context_line":"other issues, such as discovering the types of networks available on a compute"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b5954935","line":257,"range":{"start_line":257,"start_character":53,"end_line":257,"end_character":54},"in_reply_to":"df7087c5_b20e4de3","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":258,"context_line":"use this information in order to make its scheduling decisions. This is"},{"line_number":259,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":260,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":261,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b2272d65","line":261,"range":{"start_line":261,"start_character":35,"end_line":261,"end_character":75},"updated":"2018-03-08 15:41:30.000000000","message":"I think this is how bandwidth-aware scheduling is proposed to be solved in https://review.openstack.org/#/c/502306/ .","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":259,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":260,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":261,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b2ff2d66","line":263,"range":{"start_line":262,"start_character":0,"end_line":263,"end_character":19},"updated":"2018-03-08 15:41:30.000000000","message":"I agree. But it does not mean that we cannot use the ideas in the bandwidth-aware spec as a basis to formulate a proper solution for numa-aware-vswitches problem.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":260,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":261,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_9483d5cc","line":263,"range":{"start_line":263,"start_character":7,"end_line":263,"end_character":18},"updated":"2018-03-09 19:06:58.000000000","message":"to be resolved","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1d7212379c0dbbc345806eb62a6715012e4798d4","unresolved":false,"context_lines":[{"line_number":259,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":260,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":261,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_16bbff71","line":263,"range":{"start_line":262,"start_character":0,"end_line":263,"end_character":19},"in_reply_to":"df7087c5_b2ff2d66","updated":"2018-03-13 16:37:30.000000000","message":"I realized that top of the bandwidth spec we would need a definite solution for numa in placement + generic pci modelling in placement . So I see now that this is not that easy as I thought before.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":261,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"},{"line_number":267,"context_line":"mappings), which are then read by ODL and populated into Neutron DB as pseudo"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_b2dc0d47","line":264,"updated":"2018-03-08 15:41:30.000000000","message":"As far as I understand these performance issues are not due to regressions in OpenStack but simply missing features. So I don\u0027t see them as more prioritized as bandwidth-aware scheduling (as not having that also causes performance issues today).","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":261,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"},{"line_number":267,"context_line":"mappings), which are then read by ODL and populated into Neutron DB as pseudo"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_f69ede61","line":264,"in_reply_to":"df7087c5_b2dc0d47","updated":"2018-03-14 17:14:14.000000000","message":"It all depends on who you\u0027re talking to, I guess :) 50% is a drastic hit and makes using OVS-DPDK on a host with multiple NUMA nodes (which is mostly all hosts) a hard sell. If you want the high-performance networking required for NFV, you\u0027re going to have to choose between SR-IOV or a vSwitch solution like OVS-DPDK. SR-IOV comes with a fairly large set of downsides (a total lack of support for live migration, even with CPU pinning fixes/workarounds, for one) which is why these vSwitch solutions exist in the first place. As such, I would consider this of at least equal importance to bandwidth-aware scheduling.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a417884f960fd5da7494fce565388e1a8e6274ee","unresolved":false,"context_lines":[{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"},{"line_number":267,"context_line":"mappings), which are then read by ODL and populated into Neutron DB as pseudo"},{"line_number":268,"context_line":"agents. We could store the *NUMA:physnet* affinity per network on a per-host"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_72a2f59c","line":265,"updated":"2018-03-08 15:41:30.000000000","message":"I think later when neutron will use placement for bandwidth and placement will be capable of handling NUMA we will feel that the numa-aware-vswitches feature needs to be moved to neutron. I don\u0027t know how hard that move will be.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1d7212379c0dbbc345806eb62a6715012e4798d4","unresolved":false,"context_lines":[{"line_number":262,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":263,"context_line":"effort to resolved, if it\u0027s even possible. Given the performance issues being"},{"line_number":264,"context_line":"faced right now, this seems untenable."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"},{"line_number":267,"context_line":"mappings), which are then read by ODL and populated into Neutron DB as pseudo"},{"line_number":268,"context_line":"agents. We could store the *NUMA:physnet* affinity per network on a per-host"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d6b0278d","line":265,"in_reply_to":"df7087c5_72a2f59c","updated":"2018-03-13 16:37:30.000000000","message":"I\u0027ve been convinced that it will be hard to move the pci and numa filter to placement with or without this feature anyhow.","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":268,"context_line":"agents. We could store the *NUMA:physnet* affinity per network on a per-host"},{"line_number":269,"context_line":"basis in OVS, eventually propagating this info to the VIF request for nova."},{"line_number":270,"context_line":"However, this isn\u0027t feasible as networks are tenant creatable and the NUMA"},{"line_number":271,"context_line":"topology of the host is not visible to a tenant. In addition, this relies of"},{"line_number":272,"context_line":"specific features of the network backend which might not be available in other"},{"line_number":273,"context_line":"backends."},{"line_number":274,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_34720913","line":271,"range":{"start_line":271,"start_character":74,"end_line":271,"end_character":76},"updated":"2018-03-09 19:06:58.000000000","message":"on","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":268,"context_line":"agents. We could store the *NUMA:physnet* affinity per network on a per-host"},{"line_number":269,"context_line":"basis in OVS, eventually propagating this info to the VIF request for nova."},{"line_number":270,"context_line":"However, this isn\u0027t feasible as networks are tenant creatable and the NUMA"},{"line_number":271,"context_line":"topology of the host is not visible to a tenant. In addition, this relies of"},{"line_number":272,"context_line":"specific features of the network backend which might not be available in other"},{"line_number":273,"context_line":"backends."},{"line_number":274,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_35493930","line":271,"range":{"start_line":271,"start_character":74,"end_line":271,"end_character":76},"in_reply_to":"df7087c5_34720913","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":276,"context_line":"scheduling decisions in the NFV space are tough to do automatically and it"},{"line_number":277,"context_line":"might be better to delegate this to another system. To this end, we could"},{"line_number":278,"context_line":"simply expose a \"deploy on this **host** NUMA node\" knob and let users at it."},{"line_number":279,"context_line":"This is distinctly uncloudy but it would simplify our lives and satisfy the"},{"line_number":280,"context_line":"requests of these users."},{"line_number":281,"context_line":""},{"line_number":282,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_f4657146","line":279,"range":{"start_line":279,"start_character":0,"end_line":279,"end_character":27},"updated":"2018-03-09 19:06:58.000000000","message":"well, NFV is distinctly uncloudy...","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":346,"context_line":""},{"line_number":347,"context_line":"#. Add support for the new configuration option"},{"line_number":348,"context_line":""},{"line_number":349,"context_line":"#. Add a ``requested_networks`` attribute to the ``RequestSpec`` object, which"},{"line_number":350,"context_line":"   will include a list of networks requested by the instance. This information"},{"line_number":351,"context_line":"   is not used by the scheduler."},{"line_number":352,"context_line":""},{"line_number":353,"context_line":"#. Modify the compute code to read this information from the ``RequestSpec``"},{"line_number":354,"context_line":"   object and compare it to the parsed ``[neutron] backend_mapping`` options on"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_f4ca5118","line":351,"range":{"start_line":349,"start_character":0,"end_line":351,"end_character":32},"updated":"2018-03-09 19:06:58.000000000","message":"wait, say what? this is the first I see this even mentioned in this spec...","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":352,"context_line":""},{"line_number":353,"context_line":"#. Modify the compute code to read this information from the ``RequestSpec``"},{"line_number":354,"context_line":"   object and compare it to the parsed ``[neutron] backend_mapping`` options on"},{"line_number":355,"context_line":"   said node. For each requested networks that is defined in one of these"},{"line_number":356,"context_line":"   options, use the ``numa_nodes`` attribute to determine what host node to"},{"line_number":357,"context_line":"   place the instance on."},{"line_number":358,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_94d195d0","line":355,"range":{"start_line":355,"start_character":33,"end_line":355,"end_character":41},"updated":"2018-03-09 19:06:58.000000000","message":"network","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":352,"context_line":""},{"line_number":353,"context_line":"#. Modify the compute code to read this information from the ``RequestSpec``"},{"line_number":354,"context_line":"   object and compare it to the parsed ``[neutron] backend_mapping`` options on"},{"line_number":355,"context_line":"   said node. For each requested networks that is defined in one of these"},{"line_number":356,"context_line":"   options, use the ``numa_nodes`` attribute to determine what host node to"},{"line_number":357,"context_line":"   place the instance on."},{"line_number":358,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d518fd1e","line":355,"range":{"start_line":355,"start_character":33,"end_line":355,"end_character":41},"in_reply_to":"df7087c5_94d195d0","updated":"2018-03-14 17:14:14.000000000","message":"Done","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f5dbebb559939b4524437542fc9cace13f3b626","unresolved":false,"context_lines":[{"line_number":383,"context_line":""},{"line_number":384,"context_line":".. [1] https://www.slideshare.net/LF_OpenvSwitch/lfovs17ovsdpdk-for-nfv-go-live-feedback/12"},{"line_number":385,"context_line":".. [2] http://docs.openvswitch.org/en/latest/topics/dpdk/vhost-user/"},{"line_number":386,"context_line":".. [3] http://docs.openvswitch.org/en/latest/topics/dpdk/phy/"},{"line_number":387,"context_line":".. [4] http://docs.openvswitch.org/en/latest/topics/dpdk/pmd/"},{"line_number":388,"context_line":".. [5] https://software.intel.com/en-us/articles/vhost-user-numa-awareness-in-open-vswitch-with-dpdk"},{"line_number":389,"context_line":".. [6] https://docs.openstack.org/ocata/networking-guide/config-sriov.html"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_14c0257b","line":386,"range":{"start_line":386,"start_character":7,"end_line":386,"end_character":61},"updated":"2018-03-09 19:06:58.000000000","message":"404 Not Found","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"1db4977e98da4df7b66badfe36f729b07454a9e8","unresolved":false,"context_lines":[{"line_number":383,"context_line":""},{"line_number":384,"context_line":".. [1] https://www.slideshare.net/LF_OpenvSwitch/lfovs17ovsdpdk-for-nfv-go-live-feedback/12"},{"line_number":385,"context_line":".. [2] http://docs.openvswitch.org/en/latest/topics/dpdk/vhost-user/"},{"line_number":386,"context_line":".. [3] http://docs.openvswitch.org/en/latest/topics/dpdk/phy/"},{"line_number":387,"context_line":".. [4] http://docs.openvswitch.org/en/latest/topics/dpdk/pmd/"},{"line_number":388,"context_line":".. [5] https://software.intel.com/en-us/articles/vhost-user-numa-awareness-in-open-vswitch-with-dpdk"},{"line_number":389,"context_line":".. [6] https://docs.openstack.org/ocata/networking-guide/config-sriov.html"}],"source_content_type":"text/x-rst","patch_set":6,"id":"df7087c5_d987f8b4","line":386,"range":{"start_line":386,"start_character":7,"end_line":386,"end_character":61},"in_reply_to":"df7087c5_14c0257b","updated":"2018-03-14 17:14:14.000000000","message":"I submitted patches for these [1] but they haven\u0027t merged yet. I guess I should link to the patches and update one they do merge.\n\n[1] https://mail.openvswitch.org/pipermail/ovs-dev/2018-February/344371.html","commit_id":"4c881bc4808862d21fadd9923a07e0a58246b566"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6d7d6dd03564bbf68fddb5f4a126c4a77099cdc0","unresolved":false,"context_lines":[{"line_number":128,"context_line":"topology."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"Networks with a tunnel overlay, or tunneled networks, also provide external"},{"line_number":131,"context_line":"network connectivity. There can be many tunneled networks only one logical"},{"line_number":132,"context_line":"interface (you can use bonded NICs to provide failover) on a host should be"},{"line_number":133,"context_line":"handling traffic for these networks. This interface will be configured using"},{"line_number":134,"context_line":"the ``[ovs] local_ip`` *neutron* configuration option."}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_a734f4df","line":131,"range":{"start_line":131,"start_character":57,"end_line":131,"end_character":58},"updated":"2018-03-23 18:59:48.000000000","message":"but","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"59af518e58d8d7191d243c3ce3d6b04fa27ba5fd","unresolved":false,"context_lines":[{"line_number":145,"context_line":"    numa_nodes \u003d 0"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"    [neutron_physnet_physnet1]"},{"line_number":148,"context_line":"    numa_nodes \u003d 0,1"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    [neutron_tunnel]"},{"line_number":151,"context_line":"    numa_nodes \u003d 1"}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_5b132c5d","line":148,"updated":"2018-03-27 10:15:13.000000000","message":"Not sure about such use-case where we have physnet0 and physnet1 sharing the same NUMA node.","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"eda8de4279a3941a21cd429baa83c9ebf243a0d0","unresolved":false,"context_lines":[{"line_number":147,"context_line":"    [neutron_physnet_physnet1]"},{"line_number":148,"context_line":"    numa_nodes \u003d 0,1"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    [neutron_tunnel]"},{"line_number":151,"context_line":"    numa_nodes \u003d 1"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"where:"},{"line_number":154,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_063a3433","line":151,"range":{"start_line":150,"start_character":0,"end_line":151,"end_character":18},"updated":"2018-03-28 12:41:28.000000000","message":"can we have multiple tunneling agents on the same compute having different numa affinity?","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f980f1aea380b943adea4c79a1002148c5896af7","unresolved":false,"context_lines":[{"line_number":147,"context_line":"    [neutron_physnet_physnet1]"},{"line_number":148,"context_line":"    numa_nodes \u003d 0,1"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    [neutron_tunnel]"},{"line_number":151,"context_line":"    numa_nodes \u003d 1"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"where:"},{"line_number":154,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_0329a288","line":151,"range":{"start_line":150,"start_character":0,"end_line":151,"end_character":18},"in_reply_to":"bf659307_063a3433","updated":"2018-03-28 13:42:50.000000000","message":"I\u0027m told neutron only allows a single tunnel so I imagine not","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6d7d6dd03564bbf68fddb5f4a126c4a77099cdc0","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":"   A smart enough vSwitch with a active-active cross NUMA bond could use the"},{"line_number":178,"context_line":"   NUMA affinity of the VM interface as an input to the hash for selecting the"},{"line_number":179,"context_line":"   bond peer has to ensure no cross-NUMA traffic for inter-host traffic."},{"line_number":180,"context_line":"   Alternatively, a dumb one could have *os-vif* hardcode it using a OpenFlow"},{"line_number":181,"context_line":"   multipath action and some other Open vSwitch fanciness to set the MAC"},{"line_number":182,"context_line":"   affinity to NUMA local bond peer. This is outside of the scope of this spec."}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_c75fa80c","line":179,"range":{"start_line":179,"start_character":13,"end_line":179,"end_character":16},"updated":"2018-03-23 18:59:48.000000000","message":"delete","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"eda8de4279a3941a21cd429baa83c9ebf243a0d0","unresolved":false,"context_lines":[{"line_number":189,"context_line":"instance may fail in a number of cases. These are described later."},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"In addition to this configuration change, we will need to start including the"},{"line_number":192,"context_line":"list of requested networks in the ``RequestSpec`` object. We require this"},{"line_number":193,"context_line":"information to so we can use it to build the NUMA topology of the instance on"},{"line_number":194,"context_line":"the host."},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_b784b4b2","line":192,"range":{"start_line":192,"start_character":0,"end_line":192,"end_character":49},"updated":"2018-03-28 12:41:28.000000000","message":"Do we need to persist this new field of the RequestSpec in the db? \n\nThe bandwidth spec also mentions some needed extensions for the RequestSpec to store the resource request from the ports. So we might need to sync about the structure we want to add to the RequestSpec. There we proposed not to persist this information but recalculate it every time.\n\nhttps://review.openstack.org/#/c/502306/22/specs/rocky/approved/bandwidth-resource-provider.rst@104","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"5b4af2d60455e8ba9a507c6c42d8803a2e8c2e16","unresolved":false,"context_lines":[{"line_number":189,"context_line":"instance may fail in a number of cases. These are described later."},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"In addition to this configuration change, we will need to start including the"},{"line_number":192,"context_line":"list of requested networks in the ``RequestSpec`` object. We require this"},{"line_number":193,"context_line":"information to so we can use it to build the NUMA topology of the instance on"},{"line_number":194,"context_line":"the host."},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_f743e56a","line":192,"range":{"start_line":192,"start_character":0,"end_line":192,"end_character":49},"in_reply_to":"bf659307_b784b4b2","updated":"2018-03-29 16:21:28.000000000","message":"Yes, that\u0027s my plan. Still working on PoC code but I\u0027ll have something to share next week","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"eda8de4279a3941a21cd429baa83c9ebf243a0d0","unresolved":false,"context_lines":[{"line_number":190,"context_line":""},{"line_number":191,"context_line":"In addition to this configuration change, we will need to start including the"},{"line_number":192,"context_line":"list of requested networks in the ``RequestSpec`` object. We require this"},{"line_number":193,"context_line":"information to so we can use it to build the NUMA topology of the instance on"},{"line_number":194,"context_line":"the host."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"There are a number of advantages to this approach, chief among them being that"}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_97fad072","line":193,"range":{"start_line":193,"start_character":12,"end_line":193,"end_character":14},"updated":"2018-03-28 12:41:28.000000000","message":"nit: delete","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":190,"context_line":""},{"line_number":191,"context_line":"In addition to this configuration change, we will need to start including the"},{"line_number":192,"context_line":"list of requested networks in the ``RequestSpec`` object. We require this"},{"line_number":193,"context_line":"information to so we can use it to build the NUMA topology of the instance on"},{"line_number":194,"context_line":"the host."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"There are a number of advantages to this approach, chief among them being that"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9f6a8fd7_6a433543","line":193,"range":{"start_line":193,"start_character":12,"end_line":193,"end_character":14},"in_reply_to":"bf659307_97fad072","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"59af518e58d8d7191d243c3ce3d6b04fa27ba5fd","unresolved":false,"context_lines":[{"line_number":240,"context_line":"There are multiple possible solutions to this issue of various levels of"},{"line_number":241,"context_line":"complexity."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"We could store the static backend-NUMA mapping in neutron instead and pass this"},{"line_number":244,"context_line":"over the wire as part of the VIF request. As neutron is responsible for all"},{"line_number":245,"context_line":"things network\u0027y, storing this information in nova might appear to break the"},{"line_number":246,"context_line":"separation of concerns between the two services. However, this is only"},{"line_number":247,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":248,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":249,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":250,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":251,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":252,"context_line":"matters for little to no gain."},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"An even more elaborate alternative would be dynamically generate this"},{"line_number":255,"context_line":"information in neutron, rather than hardcoding it in either nova or neutron."}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_1bc0547b","line":252,"range":{"start_line":243,"start_character":0,"end_line":252,"end_character":30},"updated":"2018-03-27 10:15:13.000000000","message":"Adding such complex option in nova.conf does not look right. It seems that we are moving all the complexity to operators.\n\nI would prefer this alternative. Basically each neutron agent could return the NUMA node where the ports are created through the binding details.\n\nIn Nova we will have to add a filter or placement thing to take that into account during scheduling and driver pinning placement.","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"88850ebd196d10374ecf8dd087afb846962ed653","unresolved":false,"context_lines":[{"line_number":240,"context_line":"There are multiple possible solutions to this issue of various levels of"},{"line_number":241,"context_line":"complexity."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"We could store the static backend-NUMA mapping in neutron instead and pass this"},{"line_number":244,"context_line":"over the wire as part of the VIF request. As neutron is responsible for all"},{"line_number":245,"context_line":"things network\u0027y, storing this information in nova might appear to break the"},{"line_number":246,"context_line":"separation of concerns between the two services. However, this is only"},{"line_number":247,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":248,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":249,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":250,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":251,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":252,"context_line":"matters for little to no gain."},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"An even more elaborate alternative would be dynamically generate this"},{"line_number":255,"context_line":"information in neutron, rather than hardcoding it in either nova or neutron."}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_352200a6","line":252,"range":{"start_line":243,"start_character":0,"end_line":252,"end_character":30},"in_reply_to":"bf659307_1bc0547b","updated":"2018-03-27 13:20:22.000000000","message":"I totally agree, however, I don\u0027t think this is possible.  NUMA affinity for a vSwitch is driven from affinity of any NICs used and the switching processes themselves (PMD threads in OVS-DPDK land). In this spec, we\u0027ve chosen to focus on the former, which in most cases are PCI devices. From what I can tell, while it\u0027s possible to get the NUMA affinity of PMD processes, it is not possible to get the NUMA affinity of a  NIC attached to a bridge. We\u0027d need to implement this ourselves, which is something we really don\u0027t want to be doing in neutron. Doing so would require querying sysfs or relying on libvirt host capabilities.\n\nThe better model, which is what jaypipes, gibi and I discussed, was to use placement for this and collaboratively build up this model between nova and neutron. However, placement isn\u0027t there yet hence this solution. I plan to migrate fully away from this once we are in a position to do so.","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"4b52b50d777625ae9aa550bcdcd33810d28c0ae2","unresolved":false,"context_lines":[{"line_number":240,"context_line":"There are multiple possible solutions to this issue of various levels of"},{"line_number":241,"context_line":"complexity."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"We could store the static backend-NUMA mapping in neutron instead and pass this"},{"line_number":244,"context_line":"over the wire as part of the VIF request. As neutron is responsible for all"},{"line_number":245,"context_line":"things network\u0027y, storing this information in nova might appear to break the"},{"line_number":246,"context_line":"separation of concerns between the two services. However, this is only"},{"line_number":247,"context_line":"partially true. While neutron is responsible for creating network elements such"},{"line_number":248,"context_line":"as ports and bridges, it is nova that is responsible for wiring them up to"},{"line_number":249,"context_line":"instances. This does not get nova into the business of actually creating the"},{"line_number":250,"context_line":"underlying network elements, but rather it will now store additional metadata"},{"line_number":251,"context_line":"about said elements. Moving this configuration into neutron would complicate"},{"line_number":252,"context_line":"matters for little to no gain."},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"An even more elaborate alternative would be dynamically generate this"},{"line_number":255,"context_line":"information in neutron, rather than hardcoding it in either nova or neutron."}],"source_content_type":"text/x-rst","patch_set":7,"id":"bf659307_58ed6b89","line":252,"range":{"start_line":243,"start_character":0,"end_line":252,"end_character":30},"in_reply_to":"bf659307_352200a6","updated":"2018-03-27 14:26:15.000000000","message":"\u003e From what I can tell, while it\u0027s possible to get the NUMA affinity of PMD processes, it is not possible to get the NUMA affinity of a NIC attached to a bridge. We\u0027d need to implement this ourselves, which is something we really don\u0027t want to be doing in neutron. Doing so would require querying sysfs or relying on libvirt host capabilities.\n\nMy thinking is that, we don\u0027t need to know where the physical NICs are located. Operators are already going to configure the vSwitch based on which NUMA nodes  NICs are placed. (That seems to be a requirement in such use-case)\n\nThe vhu ports created would be placed on the corresponding NUMA node, It would be responsibility of the agent to query vSwitch to retrieve which NUMA node it is. Then update the binding details to return this information to Nova.","commit_id":"cf092a5ad10adf7f50b73aab342fcb87880e69c7"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-vswitches"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"vSwitches such as `Open vSwitch`_ (with kernel or DPDK datapath), `Lagopus`_,"},{"line_number":14,"context_line":"or `Contrail DPDK vRouter`_ all have some level of NUMA affinity. This can be"},{"line_number":15,"context_line":"incurred because they use one or more physical network interfaces, usually"},{"line_number":16,"context_line":"connected via PCIe, or they use userspace processes that are affined to a given"},{"line_number":17,"context_line":"NUMA node. This NUMA affinity is not currently taken into account when"},{"line_number":18,"context_line":"creating an instance. This can result in up to a 50% performance drop [1]_."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_ef687c88","line":15,"range":{"start_line":14,"start_character":66,"end_line":15,"end_character":8},"updated":"2018-04-26 21:24:48.000000000","message":"not sure the word \"incurred\" is appropriate here. Perhaps consider \"This can occur because...\"?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-vswitches"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"vSwitches such as `Open vSwitch`_ (with kernel or DPDK datapath), `Lagopus`_,"},{"line_number":14,"context_line":"or `Contrail DPDK vRouter`_ all have some level of NUMA affinity. This can be"},{"line_number":15,"context_line":"incurred because they use one or more physical network interfaces, usually"},{"line_number":16,"context_line":"connected via PCIe, or they use userspace processes that are affined to a given"},{"line_number":17,"context_line":"NUMA node. This NUMA affinity is not currently taken into account when"},{"line_number":18,"context_line":"creating an instance. This can result in up to a 50% performance drop [1]_."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_a4f7ddd4","line":15,"range":{"start_line":14,"start_character":66,"end_line":15,"end_character":8},"in_reply_to":"9f6a8fd7_ef687c88","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":27,"context_line":"In :doc:`/specs/kilo/implemented/input-output-based-numa-scheduling`, nova"},{"line_number":28,"context_line":"tackled the problem of NUMA affinity for PCIe devices when using the libvirt"},{"line_number":29,"context_line":"driver. This was done by utilizing the NUMA information for these PCIe devices"},{"line_number":30,"context_line":"that was provided by libvirt. However, the use of vSwitch solutions complicates"},{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Nonetheless, the vSwitch will utilize"},{"line_number":33,"context_line":"physical hardware to access physical networks and will have an affinity to"},{"line_number":34,"context_line":"specific NUMA nodes based on this. This NUMA affinity is not currently"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_cf9c986c","line":31,"range":{"start_line":30,"start_character":30,"end_line":31,"end_character":7},"updated":"2018-04-26 21:24:48.000000000","message":"Might be good to instead say \"However, the use of software switching solutions complicates matters compared to switching that is done in hardware\"?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":27,"context_line":"In :doc:`/specs/kilo/implemented/input-output-based-numa-scheduling`, nova"},{"line_number":28,"context_line":"tackled the problem of NUMA affinity for PCIe devices when using the libvirt"},{"line_number":29,"context_line":"driver. This was done by utilizing the NUMA information for these PCIe devices"},{"line_number":30,"context_line":"that was provided by libvirt. However, the use of vSwitch solutions complicates"},{"line_number":31,"context_line":"matters. In these cases, we do not pass through entire PCI devices or virtual"},{"line_number":32,"context_line":"functions but rather a VIF object. Nonetheless, the vSwitch will utilize"},{"line_number":33,"context_line":"physical hardware to access physical networks and will have an affinity to"},{"line_number":34,"context_line":"specific NUMA nodes based on this. This NUMA affinity is not currently"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_24d2ed35","line":31,"range":{"start_line":30,"start_character":30,"end_line":31,"end_character":7},"in_reply_to":"9f6a8fd7_cf9c986c","updated":"2018-04-30 10:16:42.000000000","message":"Sure, Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":32,"context_line":"functions but rather a VIF object. Nonetheless, the vSwitch will utilize"},{"line_number":33,"context_line":"physical hardware to access physical networks and will have an affinity to"},{"line_number":34,"context_line":"specific NUMA nodes based on this. This NUMA affinity is not currently"},{"line_number":35,"context_line":"accounted for, which can result in cross-NUMA traffic and significant packet"},{"line_number":36,"context_line":"processing performance penalties."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":".. important::"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_4f82c802","line":35,"range":{"start_line":35,"start_character":35,"end_line":35,"end_character":45},"updated":"2018-04-26 21:24:48.000000000","message":"cross-NUMA-node","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":32,"context_line":"functions but rather a VIF object. Nonetheless, the vSwitch will utilize"},{"line_number":33,"context_line":"physical hardware to access physical networks and will have an affinity to"},{"line_number":34,"context_line":"specific NUMA nodes based on this. This NUMA affinity is not currently"},{"line_number":35,"context_line":"accounted for, which can result in cross-NUMA traffic and significant packet"},{"line_number":36,"context_line":"processing performance penalties."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":".. important::"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_e4db954c","line":35,"range":{"start_line":35,"start_character":35,"end_line":35,"end_character":45},"in_reply_to":"9f6a8fd7_4f82c802","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":49,"context_line":"network topology:"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":".. figure:: /_images/numa-aware-vswitches-1.png"},{"line_number":52,"context_line":"   :width: 90%"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"In the above, we have two guests, each with a number of interfaces, along with"},{"line_number":55,"context_line":"a number of host interfaces. When using OVS-DPDK, the guest interfaces would"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_efee3cb3","line":52,"updated":"2018-04-26 21:24:48.000000000","message":"FYI, the \"base (mgmt)\" network in the graphics has a mispelling. \"rovider\" instead of \"provider\". Either that, or the images are being cropped perhaps?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":49,"context_line":"network topology:"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":".. figure:: /_images/numa-aware-vswitches-1.png"},{"line_number":52,"context_line":"   :width: 90%"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"In the above, we have two guests, each with a number of interfaces, along with"},{"line_number":55,"context_line":"a number of host interfaces. When using OVS-DPDK, the guest interfaces would"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_84ec19f4","line":52,"in_reply_to":"9f6a8fd7_efee3cb3","updated":"2018-04-30 10:16:42.000000000","message":"Yup, that\u0027s an issue with the source. Let me see if I can fix","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":81,"context_line":".. figure:: /_images/numa-aware-vswitches-2.png"},{"line_number":82,"context_line":"   :width: 90%"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"What we want to avoid is cross-NUMA traffic. We\u0027re not particularly concerned"},{"line_number":85,"context_line":"with VM \u003c-\u003e VM traffic (more on this in a bit), so such traffic would probably"},{"line_number":86,"context_line":"look something like this:"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_2fd674c5","line":84,"range":{"start_line":84,"start_character":25,"end_line":84,"end_character":35},"updated":"2018-04-26 21:24:48.000000000","message":"cross-NUMA-node","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":81,"context_line":".. figure:: /_images/numa-aware-vswitches-2.png"},{"line_number":82,"context_line":"   :width: 90%"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"What we want to avoid is cross-NUMA traffic. We\u0027re not particularly concerned"},{"line_number":85,"context_line":"with VM \u003c-\u003e VM traffic (more on this in a bit), so such traffic would probably"},{"line_number":86,"context_line":"look something like this:"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_44c04151","line":84,"range":{"start_line":84,"start_character":25,"end_line":84,"end_character":35},"in_reply_to":"9f6a8fd7_2fd674c5","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":93,"context_line":"    In OVS-DPDK, egress traffic (VM -\u003e Phy) is not particularly costly as the"},{"line_number":94,"context_line":"    PMD dequeuing packets from the guest queue will be located on the same NUMA"},{"line_number":95,"context_line":"    node as the guest queue itself. Meanwhile, the enqueue onto the physical"},{"line_number":96,"context_line":"    network interface\u0027s queue is efficent thanks to DMA. However, the return"},{"line_number":97,"context_line":"    path is much more costly as the enqueue to the guests queue cannot benefit"},{"line_number":98,"context_line":"    from DMA."},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_6fd46cbe","line":96,"range":{"start_line":96,"start_character":33,"end_line":96,"end_character":41},"updated":"2018-04-26 21:24:48.000000000","message":"efficient","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":93,"context_line":"    In OVS-DPDK, egress traffic (VM -\u003e Phy) is not particularly costly as the"},{"line_number":94,"context_line":"    PMD dequeuing packets from the guest queue will be located on the same NUMA"},{"line_number":95,"context_line":"    node as the guest queue itself. Meanwhile, the enqueue onto the physical"},{"line_number":96,"context_line":"    network interface\u0027s queue is efficent thanks to DMA. However, the return"},{"line_number":97,"context_line":"    path is much more costly as the enqueue to the guests queue cannot benefit"},{"line_number":98,"context_line":"    from DMA."},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_e4b4f5f4","line":96,"range":{"start_line":96,"start_character":33,"end_line":96,"end_character":41},"in_reply_to":"9f6a8fd7_6fd46cbe","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":97,"context_line":"    path is much more costly as the enqueue to the guests queue cannot benefit"},{"line_number":98,"context_line":"    from DMA."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"This is the root cause of the issue: clearly the guest should have been put on"},{"line_number":101,"context_line":"the same NUMA node as the NIC it\u0027s using, as seen below. This is what we wish"},{"line_number":102,"context_line":"to resolve with this spec."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":".. figure:: /_images/numa-aware-vswitches-4.png"},{"line_number":105,"context_line":"   :width: 90%"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_0fb0508b","line":102,"range":{"start_line":100,"start_character":0,"end_line":102,"end_character":26},"updated":"2018-04-26 21:24:48.000000000","message":"One thing that struck me as I read this is that what if the guest wanted to ingress traffic from physical network 0 in addition to physical network 1?\n\nI mean, in that case, are we expecting that *some* traffic will be penalized and other traffic won\u0027t be, depending on which NUMA node the guest ends up on? Or are we expecting that in these cases (where the guest needs ingress traffic from multiple physical networks that span multiple NICs that are connected to multiple NUMA nodes) that the guest must be configured to have multiple virtual NUMA nodes and those virtual NUMA nodes must be pinned to different host NUMA cells along. In addition to that, I would expect that the guest would need to set up PMD queues that are separately affined to each individual NUMA node. Furthermore, the guest would probably need to request multiple ports that would be consuming bandwidth resources from separate resource providers. And the scheduler would need to find a host that met all of these requirements... you can tell where this is going to end up. :(\n\n[later] I see you discuss this situation below and conclude that we will expect users to ask for a specific guest NUMA topology and manually specify the ports that they don\u0027t care about NUMA affinity for.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":123,"context_line":"provides network objects like networks, subnets, and ports. Ports aren\u0027t"},{"line_number":124,"context_line":"particularly helpful to us as we\u0027re not attaching a given port directly to the"},{"line_number":125,"context_line":"guest. Instead, we will utilize networks or, more specifically, networks that"},{"line_number":126,"context_line":"utilize physical NICs. These are usually external networks, meaning they take"},{"line_number":127,"context_line":"the form of provider networks or networks with a tunnel overlay."},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Provider networks"},{"line_number":130,"context_line":"  Provider networks provide external network connectivity for a host meaning"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_4f6ba8f0","line":127,"range":{"start_line":126,"start_character":23,"end_line":127,"end_character":64},"updated":"2018-04-26 21:24:48.000000000","message":"I\u0027ve been calling these things \"network providers\".","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  In the case of Open vSwitch, these physical networks are mapped to a given"},{"line_number":135,"context_line":"  bridge containing the external interfaces using the ``[ovs] bridge_mappings``"},{"line_number":136,"context_line":"  *neutron* configuration option. *physnet*\\s are not real network objects -"},{"line_number":137,"context_line":"  they\u0027re just arbitrary identifiers or tags which allow you can use to reflect"},{"line_number":138,"context_line":"  your physical (in the sense of actual cabling) network topology in your"},{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_af34c4fc","line":137,"range":{"start_line":137,"start_character":45,"end_line":137,"end_character":71},"updated":"2018-04-26 21:24:48.000000000","message":"which allow deployers to","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  In the case of Open vSwitch, these physical networks are mapped to a given"},{"line_number":135,"context_line":"  bridge containing the external interfaces using the ``[ovs] bridge_mappings``"},{"line_number":136,"context_line":"  *neutron* configuration option. *physnet*\\s are not real network objects -"},{"line_number":137,"context_line":"  they\u0027re just arbitrary identifiers or tags which allow you can use to reflect"},{"line_number":138,"context_line":"  your physical (in the sense of actual cabling) network topology in your"},{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_441841d2","line":137,"range":{"start_line":137,"start_character":45,"end_line":137,"end_character":71},"in_reply_to":"9f6a8fd7_af34c4fc","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":135,"context_line":"  bridge containing the external interfaces using the ``[ovs] bridge_mappings``"},{"line_number":136,"context_line":"  *neutron* configuration option. *physnet*\\s are not real network objects -"},{"line_number":137,"context_line":"  they\u0027re just arbitrary identifiers or tags which allow you can use to reflect"},{"line_number":138,"context_line":"  your physical (in the sense of actual cabling) network topology in your"},{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Tunneled networks"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_4f3968f0","line":138,"range":{"start_line":138,"start_character":2,"end_line":138,"end_character":6},"updated":"2018-04-26 21:24:48.000000000","message":"their","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":135,"context_line":"  bridge containing the external interfaces using the ``[ovs] bridge_mappings``"},{"line_number":136,"context_line":"  *neutron* configuration option. *physnet*\\s are not real network objects -"},{"line_number":137,"context_line":"  they\u0027re just arbitrary identifiers or tags which allow you can use to reflect"},{"line_number":138,"context_line":"  your physical (in the sense of actual cabling) network topology in your"},{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Tunneled networks"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_cf2c98ae","line":138,"range":{"start_line":138,"start_character":69,"end_line":138,"end_character":73},"updated":"2018-04-26 21:24:48.000000000","message":"their","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":135,"context_line":"  bridge containing the external interfaces using the ``[ovs] bridge_mappings``"},{"line_number":136,"context_line":"  *neutron* configuration option. *physnet*\\s are not real network objects -"},{"line_number":137,"context_line":"  they\u0027re just arbitrary identifiers or tags which allow you can use to reflect"},{"line_number":138,"context_line":"  your physical (in the sense of actual cabling) network topology in your"},{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Tunneled networks"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_2413cdba","line":138,"range":{"start_line":138,"start_character":2,"end_line":138,"end_character":6},"in_reply_to":"9f6a8fd7_4f3968f0","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":135,"context_line":"  bridge containing the external interfaces using the ``[ovs] bridge_mappings``"},{"line_number":136,"context_line":"  *neutron* configuration option. *physnet*\\s are not real network objects -"},{"line_number":137,"context_line":"  they\u0027re just arbitrary identifiers or tags which allow you can use to reflect"},{"line_number":138,"context_line":"  your physical (in the sense of actual cabling) network topology in your"},{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Tunneled networks"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_844419b9","line":138,"range":{"start_line":138,"start_character":69,"end_line":138,"end_character":73},"in_reply_to":"9f6a8fd7_cf2c98ae","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Tunneled networks"},{"line_number":142,"context_line":"  Networks with a tunnel overlay, or tunneled networks, also provide external"},{"line_number":143,"context_line":"  network connectivity. There can be many tunneled networks but only one"},{"line_number":144,"context_line":"  logical interface (you can use bonded NICs to provide failover) on a host"},{"line_number":145,"context_line":"  should be handling traffic for these networks. This interface is configured"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_8f18e084","line":142,"range":{"start_line":142,"start_character":56,"end_line":142,"end_character":60},"updated":"2018-04-26 21:24:48.000000000","message":"may also","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":139,"context_line":"  virtual network topology."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Tunneled networks"},{"line_number":142,"context_line":"  Networks with a tunnel overlay, or tunneled networks, also provide external"},{"line_number":143,"context_line":"  network connectivity. There can be many tunneled networks but only one"},{"line_number":144,"context_line":"  logical interface (you can use bonded NICs to provide failover) on a host"},{"line_number":145,"context_line":"  should be handling traffic for these networks. This interface is configured"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_644785be","line":142,"range":{"start_line":142,"start_character":56,"end_line":142,"end_character":60},"in_reply_to":"9f6a8fd7_8f18e084","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b948544577d0c308c47171bcc0ce51228b88df35","unresolved":false,"context_lines":[{"line_number":153,"context_line":"---------------------"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"We propose adding a new configuration option and multiple dynamically-generated"},{"line_number":156,"context_line":"configuration groups. The configuration option, ``[neutron] physnets``, will"},{"line_number":157,"context_line":"list all *physnets* for which you wish to provide NUMA affinity for. The"},{"line_number":158,"context_line":"``[neutron_tunnel]`` configuration group will allow configuration of the"},{"line_number":159,"context_line":"tunneled network while the many ``[neutron_physnet_$PHYSNET]`` configuration"},{"line_number":160,"context_line":"groups, one per each ``$PHYSNET`` in ``[neutron] physnets`` will allow"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_bd52d9dc","line":157,"range":{"start_line":156,"start_character":22,"end_line":157,"end_character":68},"updated":"2018-04-27 11:39:03.000000000","message":"I think it is worth to mention that hierarchical port binding  feature and the multiprovider network feature together makes it possible that a network segment is created dynamically in neutron and the physnet of that segment therefore created on the fly. I guess this means that for this case NUMA affinity cannot be defined.\n\nI think it is enough to document this limitation.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":153,"context_line":"---------------------"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"We propose adding a new configuration option and multiple dynamically-generated"},{"line_number":156,"context_line":"configuration groups. The configuration option, ``[neutron] physnets``, will"},{"line_number":157,"context_line":"list all *physnets* for which you wish to provide NUMA affinity for. The"},{"line_number":158,"context_line":"``[neutron_tunnel]`` configuration group will allow configuration of the"},{"line_number":159,"context_line":"tunneled network while the many ``[neutron_physnet_$PHYSNET]`` configuration"},{"line_number":160,"context_line":"groups, one per each ``$PHYSNET`` in ``[neutron] physnets`` will allow"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_04d569b9","line":157,"range":{"start_line":156,"start_character":22,"end_line":157,"end_character":68},"in_reply_to":"9f6a8fd7_bd52d9dc","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":154,"context_line":""},{"line_number":155,"context_line":"We propose adding a new configuration option and multiple dynamically-generated"},{"line_number":156,"context_line":"configuration groups. The configuration option, ``[neutron] physnets``, will"},{"line_number":157,"context_line":"list all *physnets* for which you wish to provide NUMA affinity for. The"},{"line_number":158,"context_line":"``[neutron_tunnel]`` configuration group will allow configuration of the"},{"line_number":159,"context_line":"tunneled network while the many ``[neutron_physnet_$PHYSNET]`` configuration"},{"line_number":160,"context_line":"groups, one per each ``$PHYSNET`` in ``[neutron] physnets`` will allow"},{"line_number":161,"context_line":"configuration of these *physnets*. Each of these configuration groups will"},{"line_number":162,"context_line":"contain a single configuration option each, ``numa_nodes``, which lists the"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_2aaae2d1","line":159,"range":{"start_line":157,"start_character":69,"end_line":159,"end_character":16},"updated":"2018-04-26 21:24:48.000000000","message":"As mentioned on the patch that adds these configuration options, I\u0027d prefer to not have a separate neutron_tunnel CONF group and instead just have a tunnelled or is_tunnelled or uses_tunnels CONF BoolOpt that would go in the individual neutron_physnet_xxx CONF group that represents the tunnelled network.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":154,"context_line":""},{"line_number":155,"context_line":"We propose adding a new configuration option and multiple dynamically-generated"},{"line_number":156,"context_line":"configuration groups. The configuration option, ``[neutron] physnets``, will"},{"line_number":157,"context_line":"list all *physnets* for which you wish to provide NUMA affinity for. The"},{"line_number":158,"context_line":"``[neutron_tunnel]`` configuration group will allow configuration of the"},{"line_number":159,"context_line":"tunneled network while the many ``[neutron_physnet_$PHYSNET]`` configuration"},{"line_number":160,"context_line":"groups, one per each ``$PHYSNET`` in ``[neutron] physnets`` will allow"},{"line_number":161,"context_line":"configuration of these *physnets*. Each of these configuration groups will"},{"line_number":162,"context_line":"contain a single configuration option each, ``numa_nodes``, which lists the"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_e4cf1527","line":159,"range":{"start_line":157,"start_character":69,"end_line":159,"end_character":16},"in_reply_to":"9f6a8fd7_2aaae2d1","updated":"2018-04-30 10:16:42.000000000","message":"As discussed on the code reviews, these are really quite different things. I\u0027ve expanded this here.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":159,"context_line":"tunneled network while the many ``[neutron_physnet_$PHYSNET]`` configuration"},{"line_number":160,"context_line":"groups, one per each ``$PHYSNET`` in ``[neutron] physnets`` will allow"},{"line_number":161,"context_line":"configuration of these *physnets*. Each of these configuration groups will"},{"line_number":162,"context_line":"contain a single configuration option each, ``numa_nodes``, which lists the"},{"line_number":163,"context_line":"host NUMA nodes to which that network is affined. For example::"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"    [neutron]"},{"line_number":166,"context_line":"    physnets \u003d physnet0,physnet1"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_eac4ea61","line":163,"range":{"start_line":162,"start_character":44,"end_line":163,"end_character":49},"updated":"2018-04-26 21:24:48.000000000","message":"When there is a multi-socket compute host with physical NICs or PFs that are **not affined to any particular NUMA node**, will numa_nodes be empty or will numa_nodes be expected to be the set of all NUMA nodes?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":159,"context_line":"tunneled network while the many ``[neutron_physnet_$PHYSNET]`` configuration"},{"line_number":160,"context_line":"groups, one per each ``$PHYSNET`` in ``[neutron] physnets`` will allow"},{"line_number":161,"context_line":"configuration of these *physnets*. Each of these configuration groups will"},{"line_number":162,"context_line":"contain a single configuration option each, ``numa_nodes``, which lists the"},{"line_number":163,"context_line":"host NUMA nodes to which that network is affined. For example::"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"    [neutron]"},{"line_number":166,"context_line":"    physnets \u003d physnet0,physnet1"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_644ea5c2","line":163,"range":{"start_line":162,"start_character":44,"end_line":163,"end_character":49},"in_reply_to":"9f6a8fd7_eac4ea61","updated":"2018-04-30 10:16:42.000000000","message":"I think the the former would make more sense. Clarified","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":180,"context_line":"  A list of strings corresponding to the names of all neutron provider network"},{"line_number":181,"context_line":"  *physnet*\\s (VLAN or flat) configured on this host."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"  A boolean that indicates if the backend is used for tunneled traffic (VXLAN,"},{"line_number":184,"context_line":"  GRE, GENEVE, ...). It is possible for a backend to be both a tunnel and"},{"line_number":185,"context_line":"  provider network backend."},{"line_number":186,"context_line":""},{"line_number":187,"context_line":"``[neutron_physnet_$PHYSNET]``"},{"line_number":188,"context_line":"  A section of opts corresponding to one of the *physnet*\\s defined in"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_2a6f4293","line":185,"range":{"start_line":183,"start_character":0,"end_line":185,"end_character":27},"updated":"2018-04-26 21:24:48.000000000","message":"it seems this is out of place somehow? Or is it missing the name of the proposed CONF option? Either way, I\u0027d prefer to have this be a CONF option in one of the neutron_physnet_xxx CONF groups.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":180,"context_line":"  A list of strings corresponding to the names of all neutron provider network"},{"line_number":181,"context_line":"  *physnet*\\s (VLAN or flat) configured on this host."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"  A boolean that indicates if the backend is used for tunneled traffic (VXLAN,"},{"line_number":184,"context_line":"  GRE, GENEVE, ...). It is possible for a backend to be both a tunnel and"},{"line_number":185,"context_line":"  provider network backend."},{"line_number":186,"context_line":""},{"line_number":187,"context_line":"``[neutron_physnet_$PHYSNET]``"},{"line_number":188,"context_line":"  A section of opts corresponding to one of the *physnet*\\s defined in"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_841df9be","line":185,"range":{"start_line":183,"start_character":0,"end_line":185,"end_character":27},"in_reply_to":"9f6a8fd7_2a6f4293","updated":"2018-04-30 10:16:42.000000000","message":"I meant to delete this entire thing. It\u0027s removed now.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":202,"context_line":"       vSwitch fanciness to set the MAC affinity to NUMA local bond peer. This"},{"line_number":203,"context_line":"       is outside of the scope of this spec."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"``[neutron_tunnel]``"},{"line_number":206,"context_line":"  The same as ``[neutron_physnet_$PHYSNET]`` but for the tunnel networks"},{"line_number":207,"context_line":"  (VXLAN, GRE, GENEVE, ...). It is possible for a backend to be both a tunnel"},{"line_number":208,"context_line":"  and provider network backend."},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"This configuration will be generated by an orchestration tool at deployment"},{"line_number":211,"context_line":"time in the same way that general host network configuration is generated."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_aa0832d9","line":208,"range":{"start_line":205,"start_character":0,"end_line":208,"end_character":31},"updated":"2018-04-26 21:24:48.000000000","message":"I\u0027m pretty confused by this.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b948544577d0c308c47171bcc0ce51228b88df35","unresolved":false,"context_lines":[{"line_number":229,"context_line":""},{"line_number":230,"context_line":"The network API currently provides a method,"},{"line_number":231,"context_line":"``create_pci_requests_for_sriov_ports``, which retrieves information about"},{"line_number":232,"context_line":"*physnets* and VNIC types of each port requested when booting an instance for"},{"line_number":233,"context_line":"the purpose of creating PCI requests for SRIOV ports. This will be made more"},{"line_number":234,"context_line":"generic and extended to get information about the tunneled status of any"},{"line_number":235,"context_line":"requested networks. This information will need to be stored as part of the"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_3d26c969","line":232,"range":{"start_line":232,"start_character":0,"end_line":232,"end_character":11},"updated":"2018-04-27 11:39:03.000000000","message":"It only properly handles the case when a port is associated to a single physnet as the TODO describes here:\nhttps://github.com/openstack/nova/blob/881103c9bfc2606ec9e1b7ae9447c88f0d1813be/nova/network/neutronv2/api.py#L1537\n\nSee more at my above comment","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":229,"context_line":""},{"line_number":230,"context_line":"The network API currently provides a method,"},{"line_number":231,"context_line":"``create_pci_requests_for_sriov_ports``, which retrieves information about"},{"line_number":232,"context_line":"*physnets* and VNIC types of each port requested when booting an instance for"},{"line_number":233,"context_line":"the purpose of creating PCI requests for SRIOV ports. This will be made more"},{"line_number":234,"context_line":"generic and extended to get information about the tunneled status of any"},{"line_number":235,"context_line":"requested networks. This information will need to be stored as part of the"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_a4235dbe","line":232,"range":{"start_line":232,"start_character":0,"end_line":232,"end_character":11},"in_reply_to":"9f6a8fd7_3d26c969","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":227,"context_line":"Network API"},{"line_number":228,"context_line":"-----------"},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"The network API currently provides a method,"},{"line_number":231,"context_line":"``create_pci_requests_for_sriov_ports``, which retrieves information about"},{"line_number":232,"context_line":"*physnets* and VNIC types of each port requested when booting an instance for"},{"line_number":233,"context_line":"the purpose of creating PCI requests for SRIOV ports. This will be made more"},{"line_number":234,"context_line":"generic and extended to get information about the tunneled status of any"},{"line_number":235,"context_line":"requested networks. This information will need to be stored as part of the"},{"line_number":236,"context_line":"``NetworkRequest`` object for later consumption."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_aa3112a5","line":236,"range":{"start_line":230,"start_character":0,"end_line":236,"end_character":48},"updated":"2018-04-26 21:24:48.000000000","message":"Be aware of a similar aim in the network-bandwidth-scheduling spec:\n\nhttps://review.openstack.org/#/c/502306/24/specs/rocky/approved/bandwidth-resource-provider.rst\n\nSee around line 146","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b948544577d0c308c47171bcc0ce51228b88df35","unresolved":false,"context_lines":[{"line_number":227,"context_line":"Network API"},{"line_number":228,"context_line":"-----------"},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"The network API currently provides a method,"},{"line_number":231,"context_line":"``create_pci_requests_for_sriov_ports``, which retrieves information about"},{"line_number":232,"context_line":"*physnets* and VNIC types of each port requested when booting an instance for"},{"line_number":233,"context_line":"the purpose of creating PCI requests for SRIOV ports. This will be made more"},{"line_number":234,"context_line":"generic and extended to get information about the tunneled status of any"},{"line_number":235,"context_line":"requested networks. This information will need to be stored as part of the"},{"line_number":236,"context_line":"``NetworkRequest`` object for later consumption."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_2b173af0","line":236,"range":{"start_line":230,"start_character":0,"end_line":236,"end_character":48},"in_reply_to":"9f6a8fd7_aa3112a5","updated":"2018-04-27 11:39:03.000000000","message":"I think both Stephen and I are aware of the fact that we will step on each other\u0027s toes. Stephen already have code up for this change so I can adapt to that when I start implementing the bandwidth related one.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"fa06dc8227a1321ec438d3c411e42280b2958b04","unresolved":false,"context_lines":[{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":242,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":243,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":244,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":245,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":246,"context_line":"have something to reference during the *claim* stage."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_81b0a7df","line":243,"range":{"start_line":241,"start_character":0,"end_line":243,"end_character":7},"updated":"2018-04-27 13:18:27.000000000","message":"This caches data (physnet of a neutron network) that belongs to neutron and can change in neutron. How do you make sure this data is up-to-date later during a VM move operation?\nWould it be safer to always ask Neutron when we need this information instead of persisting it?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"1b63798e8ee3114021c09cfedf6e1ca3430181b9","unresolved":false,"context_lines":[{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":242,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":243,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":244,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":245,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":246,"context_line":"have something to reference during the *claim* stage."}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_e5c2cbe9","line":243,"range":{"start_line":241,"start_character":0,"end_line":243,"end_character":7},"in_reply_to":"5f7c97a3_83899d27","updated":"2018-05-04 08:58:25.000000000","message":"I tried to think about any kind of VM move operations. So we agree that the CPU pinning works with cold migration.\n\nHowever if the cold migrated VM has a port with NUMA affinity on the source host then the same NUMA affinity will be applied on the destination host, regardless of the destination host NUMA config.\n\nDo I understand the limitation correctly?\n\nIf yes, then I think we at least state this limitation clearly in the spec.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"69bb1500d1f60e915c927cf3b6d9fe7276fe2fef","unresolved":false,"context_lines":[{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":242,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":243,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":244,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":245,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":246,"context_line":"have something to reference during the *claim* stage."}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_f16ccd9d","line":243,"range":{"start_line":241,"start_character":0,"end_line":243,"end_character":7},"in_reply_to":"5f7c97a3_e5c2cbe9","updated":"2018-05-09 15:23:06.000000000","message":"\u003e I tried to think about any kind of VM move operations. So we agree\n \u003e that the CPU pinning works with cold migration.\n \u003e \n \u003e However if the cold migrated VM has a port with NUMA affinity on\n \u003e the source host then the same NUMA affinity will be applied on the\n \u003e destination host, regardless of the destination host NUMA config.\n \u003e \n \u003e Do I understand the limitation correctly?\n \u003e \n \u003e If yes, then I think we at least state this limitation clearly in\n \u003e the spec.\n\nI don\u0027t think so. When we move the instance to the new host, we should be regenerating the guest NUMA topology. In doing so, we will use the new host\u0027s NUMA-network mapping. The only thing we _do_ need to do, I guess, is update any networks","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"805d844e55cac4cbef38e0decd9db5b740905277","unresolved":false,"context_lines":[{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":242,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":243,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":244,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":245,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":246,"context_line":"have something to reference during the *claim* stage."}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_83899d27","line":243,"range":{"start_line":241,"start_character":0,"end_line":243,"end_character":7},"in_reply_to":"9f6a8fd7_0e5a3229","updated":"2018-05-03 15:14:58.000000000","message":"\u003e Do you mean migrating instances with CPU pinning might not work as\n \u003e the pcpu ids are not recalculated on the target host? I thought\n \u003e that issue was fixed more or less already in [1].\n \u003e \n \u003e [1] https://bugs.launchpad.net/nova/+bug/1417667\n\nLive migrating is still an issue. I assumed you were referring to that, no?\n\nhttps://bugs.launchpad.net/nova/+bug/1417667/comments/47","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":242,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":243,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":244,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":245,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":246,"context_line":"have something to reference during the *claim* stage."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_e412b562","line":243,"range":{"start_line":241,"start_character":0,"end_line":243,"end_character":7},"in_reply_to":"9f6a8fd7_81b0a7df","updated":"2018-04-30 10:16:42.000000000","message":"I hadn\u0027t really considered that because it\u0027s not possible to migrate instances with CPU pinning at the moment. I imagine we\u0027ll need to update the cache when we do said migrations.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"5145e8c02fde97b0ec17aa35951f63cb15c69c75","unresolved":false,"context_lines":[{"line_number":238,"context_line":"Scheduling"},{"line_number":239,"context_line":"----------"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":242,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":243,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":244,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":245,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":246,"context_line":"have something to reference during the *claim* stage."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_0e5a3229","line":243,"range":{"start_line":241,"start_character":0,"end_line":243,"end_character":7},"in_reply_to":"9f6a8fd7_e412b562","updated":"2018-05-02 15:05:10.000000000","message":"Do you mean migrating instances with CPU pinning might not work as the pcpu ids are not recalculated on the target host? I thought that issue was fixed more or less already in [1].\n\n[1] https://bugs.launchpad.net/nova/+bug/1417667","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"fa06dc8227a1321ec438d3c411e42280b2958b04","unresolved":false,"context_lines":[{"line_number":241,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":242,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":243,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":244,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":245,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":246,"context_line":"have something to reference during the *claim* stage."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"Virtualization driver support"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_a1b5abcf","line":245,"range":{"start_line":244,"start_character":51,"end_line":245,"end_character":52},"updated":"2018-04-27 13:18:27.000000000","message":"Same question as in L243.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":268,"context_line":""},{"line_number":269,"context_line":"- The user has misconfigured something"},{"line_number":270,"context_line":"- The user does not care about optimal performance"},{"line_number":271,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Given that we can\u0027t determine if this was intentional or not, we have defined"},{"line_number":274,"context_line":"``numa_nodes`` as a list rather than a single integer."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_8a88ce22","line":271,"range":{"start_line":271,"start_character":2,"end_line":271,"end_character":68},"updated":"2018-04-26 21:24:48.000000000","message":"This is a very common scenario.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":268,"context_line":""},{"line_number":269,"context_line":"- The user has misconfigured something"},{"line_number":270,"context_line":"- The user does not care about optimal performance"},{"line_number":271,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Given that we can\u0027t determine if this was intentional or not, we have defined"},{"line_number":274,"context_line":"``numa_nodes`` as a list rather than a single integer."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_27750f99","line":271,"range":{"start_line":271,"start_character":2,"end_line":271,"end_character":68},"in_reply_to":"9f6a8fd7_6be21204","updated":"2018-04-30 10:16:42.000000000","message":"I\u0027ve clarified this","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b948544577d0c308c47171bcc0ce51228b88df35","unresolved":false,"context_lines":[{"line_number":268,"context_line":""},{"line_number":269,"context_line":"- The user has misconfigured something"},{"line_number":270,"context_line":"- The user does not care about optimal performance"},{"line_number":271,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Given that we can\u0027t determine if this was intentional or not, we have defined"},{"line_number":274,"context_line":"``numa_nodes`` as a list rather than a single integer."}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_6be21204","line":271,"range":{"start_line":271,"start_character":2,"end_line":271,"end_character":68},"in_reply_to":"9f6a8fd7_8a88ce22","updated":"2018-04-27 11:39:03.000000000","message":"\"The user wants to provide resiliency by spreading work across NICs\" and the user don\u0027t know/care about the performance impact of cross-NUMA communication.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":270,"context_line":"- The user does not care about optimal performance"},{"line_number":271,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Given that we can\u0027t determine if this was intentional or not, we have defined"},{"line_number":274,"context_line":"``numa_nodes`` as a list rather than a single integer."},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"The second situation is guests with complex, multi-NIC configurations. Most"},{"line_number":277,"context_line":"VNFs utilize multiple interfaces, some of which will be used for VM management"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_0a7cbe1e","line":274,"range":{"start_line":273,"start_character":0,"end_line":274,"end_character":54},"updated":"2018-04-26 21:24:48.000000000","message":"may a physical NIC ever be affined to more than one NUMA node in the real world?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b948544577d0c308c47171bcc0ce51228b88df35","unresolved":false,"context_lines":[{"line_number":270,"context_line":"- The user does not care about optimal performance"},{"line_number":271,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Given that we can\u0027t determine if this was intentional or not, we have defined"},{"line_number":274,"context_line":"``numa_nodes`` as a list rather than a single integer."},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"The second situation is guests with complex, multi-NIC configurations. Most"},{"line_number":277,"context_line":"VNFs utilize multiple interfaces, some of which will be used for VM management"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_ebbf02ce","line":274,"range":{"start_line":273,"start_character":0,"end_line":274,"end_character":54},"in_reply_to":"9f6a8fd7_0a7cbe1e","updated":"2018-04-27 11:39:03.000000000","message":"@Jay: that numa_nodes list defines the affinity of a physnet not an affinity of a physical device. If a physnet is served by a two bonded NIC from two different NUMA then that physnet has a mixed affinity.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":270,"context_line":"- The user does not care about optimal performance"},{"line_number":271,"context_line":"- The user wants to provide resiliency by spreading work across NICs"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Given that we can\u0027t determine if this was intentional or not, we have defined"},{"line_number":274,"context_line":"``numa_nodes`` as a list rather than a single integer."},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"The second situation is guests with complex, multi-NIC configurations. Most"},{"line_number":277,"context_line":"VNFs utilize multiple interfaces, some of which will be used for VM management"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_87465b4f","line":274,"range":{"start_line":273,"start_character":0,"end_line":274,"end_character":54},"in_reply_to":"9f6a8fd7_ebbf02ce","updated":"2018-04-30 10:16:42.000000000","message":"I\u0027ve clarified this","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":287,"context_line":"devices, we expect users to define a multi-node guest topology or to avoid"},{"line_number":288,"context_line":"specifying NUMA affinity for the non-data plane interfaces."},{"line_number":289,"context_line":""},{"line_number":290,"context_line":"The third and final situation is VM to VM traffic. Current NFV deployments do"},{"line_number":291,"context_line":"not care about VM to VM traffic as it\u0027s highly unlikely that two related VNFs"},{"line_number":292,"context_line":"would ever be intentionally collocated on a given host. As such, VM to VM"},{"line_number":293,"context_line":"traffic is considered out of scope for this spec."},{"line_number":294,"context_line":""},{"line_number":295,"context_line":"Alternatives"},{"line_number":296,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_8a60ce37","line":293,"range":{"start_line":290,"start_character":0,"end_line":293,"end_character":49},"updated":"2018-04-26 21:24:48.000000000","message":"I\u0027m really skeptical about this claim :)\n\nMultiple VNFs that comprise a service function chain would actually likely be collocated on a single compute node. Heck, for \"edge computing\" there may only *be* a couple compute nodes...\n\nBut anyway, not -1\u0027ing for this. Just noting my skepticism ;)","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":320,"context_line":"the overall network design at deployment time and discarding that in favour of"},{"line_number":321,"context_line":"dynamic generation introduces unnecessary magic to the formula."},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"We could go all in on placement from a neutron side and start storing"},{"line_number":324,"context_line":"information about the various backends from neutron. Nova would then be able to"},{"line_number":325,"context_line":"use this information in order to make its scheduling decisions. This is"},{"line_number":326,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":327,"context_line":"other issues, such as discovering the types of networks available on a compute"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_8ac52e0d","line":324,"range":{"start_line":323,"start_character":0,"end_line":324,"end_character":52},"updated":"2018-04-26 21:24:48.000000000","message":"This is already happening with the network bandwidth providers stuff, FYI.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":320,"context_line":"the overall network design at deployment time and discarding that in favour of"},{"line_number":321,"context_line":"dynamic generation introduces unnecessary magic to the formula."},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"We could go all in on placement from a neutron side and start storing"},{"line_number":324,"context_line":"information about the various backends from neutron. Nova would then be able to"},{"line_number":325,"context_line":"use this information in order to make its scheduling decisions. This is"},{"line_number":326,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":327,"context_line":"other issues, such as discovering the types of networks available on a compute"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_07748b98","line":324,"range":{"start_line":323,"start_character":0,"end_line":324,"end_character":52},"in_reply_to":"9f6a8fd7_8ac52e0d","updated":"2018-04-30 10:16:42.000000000","message":"Aye, but NUMA is not a part of this. I\u0027d like to note that but can\u0027t do so until the spec is approved, I imagine","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b948544577d0c308c47171bcc0ce51228b88df35","unresolved":false,"context_lines":[{"line_number":327,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":328,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":329,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_4b938e5d","line":330,"range":{"start_line":330,"start_character":20,"end_line":330,"end_character":41},"updated":"2018-04-27 11:39:03.000000000","message":"I have fate in bandwidth-aware scheduling ;)","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":327,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":328,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":329,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_2ad662c5","line":330,"range":{"start_line":330,"start_character":10,"end_line":330,"end_character":18},"updated":"2018-04-26 21:24:48.000000000","message":"be resolved","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":327,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":328,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":329,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_676227db","line":330,"range":{"start_line":330,"start_character":10,"end_line":330,"end_character":18},"in_reply_to":"9f6a8fd7_2ad662c5","updated":"2018-04-30 10:16:42.000000000","message":"Done","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"5145e8c02fde97b0ec17aa35951f63cb15c69c75","unresolved":false,"context_lines":[{"line_number":327,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":328,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":329,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_6eb88e29","line":330,"range":{"start_line":330,"start_character":20,"end_line":330,"end_character":41},"in_reply_to":"9f6a8fd7_479fe3b3","updated":"2018-05-02 15:05:10.000000000","message":"right, numa is not considered there.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":327,"context_line":"other issues, such as discovering the types of networks available on a compute"},{"line_number":328,"context_line":"node _before_ landing on that node and possibly bandwidth-aware scheduling."},{"line_number":329,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_479fe3b3","line":330,"range":{"start_line":330,"start_character":20,"end_line":330,"end_character":41},"in_reply_to":"9f6a8fd7_4b938e5d","updated":"2018-04-30 10:16:42.000000000","message":"\u003e I have fate in bandwidth-aware scheduling ;)\n\nAs do I but afaict you\u0027re not modeling NUMA there, right?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"2ea956216ac4db846c40a14303c5a0bc584c43e5","unresolved":false,"context_lines":[{"line_number":329,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"},{"line_number":334,"context_line":"some of this information to placement in the future."},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_4ae8767b","line":333,"range":{"start_line":332,"start_character":37,"end_line":333,"end_character":32},"updated":"2018-04-26 21:24:48.000000000","message":"I\u0027m presuming the \"performance issues being faced\" are referring to tenant data plane performance issues you are seeing in the field and not performance of the placement or scheduler service?","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"87489bf611360a4db9ae3bd63cefd3e2c1f5a048","unresolved":false,"context_lines":[{"line_number":329,"context_line":"However, it is a huge issue with its own issues and would require a multi-cycle"},{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"},{"line_number":334,"context_line":"some of this information to placement in the future."},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_e7a397ff","line":333,"range":{"start_line":332,"start_character":37,"end_line":333,"end_character":32},"in_reply_to":"9f6a8fd7_4ae8767b","updated":"2018-04-30 10:16:42.000000000","message":"Correct. Clarified","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b948544577d0c308c47171bcc0ce51228b88df35","unresolved":false,"context_lines":[{"line_number":330,"context_line":"effort to resolved, if it\u0027s even possible. We don\u0027t currently have a clear"},{"line_number":331,"context_line":"story for modelling NUMA resources in placement and neutron is not currently"},{"line_number":332,"context_line":"communicating anything to placement. Given the performance issues being faced"},{"line_number":333,"context_line":"right now, this seems untenable. However, that\u0027s not to say we can\u0027t migrate"},{"line_number":334,"context_line":"some of this information to placement in the future."},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"OpenDaylight (ODL) uses per host configuration stored in OVS (like bridge"},{"line_number":337,"context_line":"mappings), which are then read by ODL and populated into Neutron DB as pseudo"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9f6a8fd7_4b7cee1f","line":334,"range":{"start_line":333,"start_character":33,"end_line":334,"end_character":52},"updated":"2018-04-27 11:39:03.000000000","message":"I\u0027m totally OK to consider the transition to a placement based solution later.","commit_id":"b6f3c20b867c5050a06d2c1182c3dd3a044fb522"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"5f7c97a3_850c6e94","updated":"2018-05-15 16:02:55.000000000","message":"In multiple places in this file you use either \"tunneled\" or \"tunnelled\". IMHO, the former is the correct form, but it should be consistent regardless (especially given this is going on an object).","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"5f7c97a3_010b5b29","in_reply_to":"5f7c97a3_850c6e94","updated":"2018-05-16 16:12:04.000000000","message":"US English strikes again. Tunneled it is","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":21,"context_line":".. _Lagopus: http://www.lagopus.org/"},{"line_number":22,"context_line":".. _Contrail DPDK vRouter: https://www.juniper.net/documentation/en_US/contrail4.1/topics/concept/dpdk-with-vrouter-vnc-40.html"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":".. contents::"},{"line_number":25,"context_line":"   :local:"},{"line_number":26,"context_line":"   :depth: 2"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Problem description"},{"line_number":29,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_f4d87a1e","line":26,"range":{"start_line":24,"start_character":0,"end_line":26,"end_character":12},"updated":"2018-05-15 14:39:31.000000000","message":"Is this intentional? We don\u0027t have any other specs that do this since we already have the navigation sidebar in the nova-specs repo that shows the toc:\n\nhttp://logs.openstack.org/90/541290/11/check/build-openstack-sphinx-docs/5a5af59/html/specs/rocky/approved/numa-aware-vswitches.html","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":21,"context_line":".. _Lagopus: http://www.lagopus.org/"},{"line_number":22,"context_line":".. _Contrail DPDK vRouter: https://www.juniper.net/documentation/en_US/contrail4.1/topics/concept/dpdk-with-vrouter-vnc-40.html"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":".. contents::"},{"line_number":25,"context_line":"   :local:"},{"line_number":26,"context_line":"   :depth: 2"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Problem description"},{"line_number":29,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5c013a04","line":26,"range":{"start_line":24,"start_character":0,"end_line":26,"end_character":12},"in_reply_to":"5f7c97a3_f4d87a1e","updated":"2018-05-16 16:12:04.000000000","message":"It was, but I hadn\u0027t even noticed the sidebar. Dropped now.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":73,"context_line":"- One of the NUMA nodes associated with the guest, for a ``dpdkvhostuser`` or"},{"line_number":74,"context_line":"  ``dpdkvhostuserclient`` interface"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"If multiple queues exist (i.e. multi-queue), assigning is is done per queue,"},{"line_number":77,"context_line":"ensuring that an individual PMD does not become a bottleneck. Where multiple"},{"line_number":78,"context_line":"PMDs exist for a given NUMA node, as is often the case, assigning is done in a"},{"line_number":79,"context_line":"round-robin fashion. If there is no local (in the sense of NUMA locality) PMD"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_546a8e19","line":76,"range":{"start_line":76,"start_character":55,"end_line":76,"end_character":60},"updated":"2018-05-15 14:39:31.000000000","message":"is","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":73,"context_line":"- One of the NUMA nodes associated with the guest, for a ``dpdkvhostuser`` or"},{"line_number":74,"context_line":"  ``dpdkvhostuserclient`` interface"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"If multiple queues exist (i.e. multi-queue), assigning is is done per queue,"},{"line_number":77,"context_line":"ensuring that an individual PMD does not become a bottleneck. Where multiple"},{"line_number":78,"context_line":"PMDs exist for a given NUMA node, as is often the case, assigning is done in a"},{"line_number":79,"context_line":"round-robin fashion. If there is no local (in the sense of NUMA locality) PMD"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_1c07421a","line":76,"range":{"start_line":76,"start_character":55,"end_line":76,"end_character":60},"in_reply_to":"5f7c97a3_546a8e19","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":116,"context_line":"  tuned, high-performance vSwitch solution"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"* As an operator, I don\u0027t want to revert to custom hacks like dynamically"},{"line_number":119,"context_line":"  adjusting ``vcpu_pin_set`` just so I can deliver the performance my customers"},{"line_number":120,"context_line":"  demand."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_d4c3feeb","line":119,"range":{"start_line":119,"start_character":14,"end_line":119,"end_character":26},"updated":"2018-05-15 14:39:31.000000000","message":"nit: link to https://docs.openstack.org/nova/latest/configuration/config.html#DEFAULT.vcpu_pin_set","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":116,"context_line":"  tuned, high-performance vSwitch solution"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"* As an operator, I don\u0027t want to revert to custom hacks like dynamically"},{"line_number":119,"context_line":"  adjusting ``vcpu_pin_set`` just so I can deliver the performance my customers"},{"line_number":120,"context_line":"  demand."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_fc0fe62a","line":119,"range":{"start_line":119,"start_character":14,"end_line":119,"end_character":26},"in_reply_to":"5f7c97a3_d4c3feeb","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":128,"context_line":"provides network objects like networks, subnets, and ports. Ports aren\u0027t"},{"line_number":129,"context_line":"particularly helpful to us as we\u0027re not attaching a given port directly to the"},{"line_number":130,"context_line":"guest. Instead, we will utilize networks or, more specifically, networks that"},{"line_number":131,"context_line":"utilize physical NICs. These are two types of network in neutron and these can"},{"line_number":132,"context_line":"offer external connectivity in different ways."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"Provider networks"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_54afee28","line":131,"range":{"start_line":131,"start_character":23,"end_line":131,"end_character":28},"updated":"2018-05-15 14:39:31.000000000","message":"There","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":128,"context_line":"provides network objects like networks, subnets, and ports. Ports aren\u0027t"},{"line_number":129,"context_line":"particularly helpful to us as we\u0027re not attaching a given port directly to the"},{"line_number":130,"context_line":"guest. Instead, we will utilize networks or, more specifically, networks that"},{"line_number":131,"context_line":"utilize physical NICs. These are two types of network in neutron and these can"},{"line_number":132,"context_line":"offer external connectivity in different ways."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"Provider networks"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_bc15ee3e","line":131,"range":{"start_line":131,"start_character":23,"end_line":131,"end_character":28},"in_reply_to":"5f7c97a3_54afee28","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"be7531cf33720449962bdeaeec58ab0b1ee675a8","unresolved":false,"context_lines":[{"line_number":187,"context_line":""},{"line_number":188,"context_line":"While provider networks are clearly configured differently from tenant"},{"line_number":189,"context_line":"networks, this is more to do with *who* can configure them (users for tenant"},{"line_number":190,"context_line":"networks, admin for both) and *where* that configuration is found (generally"},{"line_number":191,"context_line":"configuration files, but an admin can explicitly override this). The difference"},{"line_number":192,"context_line":"between physical networks and tunnelled networks is greater though and it\u0027s"},{"line_number":193,"context_line":"these *types* of network that we choose to focus on."},{"line_number":194,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_6d12e20b","line":191,"range":{"start_line":190,"start_character":67,"end_line":191,"end_character":18},"updated":"2018-05-08 19:35:49.000000000","message":"\"for provider networks, generally configure files, ...\"","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":187,"context_line":""},{"line_number":188,"context_line":"While provider networks are clearly configured differently from tenant"},{"line_number":189,"context_line":"networks, this is more to do with *who* can configure them (users for tenant"},{"line_number":190,"context_line":"networks, admin for both) and *where* that configuration is found (generally"},{"line_number":191,"context_line":"configuration files, but an admin can explicitly override this). The difference"},{"line_number":192,"context_line":"between physical networks and tunnelled networks is greater though and it\u0027s"},{"line_number":193,"context_line":"these *types* of network that we choose to focus on."},{"line_number":194,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_7c2d1693","line":191,"range":{"start_line":190,"start_character":67,"end_line":191,"end_character":18},"in_reply_to":"5f7c97a3_6d12e20b","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"be7531cf33720449962bdeaeec58ab0b1ee675a8","unresolved":false,"context_lines":[{"line_number":196,"context_line":"  As noted above, physical networks or *physnet*\\s are identified through the"},{"line_number":197,"context_line":"  use of an arbitrary label. In the case of the Open vSwitch (OVS) ML2 driver,"},{"line_number":198,"context_line":"  these labels are mapped to a given OVS bridge containing the physical"},{"line_number":199,"context_line":"  interfaces using the ``[ovs] bridge_mappings`` configuration option. For"},{"line_number":200,"context_line":"  example:"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"  .. code-block:: ini"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_4ddda640","line":199,"range":{"start_line":199,"start_character":49,"end_line":199,"end_character":62},"updated":"2018-05-08 19:35:49.000000000","message":"\"neutron configuration option\" (important to be explicit in this spec since we talk about both neutron and nova conf options)","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":196,"context_line":"  As noted above, physical networks or *physnet*\\s are identified through the"},{"line_number":197,"context_line":"  use of an arbitrary label. In the case of the Open vSwitch (OVS) ML2 driver,"},{"line_number":198,"context_line":"  these labels are mapped to a given OVS bridge containing the physical"},{"line_number":199,"context_line":"  interfaces using the ``[ovs] bridge_mappings`` configuration option. For"},{"line_number":200,"context_line":"  example:"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"  .. code-block:: ini"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_fc3846d3","line":199,"range":{"start_line":199,"start_character":49,"end_line":199,"end_character":62},"in_reply_to":"5f7c97a3_4ddda640","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":241,"context_line":"configuration groups."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"* The ``[neutron] physnets`` configuration option will list all *physnets* for"},{"line_number":244,"context_line":"  which you wish to provide NUMA affinity for."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"* The ``[neutron_tunnel]`` configuration group will allow configuration of the"},{"line_number":247,"context_line":"  tunneled networks. Only one configuration group is required for these since"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_b701a407","line":244,"range":{"start_line":244,"start_character":42,"end_line":244,"end_character":45},"updated":"2018-05-15 14:39:31.000000000","message":"s/for//","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":241,"context_line":"configuration groups."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"* The ``[neutron] physnets`` configuration option will list all *physnets* for"},{"line_number":244,"context_line":"  which you wish to provide NUMA affinity for."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"* The ``[neutron_tunnel]`` configuration group will allow configuration of the"},{"line_number":247,"context_line":"  tunneled networks. Only one configuration group is required for these since"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_7c52f614","line":244,"range":{"start_line":244,"start_character":42,"end_line":244,"end_character":45},"in_reply_to":"5f7c97a3_b701a407","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":251,"context_line":""},{"line_number":252,"context_line":"* Multiple ``[neutron_physnet_$PHYSNET]`` configuration groups, one per each"},{"line_number":253,"context_line":"  ``$PHYSNET`` in ``[neutron] physnets``, will allow configuration of these"},{"line_number":254,"context_line":"  *physnets*. Each of these configuration groups will contain a single"},{"line_number":255,"context_line":"  configuration option each, ``numa_nodes``, which lists the host NUMA node(s)"},{"line_number":256,"context_line":"  to which networks using this *physnet* are affined."},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"The groups will all be generated dynamically, which is required as the values"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_ba08a991","line":255,"range":{"start_line":254,"start_character":49,"end_line":255,"end_character":27},"updated":"2018-05-15 16:02:55.000000000","message":"If it\u0027s just a single option, then generating a group for it might be overkill. I was thinking we\u0027d likely have more than one option per physnet group, and I thought there had been other things we needed to assign, per-physnet, in the past?\n\nI don\u0027t want to force us into a more complicated config situation just for one option per physnet (we could generate option names within [neutron] instead of groups I imagine). Having one group per physnet surely will come in handy later though don\u0027t you think?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a1b2b73a6eb6b2b2cb00acc28c91609386a4236f","unresolved":false,"context_lines":[{"line_number":251,"context_line":""},{"line_number":252,"context_line":"* Multiple ``[neutron_physnet_$PHYSNET]`` configuration groups, one per each"},{"line_number":253,"context_line":"  ``$PHYSNET`` in ``[neutron] physnets``, will allow configuration of these"},{"line_number":254,"context_line":"  *physnets*. Each of these configuration groups will contain a single"},{"line_number":255,"context_line":"  configuration option each, ``numa_nodes``, which lists the host NUMA node(s)"},{"line_number":256,"context_line":"  to which networks using this *physnet* are affined."},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"The groups will all be generated dynamically, which is required as the values"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_f80dad35","line":255,"range":{"start_line":254,"start_character":49,"end_line":255,"end_character":27},"in_reply_to":"5f7c97a3_9cb6f23d","updated":"2018-05-16 18:52:04.000000000","message":"Cool, as long as there\u0027s a possibility for another option I think this is good.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":251,"context_line":""},{"line_number":252,"context_line":"* Multiple ``[neutron_physnet_$PHYSNET]`` configuration groups, one per each"},{"line_number":253,"context_line":"  ``$PHYSNET`` in ``[neutron] physnets``, will allow configuration of these"},{"line_number":254,"context_line":"  *physnets*. Each of these configuration groups will contain a single"},{"line_number":255,"context_line":"  configuration option each, ``numa_nodes``, which lists the host NUMA node(s)"},{"line_number":256,"context_line":"  to which networks using this *physnet* are affined."},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"The groups will all be generated dynamically, which is required as the values"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_9cb6f23d","line":255,"range":{"start_line":254,"start_character":49,"end_line":255,"end_character":27},"in_reply_to":"5f7c97a3_ba08a991","updated":"2018-05-16 16:12:04.000000000","message":"Yeah, the code required to implement this wasn\u0027t too complicated and it is definitely better than a JSON blob or dynamic options (rather than dynamic groups). There is only one option needed at the moment but I could envision a \u0027policy\u0027 key being added in the future too so this could grow. I\u0027m happy to keep this as is.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":294,"context_line":"  ``[neutron] physnets``. This in turn has the following keys:"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"  ``numa_nodes``"},{"line_number":297,"context_line":"    An list of integers of NUMA nodes associated with this backend. It is"},{"line_number":298,"context_line":"    defined as a list to cater for cross NUMA bonds and multipath routing. If"},{"line_number":299,"context_line":"    this is empty, the *physnet* has no NUMA affinity assigned to it."},{"line_number":300,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_9ab36de9","line":297,"range":{"start_line":297,"start_character":4,"end_line":297,"end_character":6},"updated":"2018-05-15 16:02:55.000000000","message":"s/An/A/","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":294,"context_line":"  ``[neutron] physnets``. This in turn has the following keys:"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"  ``numa_nodes``"},{"line_number":297,"context_line":"    An list of integers of NUMA nodes associated with this backend. It is"},{"line_number":298,"context_line":"    defined as a list to cater for cross NUMA bonds and multipath routing. If"},{"line_number":299,"context_line":"    this is empty, the *physnet* has no NUMA affinity assigned to it."},{"line_number":300,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_1c7c2270","line":297,"range":{"start_line":297,"start_character":4,"end_line":297,"end_character":6},"in_reply_to":"5f7c97a3_9ab36de9","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"be7531cf33720449962bdeaeec58ab0b1ee675a8","unresolved":false,"context_lines":[{"line_number":300,"context_line":""},{"line_number":301,"context_line":"    .. note::"},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"       A smart enough vSwitch with a active-active cross-NUMA bond could use"},{"line_number":304,"context_line":"       the NUMA affinity of the VM interface as an input to the hash for"},{"line_number":305,"context_line":"       selecting the bond peer has to ensure no cross-NUMA traffic for"},{"line_number":306,"context_line":"       inter-host traffic.  Alternatively, a dumb one could have *os-vif*"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_8d797efd","line":303,"range":{"start_line":303,"start_character":51,"end_line":303,"end_character":61},"updated":"2018-05-08 19:35:49.000000000","message":"cross-NUMA-node","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":300,"context_line":""},{"line_number":301,"context_line":"    .. note::"},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"       A smart enough vSwitch with a active-active cross-NUMA bond could use"},{"line_number":304,"context_line":"       the NUMA affinity of the VM interface as an input to the hash for"},{"line_number":305,"context_line":"       selecting the bond peer has to ensure no cross-NUMA traffic for"},{"line_number":306,"context_line":"       inter-host traffic.  Alternatively, a dumb one could have *os-vif*"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_fc94c68c","line":303,"range":{"start_line":303,"start_character":51,"end_line":303,"end_character":61},"in_reply_to":"5f7c97a3_8d797efd","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":323,"context_line":"VNFs. The orchestration tool is responsible for configuring how physical NICs"},{"line_number":324,"context_line":"are used by the various neutron networks and with only a little extra work can"},{"line_number":325,"context_line":"extend this to include NUMA affinity (for example, by combining information"},{"line_number":326,"context_line":"from tools like :command:`ethtool` with information from *sysfs*)."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"NUMA affinity will be provided for all networks with a *physnet* and a defined"},{"line_number":329,"context_line":"``[neutron_physnet_$PHYSNET]`` group and for all tunnelled networks, assuming"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_da19e5d6","line":326,"updated":"2018-05-15 16:02:55.000000000","message":"I guess I\u0027m surprised it wouldn\u0027t make more sense to just tell us which physnets are in use, surely we can map those to an interface, and then do the lookup of the nodes ourselves, no?\n\nObviously if the deployment tool is better suited, then that\u0027s fine, but doing lots of logic and math in puppet or the like isn\u0027t great, and it seems like asking for a higher-level mapping (i.e. network to interface) and then grabbing the nodes ourselves would be less error-prone.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"a734512867d9ba68927ffa215ff2da21255ffc1a","unresolved":false,"context_lines":[{"line_number":323,"context_line":"VNFs. The orchestration tool is responsible for configuring how physical NICs"},{"line_number":324,"context_line":"are used by the various neutron networks and with only a little extra work can"},{"line_number":325,"context_line":"extend this to include NUMA affinity (for example, by combining information"},{"line_number":326,"context_line":"from tools like :command:`ethtool` with information from *sysfs*)."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"NUMA affinity will be provided for all networks with a *physnet* and a defined"},{"line_number":329,"context_line":"``[neutron_physnet_$PHYSNET]`` group and for all tunnelled networks, assuming"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_9757ad26","line":326,"in_reply_to":"5f7c97a3_da19e5d6","updated":"2018-05-16 16:19:58.000000000","message":"\u003e I guess I\u0027m surprised it wouldn\u0027t make more sense to just tell us\n \u003e which physnets are in use, surely we can map those to an interface,\n \u003e and then do the lookup of the nodes ourselves, no?\n \u003e \n \u003e Obviously if the deployment tool is better suited, then that\u0027s\n \u003e fine, but doing lots of logic and math in puppet or the like isn\u0027t\n \u003e great, and it seems like asking for a higher-level mapping (i.e.\n \u003e network to interface) and then grabbing the nodes ourselves would\n \u003e be less error-prone.\n\nI\u0027d actually had the idea of building upon the PCI framework for this instead of the current approach. I avoided this because it brings us back into the territory of identifying what a NIC is. NICs are generally attached over PCI but they don\u0027t have to be and even then efried in particular has been very vocal about not using thing like PCI slots to identify these. You\u0027ve also got things like bonded NICs, in which case you\u0027d be trying to find the NUMA info for a logical NIC and would need inside knowledge on how to get the component NICs used. By keeping this in the deployment tool, we allow deployers to customize this selection process. As noted above, this is a tough problem to solve.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":340,"context_line":"~~~~~~~~~~~"},{"line_number":341,"context_line":""},{"line_number":342,"context_line":"The network API currently provides a method,"},{"line_number":343,"context_line":"``create_pci_requests_for_sriov_ports``, which retrieves information about"},{"line_number":344,"context_line":"*physnets* and VNIC types of each port requested when booting an instance for"},{"line_number":345,"context_line":"the purpose of creating PCI requests for SRIOV ports. This will be made more"},{"line_number":346,"context_line":"generic and extended to get information about the tunneled status of any"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_37435444","line":343,"range":{"start_line":343,"start_character":2,"end_line":343,"end_character":37},"updated":"2018-05-15 14:39:31.000000000","message":"You should probably link to the nova code for this because when I started reading this section I assume by network API you mean the actual neutron REST API, not the nova neutron internal client code.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":340,"context_line":"~~~~~~~~~~~"},{"line_number":341,"context_line":""},{"line_number":342,"context_line":"The network API currently provides a method,"},{"line_number":343,"context_line":"``create_pci_requests_for_sriov_ports``, which retrieves information about"},{"line_number":344,"context_line":"*physnets* and VNIC types of each port requested when booting an instance for"},{"line_number":345,"context_line":"the purpose of creating PCI requests for SRIOV ports. This will be made more"},{"line_number":346,"context_line":"generic and extended to get information about the tunneled status of any"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_bc4dee3b","line":343,"range":{"start_line":343,"start_character":2,"end_line":343,"end_character":37},"in_reply_to":"5f7c97a3_37435444","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":358,"context_line":"Scheduling"},{"line_number":359,"context_line":"~~~~~~~~~~"},{"line_number":360,"context_line":""},{"line_number":361,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":362,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":363,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":364,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":365,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":366,"context_line":"have something to reference during the *claim* stage. In both cases, the cached"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_977108ab","line":363,"range":{"start_line":361,"start_character":0,"end_line":363,"end_character":7},"updated":"2018-05-15 14:39:31.000000000","message":"So, now we\u0027re overlapping with gibi\u0027s spec on bandwidth-aware scheduling:\n\nhttps://specs.openstack.org/openstack/nova-specs/specs/rocky/approved/bandwidth-resource-provider.html#data-model-impact\n\nHe had the option of storing the NetworkRequestList in the RequestSpec but opted to not do that an instead store something more generic for requesting resources from placement during scheduling. Should probably have him look at this to make sure we don\u0027t have overlap.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"a734512867d9ba68927ffa215ff2da21255ffc1a","unresolved":false,"context_lines":[{"line_number":358,"context_line":"Scheduling"},{"line_number":359,"context_line":"~~~~~~~~~~"},{"line_number":360,"context_line":""},{"line_number":361,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":362,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":363,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":364,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":365,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":366,"context_line":"have something to reference during the *claim* stage. In both cases, the cached"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_17395df0","line":363,"range":{"start_line":361,"start_character":0,"end_line":363,"end_character":7},"in_reply_to":"5f7c97a3_977108ab","updated":"2018-05-16 16:19:58.000000000","message":"Yeah, I\u0027m keeping this as-is now but I\u0027m imagining he\u0027ll work with me on the code and we can update the spec after.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":362,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":363,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":364,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":365,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":366,"context_line":"have something to reference during the *claim* stage. In both cases, the cached"},{"line_number":367,"context_line":"information will need to be updated in the event of a move operation."},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"Virtualization driver"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_77f1cc00","line":366,"range":{"start_line":365,"start_character":54,"end_line":366,"end_character":53},"updated":"2018-05-15 14:39:31.000000000","message":"What claim stage? The numa topology limit claim that happens in nova-compute today? As far as I know, that\u0027s the only resource claim we make anymore using the old resource tracker claims code, since vcpu, ram and disk are \"claimed\" via the FilterScheduler during scheduling now since we model those in placement. I know Tetsuro and Sylvain are working on how to model NUMA in placement and make claims during scheduling, so it seems that should be a dependency/priority so we don\u0027t have to add more claims logic in the compute which we eventually want to rip out.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":362,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":363,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":364,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":365,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":366,"context_line":"have something to reference during the *claim* stage. In both cases, the cached"},{"line_number":367,"context_line":"information will need to be updated in the event of a move operation."},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"Virtualization driver"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_bc140eea","line":366,"range":{"start_line":365,"start_character":54,"end_line":366,"end_character":53},"in_reply_to":"5f7c97a3_77f1cc00","updated":"2018-05-16 16:12:04.000000000","message":"I\u0027m referring to this step.\n\n  https://review.openstack.org/#/c/564449/3/nova/compute/claims.py\n\nDo your above comments apply to this?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":369,"context_line":"Virtualization driver"},{"line_number":370,"context_line":"~~~~~~~~~~~~~~~~~~~~~"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"Only the libvirt driver currently supports the full breadth of NUMA affinity"},{"line_number":373,"context_line":"features required for this feature. The NUMA fitting code, found in"},{"line_number":374,"context_line":"``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":375,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_57da9077","line":372,"updated":"2018-05-15 14:39:31.000000000","message":"The hyper-v driver also supports NUMA...","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":369,"context_line":"Virtualization driver"},{"line_number":370,"context_line":"~~~~~~~~~~~~~~~~~~~~~"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"Only the libvirt driver currently supports the full breadth of NUMA affinity"},{"line_number":373,"context_line":"features required for this feature. The NUMA fitting code, found in"},{"line_number":374,"context_line":"``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":375,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_bcfbaebb","line":372,"in_reply_to":"5f7c97a3_254eba37","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":369,"context_line":"Virtualization driver"},{"line_number":370,"context_line":"~~~~~~~~~~~~~~~~~~~~~"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"Only the libvirt driver currently supports the full breadth of NUMA affinity"},{"line_number":373,"context_line":"features required for this feature. The NUMA fitting code, found in"},{"line_number":374,"context_line":"``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":375,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_254eba37","line":372,"in_reply_to":"5f7c97a3_57da9077","updated":"2018-05-15 16:02:55.000000000","message":"Well, he does say \"the full breadth\" but I think it would be good to qualify which parts are not implemented by hyper-v that exclude it from consideration here.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7e28499da0ef9430f6fe8d3b93f09d1f053c5d5a","unresolved":false,"context_lines":[{"line_number":374,"context_line":"``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":375,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"},{"line_number":376,"context_line":"to use. The ``NetworkRequestList`` object will be extracted from"},{"line_number":377,"context_line":"``RequestSpec`` or ``Instance`` objects and passed in by the caller of this"},{"line_number":378,"context_line":"function."},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"Potential issues"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_f7bddc01","line":377,"updated":"2018-05-15 14:39:31.000000000","message":"I\u0027m not sure why we need to store the requested networks in the instance object - before we get to spawn() in the driver, we have allocated the networking (ports) and should be able to pull this information from the instance.info_cache, right?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":374,"context_line":"``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":375,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"},{"line_number":376,"context_line":"to use. The ``NetworkRequestList`` object will be extracted from"},{"line_number":377,"context_line":"``RequestSpec`` or ``Instance`` objects and passed in by the caller of this"},{"line_number":378,"context_line":"function."},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"Potential issues"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_7c30b656","line":377,"in_reply_to":"5f7c97a3_454df63b","updated":"2018-05-16 16:12:04.000000000","message":"Oh, I wasn\u0027t even aware that info_cache was an option, hence network_requests. I need to look into this a little more.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":374,"context_line":"``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":375,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"},{"line_number":376,"context_line":"to use. The ``NetworkRequestList`` object will be extracted from"},{"line_number":377,"context_line":"``RequestSpec`` or ``Instance`` objects and passed in by the caller of this"},{"line_number":378,"context_line":"function."},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"Potential issues"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_454df63b","line":377,"in_reply_to":"5f7c97a3_f7bddc01","updated":"2018-05-15 16:02:55.000000000","message":"And, if any attachments are made, we\u0027d need to update reqspec or instance, wherever we stored it. I\u0027m definitely concerned about needing to do this as we don\u0027t store that information anywhere currently. What specifically from the NetworkRequest do we need that we can\u0027t get from the info_cache?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bf37034a8c6e3d524ca9525ad6b957fc037bfb42","unresolved":false,"context_lines":[{"line_number":377,"context_line":"``RequestSpec`` or ``Instance`` objects and passed in by the caller of this"},{"line_number":378,"context_line":"function."},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"Potential issues"},{"line_number":381,"context_line":"----------------"},{"line_number":382,"context_line":""},{"line_number":383,"context_line":"As noted above, this is a rather difficult issue to solve sanely and, as a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_8581cec5","line":380,"updated":"2018-05-15 16:13:36.000000000","message":"What happens when someone attaches a network or port to an existing guest? In that scenario we don\u0027t do scheduling or any kind of resource claim logic. Would we need to add code to the attach_interface flow to detect that you\u0027re trying to attach an interface that has numa affinity but since we can\u0027t honor that we would need to fail the request?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":377,"context_line":"``RequestSpec`` or ``Instance`` objects and passed in by the caller of this"},{"line_number":378,"context_line":"function."},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"Potential issues"},{"line_number":381,"context_line":"----------------"},{"line_number":382,"context_line":""},{"line_number":383,"context_line":"As noted above, this is a rather difficult issue to solve sanely and, as a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_9c9cb26a","line":380,"in_reply_to":"5f7c97a3_8581cec5","updated":"2018-05-16 16:12:04.000000000","message":"I had considered this out-of-scope for now as the feature is already big/complex enough. I should have called that out though. Done now.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":387,"context_line":"performance. This exact situation can arise in SR-IOV too and generally occurs"},{"line_number":388,"context_line":"because:"},{"line_number":389,"context_line":""},{"line_number":390,"context_line":"- The user has misconfigured something"},{"line_number":391,"context_line":"- The user does not care about optimal performance"},{"line_number":392,"context_line":"- The user wants to provide resiliency by spreading work across NICs and cannot"},{"line_number":393,"context_line":"  or will not use two NICs from the same NUMA node"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_056cde99","line":390,"range":{"start_line":390,"start_character":6,"end_line":390,"end_character":10},"updated":"2018-05-15 16:02:55.000000000","message":"User or operator?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":387,"context_line":"performance. This exact situation can arise in SR-IOV too and generally occurs"},{"line_number":388,"context_line":"because:"},{"line_number":389,"context_line":""},{"line_number":390,"context_line":"- The user has misconfigured something"},{"line_number":391,"context_line":"- The user does not care about optimal performance"},{"line_number":392,"context_line":"- The user wants to provide resiliency by spreading work across NICs and cannot"},{"line_number":393,"context_line":"  or will not use two NICs from the same NUMA node"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_fce36603","line":390,"range":{"start_line":390,"start_character":6,"end_line":390,"end_character":10},"in_reply_to":"5f7c97a3_056cde99","updated":"2018-05-16 16:12:04.000000000","message":"operator. I was using \u0027user\u0027 in the generic sense (not developer)","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bf37034a8c6e3d524ca9525ad6b957fc037bfb42","unresolved":false,"context_lines":[{"line_number":447,"context_line":"information about the various backends from neutron. Nova would then be able to"},{"line_number":448,"context_line":"use this information in order to make its scheduling decisions. This is"},{"line_number":449,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":450,"context_line":"other issues, such as bandwidth-aware scheduling and discovery of the types of"},{"line_number":451,"context_line":"networks available on a compute node _before_ landing on that node. However,"},{"line_number":452,"context_line":"we don\u0027t currently have a clear story for modelling NUMA resources in placement"},{"line_number":453,"context_line":"and neutron is not currently communicating anything to placement. Given the"},{"line_number":454,"context_line":"tenant dataplane performance issues being faced right now, this seems"},{"line_number":455,"context_line":"untenable. However, that\u0027s not to say we can\u0027t migrate some of this information"},{"line_number":456,"context_line":"to placement in the future."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_45065654","line":453,"range":{"start_line":450,"start_character":14,"end_line":453,"end_character":65},"updated":"2018-05-15 16:13:36.000000000","message":"The bandwidth-aware spec is approved and relies on neutron modeling resources in placement, so yes we have part of this in progress already. The NUMA modeling is still TBD given https://review.openstack.org/#/c/552924/ and I\u0027m not sure what the status of all that is right now.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":450,"context_line":"other issues, such as bandwidth-aware scheduling and discovery of the types of"},{"line_number":451,"context_line":"networks available on a compute node _before_ landing on that node. However,"},{"line_number":452,"context_line":"we don\u0027t currently have a clear story for modelling NUMA resources in placement"},{"line_number":453,"context_line":"and neutron is not currently communicating anything to placement. Given the"},{"line_number":454,"context_line":"tenant dataplane performance issues being faced right now, this seems"},{"line_number":455,"context_line":"untenable. However, that\u0027s not to say we can\u0027t migrate some of this information"},{"line_number":456,"context_line":"to placement in the future."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_25ea3ad7","line":453,"range":{"start_line":453,"start_character":4,"end_line":453,"end_character":64},"updated":"2018-05-15 16:02:55.000000000","message":"This is not true:\n\nhttps://docs.openstack.org/ocata/networking-guide/config-routed-networks.html","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":450,"context_line":"other issues, such as bandwidth-aware scheduling and discovery of the types of"},{"line_number":451,"context_line":"networks available on a compute node _before_ landing on that node. However,"},{"line_number":452,"context_line":"we don\u0027t currently have a clear story for modelling NUMA resources in placement"},{"line_number":453,"context_line":"and neutron is not currently communicating anything to placement. Given the"},{"line_number":454,"context_line":"tenant dataplane performance issues being faced right now, this seems"},{"line_number":455,"context_line":"untenable. However, that\u0027s not to say we can\u0027t migrate some of this information"},{"line_number":456,"context_line":"to placement in the future."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_772d3193","line":453,"range":{"start_line":453,"start_character":4,"end_line":453,"end_character":64},"in_reply_to":"5f7c97a3_25ea3ad7","updated":"2018-05-16 16:12:04.000000000","message":"\u003e This is not true:\n \u003e \n \u003e https://docs.openstack.org/ocata/networking-guide/config-routed-networks.html\n\nAh, so neutron is communicating with placement. I thought bandwidth-aware-scheduling was going to be the first user of this in neutron. I\u0027ve removed that piece but the main argument (that NUMA isn\u0027t ready for primetime here) is still valid.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"19d1077584cc3205ec336255965538a65154c863","unresolved":false,"context_lines":[{"line_number":447,"context_line":"information about the various backends from neutron. Nova would then be able to"},{"line_number":448,"context_line":"use this information in order to make its scheduling decisions. This is"},{"line_number":449,"context_line":"technically the best solution as it solves both this issue and a number of"},{"line_number":450,"context_line":"other issues, such as bandwidth-aware scheduling and discovery of the types of"},{"line_number":451,"context_line":"networks available on a compute node _before_ landing on that node. However,"},{"line_number":452,"context_line":"we don\u0027t currently have a clear story for modelling NUMA resources in placement"},{"line_number":453,"context_line":"and neutron is not currently communicating anything to placement. Given the"},{"line_number":454,"context_line":"tenant dataplane performance issues being faced right now, this seems"},{"line_number":455,"context_line":"untenable. However, that\u0027s not to say we can\u0027t migrate some of this information"},{"line_number":456,"context_line":"to placement in the future."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_c04ed495","line":453,"range":{"start_line":450,"start_character":14,"end_line":453,"end_character":65},"in_reply_to":"5f7c97a3_45065654","updated":"2018-05-15 16:54:05.000000000","message":"FWIW, I do remember agreement from the Rocky PTG in Dublin that we wouldn\u0027t make this dependent on placement:\n\nhttps://etherpad.openstack.org/p/nova-ptg-rocky\n\nSince that would likely push this out at least a release. You should probably mention that somewhere in here.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":473,"context_line":"Data model impact"},{"line_number":474,"context_line":"-----------------"},{"line_number":475,"context_line":""},{"line_number":476,"context_line":"The ``instance_extras`` table will gain a new column, ``requested_networks`` to"},{"line_number":477,"context_line":"hold the serialized representation of the ``NetworkRequestList`` object."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"REST API impact"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_45d2d629","line":476,"range":{"start_line":476,"start_character":6,"end_line":476,"end_character":21},"updated":"2018-05-15 16:02:55.000000000","message":"instance_extra","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"a734512867d9ba68927ffa215ff2da21255ffc1a","unresolved":false,"context_lines":[{"line_number":473,"context_line":"Data model impact"},{"line_number":474,"context_line":"-----------------"},{"line_number":475,"context_line":""},{"line_number":476,"context_line":"The ``instance_extras`` table will gain a new column, ``requested_networks`` to"},{"line_number":477,"context_line":"hold the serialized representation of the ``NetworkRequestList`` object."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"REST API impact"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_d72685ce","line":476,"range":{"start_line":476,"start_character":6,"end_line":476,"end_character":21},"in_reply_to":"5f7c97a3_45d2d629","updated":"2018-05-16 16:19:58.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bf37034a8c6e3d524ca9525ad6b957fc037bfb42","unresolved":false,"context_lines":[{"line_number":473,"context_line":"Data model impact"},{"line_number":474,"context_line":"-----------------"},{"line_number":475,"context_line":""},{"line_number":476,"context_line":"The ``instance_extras`` table will gain a new column, ``requested_networks`` to"},{"line_number":477,"context_line":"hold the serialized representation of the ``NetworkRequestList`` object."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"REST API impact"},{"line_number":480,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_a5b96a79","line":477,"range":{"start_line":476,"start_character":0,"end_line":477,"end_character":72},"updated":"2018-05-15 16:13:36.000000000","message":"Again, don\u0027t think we need this since we have the instance info cache already...the question is when this data is needed. If we need the requested networks during scheduling, then that\u0027s the RequestSpec. If it\u0027s during the virt driver spawn(), that\u0027s the info cache.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":474,"context_line":"-----------------"},{"line_number":475,"context_line":""},{"line_number":476,"context_line":"The ``instance_extras`` table will gain a new column, ``requested_networks`` to"},{"line_number":477,"context_line":"hold the serialized representation of the ``NetworkRequestList`` object."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"REST API impact"},{"line_number":480,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_65927259","line":477,"updated":"2018-05-15 16:02:55.000000000","message":"So, if we do this, we will need to merge new requests with the existing ones when doing an interface-attach.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"a734512867d9ba68927ffa215ff2da21255ffc1a","unresolved":false,"context_lines":[{"line_number":473,"context_line":"Data model impact"},{"line_number":474,"context_line":"-----------------"},{"line_number":475,"context_line":""},{"line_number":476,"context_line":"The ``instance_extras`` table will gain a new column, ``requested_networks`` to"},{"line_number":477,"context_line":"hold the serialized representation of the ``NetworkRequestList`` object."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"REST API impact"},{"line_number":480,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_771f716b","line":477,"range":{"start_line":476,"start_character":0,"end_line":477,"end_character":72},"in_reply_to":"5f7c97a3_a5b96a79","updated":"2018-05-16 16:19:58.000000000","message":"I don\u0027t think I have this cache generated when I need it though. Would appreciate your input on [1] to figure this out.\n\n[1] https://review.openstack.org/#/c/564449/3","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":501,"context_line":""},{"line_number":502,"context_line":"There will be a negligible increase in the time taken to schedule an instance"},{"line_number":503,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":504,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":505,"context_line":""},{"line_number":506,"context_line":"Other deployer impact"},{"line_number":507,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_85822eff","line":504,"updated":"2018-05-15 16:02:55.000000000","message":"None of your changes above describe anything related to the scheduler, so why is there a performance impact to scheduling?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a1b2b73a6eb6b2b2cb00acc28c91609386a4236f","unresolved":false,"context_lines":[{"line_number":501,"context_line":""},{"line_number":502,"context_line":"There will be a negligible increase in the time taken to schedule an instance"},{"line_number":503,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":504,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":505,"context_line":""},{"line_number":506,"context_line":"Other deployer impact"},{"line_number":507,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_18a3c902","line":504,"in_reply_to":"5f7c97a3_77cc5134","updated":"2018-05-16 18:52:04.000000000","message":"Not really. You say:\n\n \"We require this information so we can use it to build the\n NUMA topology of the instance on the host as part of scheduling\"\n\nTo me, that says \"we will do some stuff with it\" which doesn\u0027t really give me a feeling of what kind of \"negligible increase\" we\u0027re talking about.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":501,"context_line":""},{"line_number":502,"context_line":"There will be a negligible increase in the time taken to schedule an instance"},{"line_number":503,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":504,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":505,"context_line":""},{"line_number":506,"context_line":"Other deployer impact"},{"line_number":507,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_77cc5134","line":504,"in_reply_to":"5f7c97a3_85822eff","updated":"2018-05-16 16:12:04.000000000","message":"Don\u0027t I describe this in the \u0027scheduling\u0027 section above?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":512,"context_line":"Developer impact"},{"line_number":513,"context_line":"----------------"},{"line_number":514,"context_line":""},{"line_number":515,"context_line":"This change will only target the libvirt driver. Additional drivers do not"},{"line_number":516,"context_line":"currently provide sufficient awareness of host NUMA information for this change"},{"line_number":517,"context_line":"to be incorporated in a meaningful manner. If this changes in the future, the"},{"line_number":518,"context_line":"drivers will need to be extended to consume the configuration option and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_e5944239","line":515,"range":{"start_line":515,"start_character":0,"end_line":515,"end_character":47},"updated":"2018-05-15 16:02:55.000000000","message":"Apparently not, given your perf impact above, and your second work item below.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":512,"context_line":"Developer impact"},{"line_number":513,"context_line":"----------------"},{"line_number":514,"context_line":""},{"line_number":515,"context_line":"This change will only target the libvirt driver. Additional drivers do not"},{"line_number":516,"context_line":"currently provide sufficient awareness of host NUMA information for this change"},{"line_number":517,"context_line":"to be incorporated in a meaningful manner. If this changes in the future, the"},{"line_number":518,"context_line":"drivers will need to be extended to consume the configuration option and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_d7b0c545","line":515,"range":{"start_line":515,"start_character":0,"end_line":515,"end_character":47},"in_reply_to":"5f7c97a3_e5944239","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bf37034a8c6e3d524ca9525ad6b957fc037bfb42","unresolved":false,"context_lines":[{"line_number":512,"context_line":"Developer impact"},{"line_number":513,"context_line":"----------------"},{"line_number":514,"context_line":""},{"line_number":515,"context_line":"This change will only target the libvirt driver. Additional drivers do not"},{"line_number":516,"context_line":"currently provide sufficient awareness of host NUMA information for this change"},{"line_number":517,"context_line":"to be incorporated in a meaningful manner. If this changes in the future, the"},{"line_number":518,"context_line":"drivers will need to be extended to consume the configuration option and"},{"line_number":519,"context_line":"request spec attributes."},{"line_number":520,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_e568c2f5","line":517,"range":{"start_line":515,"start_character":49,"end_line":517,"end_character":42},"updated":"2018-05-15 16:13:36.000000000","message":"What about hyper-v?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":512,"context_line":"Developer impact"},{"line_number":513,"context_line":"----------------"},{"line_number":514,"context_line":""},{"line_number":515,"context_line":"This change will only target the libvirt driver. Additional drivers do not"},{"line_number":516,"context_line":"currently provide sufficient awareness of host NUMA information for this change"},{"line_number":517,"context_line":"to be incorporated in a meaningful manner. If this changes in the future, the"},{"line_number":518,"context_line":"drivers will need to be extended to consume the configuration option and"},{"line_number":519,"context_line":"request spec attributes."},{"line_number":520,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_f7ab81d2","line":517,"range":{"start_line":515,"start_character":49,"end_line":517,"end_character":42},"in_reply_to":"5f7c97a3_e568c2f5","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":544,"context_line":"#. Add support for the new configuration option and dynamic configuration"},{"line_number":545,"context_line":"   groups."},{"line_number":546,"context_line":""},{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_a59aca5e","line":547,"range":{"start_line":547,"start_character":3,"end_line":547,"end_character":54},"updated":"2018-05-15 16:02:55.000000000","message":"Sounds like this is another data model change for the section above.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":544,"context_line":"#. Add support for the new configuration option and dynamic configuration"},{"line_number":545,"context_line":"   groups."},{"line_number":546,"context_line":""},{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_77d7714c","line":547,"range":{"start_line":547,"start_character":3,"end_line":547,"end_character":54},"in_reply_to":"5f7c97a3_a59aca5e","updated":"2018-05-16 16:12:04.000000000","message":"I thought that section just referred to database schema changes. Far as I can tell, the rest of these changes affect objects that are not persisted anywhere. I\u0027ve updated this all the same though","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bf37034a8c6e3d524ca9525ad6b957fc037bfb42","unresolved":false,"context_lines":[{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_85446e75","line":550,"range":{"start_line":550,"start_character":18,"end_line":550,"end_character":26},"updated":"2018-05-15 16:13:36.000000000","message":"Neither the scheduler nor conductor do any NUMA resource claims, that\u0027s all done in the nova-compute resource tracker code still. Long-term we want to model NUMA in placement and claim in the scheduler, but that\u0027s Sylvain\u0027s spec.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_85b0eede","line":550,"updated":"2018-05-15 16:02:55.000000000","message":"This seems to imply changes to the NUMA filter, is that right? Can you describe what those would be in the changes section above? And if I\u0027m missing it, please smack me :)","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a1b2b73a6eb6b2b2cb00acc28c91609386a4236f","unresolved":false,"context_lines":[{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5835c1a7","line":550,"range":{"start_line":550,"start_character":18,"end_line":550,"end_character":26},"in_reply_to":"5f7c97a3_175abd81","updated":"2018-05-16 18:52:04.000000000","message":"No, that actually happens in the API service.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a1b2b73a6eb6b2b2cb00acc28c91609386a4236f","unresolved":false,"context_lines":[{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_b84f3533","line":550,"in_reply_to":"5f7c97a3_17e5dddb","updated":"2018-05-16 18:52:04.000000000","message":"Well, (a) that further supports my comment above about not actually describing what the scheduler will be doing differently with this information and (b) it needs to be in the spec and not a link to code, IMHO.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_175abd81","line":550,"range":{"start_line":550,"start_character":18,"end_line":550,"end_character":26},"in_reply_to":"5f7c97a3_85446e75","updated":"2018-05-16 16:12:04.000000000","message":"What\u0027s the service that runs this code so?\n\n  https://review.openstack.org/#/c/564449/3\n\nI thought it was the conductor?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":547,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":548,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_17e5dddb","line":550,"in_reply_to":"5f7c97a3_85b0eede","updated":"2018-05-16 16:12:04.000000000","message":"\u003e This seems to imply changes to the NUMA filter, is that right? Can\n \u003e you describe what those would be in the changes section above? And\n \u003e if I\u0027m missing it, please smack me :)\n\nNo changes to the NUMA topology filter, no - we\u0027re going to harness what we already have. See https://review.openstack.org/#/c/564448/3","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"},{"line_number":554,"context_line":"   neutron but discarding."},{"line_number":555,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_45d3b604","line":552,"range":{"start_line":552,"start_character":3,"end_line":552,"end_character":77},"updated":"2018-05-15 16:02:55.000000000","message":"These are large data model changes not described above. Did I miss where you said what you\u0027d do with these? Who is responsible for filling these out? I would assume compute?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"4ecfe9d87d70a6d36d8930cb234f69d517fb732a","unresolved":false,"context_lines":[{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"},{"line_number":554,"context_line":"   neutron but discarding."},{"line_number":555,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_a07158aa","line":552,"range":{"start_line":552,"start_character":3,"end_line":552,"end_character":77},"in_reply_to":"5f7c97a3_45d3b604","updated":"2018-05-15 16:19:28.000000000","message":"I believe the intent is to gather this information during the call to create_pci_requests_for_sriov_ports() from the API today before we cast off to do scheduling.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":549,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":550,"context_line":"   scheduling and claiming, respectively."},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"#. Add ``physnet`` and ``tunneled`` fields to the ``RequestedNetwork`` object,"},{"line_number":553,"context_line":"   which will be used to retrieve the information we\u0027re currently reading from"},{"line_number":554,"context_line":"   neutron but discarding."},{"line_number":555,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_17587df7","line":552,"range":{"start_line":552,"start_character":3,"end_line":552,"end_character":77},"in_reply_to":"5f7c97a3_a07158aa","updated":"2018-05-16 16:12:04.000000000","message":"Done","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c23dd8d7c969d7a7f6d2a30025eaac1ef1868188","unresolved":false,"context_lines":[{"line_number":557,"context_line":"   to fetch most of the information we need to populate the new"},{"line_number":558,"context_line":"   ``RequestedNetwork`` fields."},{"line_number":559,"context_line":""},{"line_number":560,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":561,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":562,"context_line":"   ``NetworkRequestList`` object. This information will be used by the"},{"line_number":563,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_855a2e80","line":560,"range":{"start_line":560,"start_character":3,"end_line":560,"end_character":67},"updated":"2018-05-15 16:02:55.000000000","message":"Also a data model change. Are you really going to store requested_networks in the reqspec _and_ in instance_extra? Why?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":557,"context_line":"   to fetch most of the information we need to populate the new"},{"line_number":558,"context_line":"   ``RequestedNetwork`` fields."},{"line_number":559,"context_line":""},{"line_number":560,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":561,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":562,"context_line":"   ``NetworkRequestList`` object. This information will be used by the"},{"line_number":563,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_57bef5d4","line":560,"range":{"start_line":560,"start_character":3,"end_line":560,"end_character":67},"in_reply_to":"5f7c97a3_855a2e80","updated":"2018-05-16 16:12:04.000000000","message":"I need to include it in the Instance object which is persisted and therefore needs this column to store the serialized object. At least, that\u0027s what I saw in [1] and [2].\n\n[1] https://review.openstack.org/#/c/564446/3/\n[2] https://review.openstack.org/#/c/564447/3/","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bf37034a8c6e3d524ca9525ad6b957fc037bfb42","unresolved":false,"context_lines":[{"line_number":559,"context_line":""},{"line_number":560,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":561,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":562,"context_line":"   ``NetworkRequestList`` object. This information will be used by the"},{"line_number":563,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"},{"line_number":564,"context_line":"   request."},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"#. Add a ``requested_networks`` field to the ``Instance`` object, which will"},{"line_number":567,"context_line":"   include a list of networks requested by the instance in the form of a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_052dbea6","line":564,"range":{"start_line":562,"start_character":34,"end_line":564,"end_character":11},"updated":"2018-05-15 16:13:36.000000000","message":"We should have some more details in the proposed change section on this - the spec is pretty light in this regard. I\u0027m assuming the additional filtering needed is checking the requested_networks phsynet and tunneled values, populated via create_pci_requests_for_sriov_ports, and comparing those against the configuration on the compute host.\n\nHow will the scheduler actually compare these values? The NUMATopologyFilter doesn\u0027t RPC call the nova-compute host, so I\u0027m assuming the compute host configuration will get modeled somehow on the related ComputeNode object which in turn gets wrapped by the HostState object which the filter acts upon. Maybe that\u0027s related to #2 above about the NUMACell.networks change?","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":559,"context_line":""},{"line_number":560,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":561,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":562,"context_line":"   ``NetworkRequestList`` object. This information will be used by the"},{"line_number":563,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"},{"line_number":564,"context_line":"   request."},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"#. Add a ``requested_networks`` field to the ``Instance`` object, which will"},{"line_number":567,"context_line":"   include a list of networks requested by the instance in the form of a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5764b549","line":564,"range":{"start_line":562,"start_character":34,"end_line":564,"end_character":11},"in_reply_to":"5f7c97a3_052dbea6","updated":"2018-05-16 16:12:04.000000000","message":"\u003e We should have some more details in the proposed change section on\n \u003e this - the spec is pretty light in this regard. I\u0027m assuming the\n \u003e additional filtering needed is checking the requested_networks\n \u003e phsynet and tunneled values, populated via create_pci_requests_for_sriov_ports,\n \u003e and comparing those against the configuration on the compute host.\n\nWe simply extend the existing functionality. See https://review.openstack.org/#/c/564448/3\n\n \u003e How will the scheduler actually compare these values? The\n \u003e NUMATopologyFilter doesn\u0027t RPC call the nova-compute host, so I\u0027m\n \u003e assuming the compute host configuration will get modeled somehow on\n \u003e the related ComputeNode object which in turn gets wrapped by the\n \u003e HostState object which the filter acts upon. Maybe that\u0027s related\n \u003e to #2 above about the NUMACell.networks change?\n\nYeah, HostState is built from a ComputeNode object, which contains a NUMATopology object. We\u0027re extending NUMATopology so we get this information for free. See https://review.openstack.org/#/c/564439/3","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bf37034a8c6e3d524ca9525ad6b957fc037bfb42","unresolved":false,"context_lines":[{"line_number":563,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"},{"line_number":564,"context_line":"   request."},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"#. Add a ``requested_networks`` field to the ``Instance`` object, which will"},{"line_number":567,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":568,"context_line":"   ``NetworkRequestList`` object. This information will be used during"},{"line_number":569,"context_line":"   claiming."},{"line_number":570,"context_line":""},{"line_number":571,"context_line":""},{"line_number":572,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_85122e61","line":569,"range":{"start_line":566,"start_character":3,"end_line":569,"end_character":12},"updated":"2018-05-15 16:13:36.000000000","message":"Again, don\u0027t understand the need for storing this on the instance. If we need it during spawn(), we have it on the instance info cache. If we need it during the actual claim in the resource tracker, it should be part of the existing numa topology limits passed down from the scheduler (populated by the NUMATopologyFilter) and to the ResourceTracker in nova-compute.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"aeb087abff923c1a2aeb310b411b86c71919b76a","unresolved":false,"context_lines":[{"line_number":563,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"},{"line_number":564,"context_line":"   request."},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"#. Add a ``requested_networks`` field to the ``Instance`` object, which will"},{"line_number":567,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":568,"context_line":"   ``NetworkRequestList`` object. This information will be used during"},{"line_number":569,"context_line":"   claiming."},{"line_number":570,"context_line":""},{"line_number":571,"context_line":""},{"line_number":572,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_b7ec29bb","line":569,"range":{"start_line":566,"start_character":3,"end_line":569,"end_character":12},"in_reply_to":"5f7c97a3_85122e61","updated":"2018-05-16 16:12:04.000000000","message":"They\u0027re basically the same thing, no? I opted to store this as Instance.requested_networks but I could also have stored them as Instance.numa_topology.requested_networks. Either-or would suit me.","commit_id":"c50e840b4bf2267c6f2c9c945b0ce24d728ba360"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"062cf2db5b7d9daf529a97da8750e03f79266875","unresolved":false,"context_lines":[{"line_number":360,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":361,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":362,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":363,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":364,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":365,"context_line":"have something to reference during the *claim* stage. In both cases, the cached"},{"line_number":366,"context_line":"information will need to be updated in the event of a move operation."},{"line_number":367,"context_line":""},{"line_number":368,"context_line":"Virtualization driver"}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f7c97a3_8f43a0af","line":365,"range":{"start_line":363,"start_character":51,"end_line":365,"end_character":53},"updated":"2018-05-23 18:08:22.000000000","message":"This has to be updated, per my notes in the top-level comment.","commit_id":"1154c3a466414823d2e882bb38998131129d1cff"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"062cf2db5b7d9daf529a97da8750e03f79266875","unresolved":false,"context_lines":[{"line_number":480,"context_line":"Data model impact"},{"line_number":481,"context_line":"-----------------"},{"line_number":482,"context_line":""},{"line_number":483,"context_line":"The ``RequestedNetwork`` object will gain two new fields, a list of strings"},{"line_number":484,"context_line":"field called ``physnet`` and a boolean field called ``tunneled``. These will"},{"line_number":485,"context_line":"store additional information about a given network to avoid having to fetch"},{"line_number":486,"context_line":"this information repeatedly. They will be populated by a more generic version"}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f7c97a3_cf22f801","line":483,"range":{"start_line":483,"start_character":6,"end_line":483,"end_character":22},"updated":"2018-05-23 18:08:22.000000000","message":"NetworkRequest?","commit_id":"1154c3a466414823d2e882bb38998131129d1cff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3173dc958073bd580b72534d5096ce0b3985b156","unresolved":false,"context_lines":[{"line_number":480,"context_line":"Data model impact"},{"line_number":481,"context_line":"-----------------"},{"line_number":482,"context_line":""},{"line_number":483,"context_line":"The ``RequestedNetwork`` object will gain two new fields, a list of strings"},{"line_number":484,"context_line":"field called ``physnet`` and a boolean field called ``tunneled``. These will"},{"line_number":485,"context_line":"store additional information about a given network to avoid having to fetch"},{"line_number":486,"context_line":"this information repeatedly. They will be populated by a more generic version"}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f7c97a3_b324780e","line":483,"range":{"start_line":483,"start_character":6,"end_line":483,"end_character":22},"in_reply_to":"5f7c97a3_cf22f801","updated":"2018-05-31 12:58:21.000000000","message":"Done","commit_id":"1154c3a466414823d2e882bb38998131129d1cff"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"062cf2db5b7d9daf529a97da8750e03f79266875","unresolved":false,"context_lines":[{"line_number":482,"context_line":""},{"line_number":483,"context_line":"The ``RequestedNetwork`` object will gain two new fields, a list of strings"},{"line_number":484,"context_line":"field called ``physnet`` and a boolean field called ``tunneled``. These will"},{"line_number":485,"context_line":"store additional information about a given network to avoid having to fetch"},{"line_number":486,"context_line":"this information repeatedly. They will be populated by a more generic version"},{"line_number":487,"context_line":"of the ``create_pci_requests_for_sriov_ports`` function."},{"line_number":488,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f7c97a3_0f86b0dc","line":485,"updated":"2018-05-23 18:08:22.000000000","message":"We need to make a note that we won\u0027t persist this stuff in the request spec - it is just being put there as a way to get it from the API to the numa filter in the scheduler. Then the limits from the filter get passed down to the compute to do the claim.","commit_id":"1154c3a466414823d2e882bb38998131129d1cff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3173dc958073bd580b72534d5096ce0b3985b156","unresolved":false,"context_lines":[{"line_number":482,"context_line":""},{"line_number":483,"context_line":"The ``RequestedNetwork`` object will gain two new fields, a list of strings"},{"line_number":484,"context_line":"field called ``physnet`` and a boolean field called ``tunneled``. These will"},{"line_number":485,"context_line":"store additional information about a given network to avoid having to fetch"},{"line_number":486,"context_line":"this information repeatedly. They will be populated by a more generic version"},{"line_number":487,"context_line":"of the ``create_pci_requests_for_sriov_ports`` function."},{"line_number":488,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f7c97a3_93293ce7","line":485,"in_reply_to":"5f7c97a3_0f86b0dc","updated":"2018-05-31 12:58:21.000000000","message":"Done","commit_id":"1154c3a466414823d2e882bb38998131129d1cff"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"062cf2db5b7d9daf529a97da8750e03f79266875","unresolved":false,"context_lines":[{"line_number":490,"context_line":"networks associated with a host NUMA node. This will be mapped to a parent"},{"line_number":491,"context_line":"``NUMACell`` object using a ``network_info`` field."},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"The ``Instance`` object will gain a ``requested_networks`` attribute, which"},{"line_number":494,"context_line":"will hold a ``NetworkRequestList`` object. This will require changes to the"},{"line_number":495,"context_line":"``instance_extra`` database table to allow us to hold the serialized"},{"line_number":496,"context_line":"representation of this ``NetworkRequestList`` object for a given ``Instance``."}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f7c97a3_af6c5c36","line":493,"updated":"2018-05-23 18:08:22.000000000","message":"nix - the instance info_cache will get the network bits but we won\u0027t store top-level things in the instance object","commit_id":"1154c3a466414823d2e882bb38998131129d1cff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3173dc958073bd580b72534d5096ce0b3985b156","unresolved":false,"context_lines":[{"line_number":490,"context_line":"networks associated with a host NUMA node. This will be mapped to a parent"},{"line_number":491,"context_line":"``NUMACell`` object using a ``network_info`` field."},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"The ``Instance`` object will gain a ``requested_networks`` attribute, which"},{"line_number":494,"context_line":"will hold a ``NetworkRequestList`` object. This will require changes to the"},{"line_number":495,"context_line":"``instance_extra`` database table to allow us to hold the serialized"},{"line_number":496,"context_line":"representation of this ``NetworkRequestList`` object for a given ``Instance``."}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f7c97a3_734100a6","line":493,"in_reply_to":"5f7c97a3_af6c5c36","updated":"2018-05-31 12:58:21.000000000","message":"Done. It looks like \u0027nova.network.VIF\u0027, a list of which is stored in \u0027Instance.info_cache.network_info\u0027 already provides most of the functionality needed so I dropped this whole piece.","commit_id":"1154c3a466414823d2e882bb38998131129d1cff"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":339,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":340,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":341,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":342,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":343,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":344,"context_line":"have something to reference during the *claim* stage. In both cases, the cached"},{"line_number":345,"context_line":"information will need to be updated in the event of a move operation."},{"line_number":346,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_0fde543d","line":343,"range":{"start_line":342,"start_character":51,"end_line":343,"end_character":53},"updated":"2018-05-30 22:29:17.000000000","message":"This can go away right? We\u0027ll store the network information via the info_cache, so we don\u0027t need a new top-level field on the instances table (or Instance object).\n\nRemember, putting the requested_networks information into the RequestSpec is just a means to get the information to the NUMATopologyFilter in the scheduler, but we won\u0027t persist it in the RequestSpec. The numa filter will return the limits dict that gets sent to the compute service which will use it for the resource claim.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":339,"context_line":"In addition to the configuration change, we will need to start including the"},{"line_number":340,"context_line":"list of requested networks, ``NetworkRequestList``, in the ``RequestSpec``"},{"line_number":341,"context_line":"object. We require this information so we can use it to build the NUMA topology"},{"line_number":342,"context_line":"of the instance on the host as part of scheduling. We also need to extend the"},{"line_number":343,"context_line":"``Instance`` object to store these networks requests. This is necessary so we"},{"line_number":344,"context_line":"have something to reference during the *claim* stage. In both cases, the cached"},{"line_number":345,"context_line":"information will need to be updated in the event of a move operation."},{"line_number":346,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_50b06443","line":343,"range":{"start_line":342,"start_character":51,"end_line":343,"end_character":53},"in_reply_to":"5f7c97a3_0fde543d","updated":"2018-06-01 13:01:31.000000000","message":"Yup, I missed this piece. Done.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":354,"context_line":"found in ``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":355,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"},{"line_number":356,"context_line":"to use. The ``NetworkRequestList`` object will be extracted from"},{"line_number":357,"context_line":"``RequestSpec`` or ``Instance`` objects and passed in by the caller of this"},{"line_number":358,"context_line":"function."},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"Potential issues"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_6fc7c865","line":357,"range":{"start_line":357,"start_character":16,"end_line":357,"end_character":31},"updated":"2018-05-30 22:29:17.000000000","message":"As far as I can tell, numa_fit_instance_to_host is only called from the NUMATopologyFilter where we\u0027ll get the requested networks from the RequestSpec, not the Instance (the scheduler doesn\u0027t even have the instance). The other place is in the resource tracker claims code, and we\u0027ll get the information from the limits dict, as noted above. So we can remove \u0027or Instance\u0027 here and replace it with something about the limits dict passed from the scheduler to the compute.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":354,"context_line":"found in ``numa_fit_instance_to_host``, will be updated to consume a"},{"line_number":355,"context_line":"``NetworkRequestList`` object and use this to determine which host NUMA nodes"},{"line_number":356,"context_line":"to use. The ``NetworkRequestList`` object will be extracted from"},{"line_number":357,"context_line":"``RequestSpec`` or ``Instance`` objects and passed in by the caller of this"},{"line_number":358,"context_line":"function."},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"Potential issues"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_9dccd089","line":357,"range":{"start_line":357,"start_character":16,"end_line":357,"end_character":31},"in_reply_to":"5f7c97a3_6fc7c865","updated":"2018-06-01 13:01:31.000000000","message":"Done","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":367,"context_line":"performance. This exact situation can arise in SR-IOV too and generally occurs"},{"line_number":368,"context_line":"because:"},{"line_number":369,"context_line":""},{"line_number":370,"context_line":"- The operator has misconfigured something"},{"line_number":371,"context_line":"- The operator does not care about optimal performance"},{"line_number":372,"context_line":"- The operator wants to provide resiliency by spreading work across NICs and"},{"line_number":373,"context_line":"  cannot or will not use two NICs from the same NUMA node"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_8fa584a1","line":370,"updated":"2018-05-30 22:29:17.000000000","message":"What happens if we get a request for one of these types of ports during server create, but there aren\u0027t any compute hosts which will satisfy the request, either because they are back-level computes (Queens) or they aren\u0027t using the libvirt driver - will scheduling fail with NoValidHost because numa_fit_instance_to_host will reject it?","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":367,"context_line":"performance. This exact situation can arise in SR-IOV too and generally occurs"},{"line_number":368,"context_line":"because:"},{"line_number":369,"context_line":""},{"line_number":370,"context_line":"- The operator has misconfigured something"},{"line_number":371,"context_line":"- The operator does not care about optimal performance"},{"line_number":372,"context_line":"- The operator wants to provide resiliency by spreading work across NICs and"},{"line_number":373,"context_line":"  cannot or will not use two NICs from the same NUMA node"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_30fb68be","line":370,"in_reply_to":"5f7c97a3_8fa584a1","updated":"2018-06-01 13:01:31.000000000","message":"No, it will not reject it as there will be no NUMA affinity defined for those hosts. This is opt-in rather than opt-out: if a user does not define NUMA affinity for a network (or can\u0027t, because of old versions or different virt drivers) then we assume lack of affinity is ok.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":396,"context_line":"traffic is considered out of scope for this spec."},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"The fourth and final situation is interfaces that are attached after creating"},{"line_number":399,"context_line":"the instance. Interfaces must be requested when creating instances and any"},{"line_number":400,"context_line":"interfaces attached after this will not have NUMA affinity applied to them."},{"line_number":401,"context_line":"This is required to limit the scope of this feature and will be resolved in a"},{"line_number":402,"context_line":"future spec."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_8f97c445","line":399,"range":{"start_line":399,"start_character":14,"end_line":399,"end_character":24},"updated":"2018-05-30 22:29:17.000000000","message":"Well, the types of interfaces with numa affinity, right? Can we clarify that here? Maybe just \"NUMA-aware interfaces must be...\".","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":396,"context_line":"traffic is considered out of scope for this spec."},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"The fourth and final situation is interfaces that are attached after creating"},{"line_number":399,"context_line":"the instance. Interfaces must be requested when creating instances and any"},{"line_number":400,"context_line":"interfaces attached after this will not have NUMA affinity applied to them."},{"line_number":401,"context_line":"This is required to limit the scope of this feature and will be resolved in a"},{"line_number":402,"context_line":"future spec."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_108f4c75","line":399,"range":{"start_line":399,"start_character":14,"end_line":399,"end_character":24},"in_reply_to":"5f7c97a3_8f97c445","updated":"2018-06-01 13:01:31.000000000","message":"Done","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":467,"context_line":"persisted and is merely stored as a way to get this information from the API to"},{"line_number":468,"context_line":"the scheduler for use in filtering."},{"line_number":469,"context_line":""},{"line_number":470,"context_line":"A new object, ``NUMANetworkInfo``, will be added to store information about"},{"line_number":471,"context_line":"networks associated with a host NUMA node. This will be mapped to a parent"},{"line_number":472,"context_line":"``NUMACell`` object using a ``network_info`` field."},{"line_number":473,"context_line":""},{"line_number":474,"context_line":"REST API impact"},{"line_number":475,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_af7f2013","line":472,"range":{"start_line":470,"start_character":0,"end_line":472,"end_character":51},"updated":"2018-05-30 22:29:17.000000000","message":"And this is what will get used to compare against the requested networks during numa_fit_instance_to_host via the scheduler and claims code, right?","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":467,"context_line":"persisted and is merely stored as a way to get this information from the API to"},{"line_number":468,"context_line":"the scheduler for use in filtering."},{"line_number":469,"context_line":""},{"line_number":470,"context_line":"A new object, ``NUMANetworkInfo``, will be added to store information about"},{"line_number":471,"context_line":"networks associated with a host NUMA node. This will be mapped to a parent"},{"line_number":472,"context_line":"``NUMACell`` object using a ``network_info`` field."},{"line_number":473,"context_line":""},{"line_number":474,"context_line":"REST API impact"},{"line_number":475,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_90f03cff","line":472,"range":{"start_line":470,"start_character":0,"end_line":472,"end_character":51},"in_reply_to":"5f7c97a3_af7f2013","updated":"2018-06-01 13:01:31.000000000","message":"Correct. Clarified.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":507,"context_line":"Developer impact"},{"line_number":508,"context_line":"----------------"},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"This change will require changes to the scheduler but will only target the"},{"line_number":511,"context_line":"libvirt driver. Additional drivers do not currently provide sufficient"},{"line_number":512,"context_line":"awareness of host NUMA information for this change to be incorporated in a"},{"line_number":513,"context_line":"meaningful manner. If this changes in the future, the drivers will need to be"},{"line_number":514,"context_line":"extended to consume the configuration option and request spec attributes."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_cf9ffc4d","line":511,"range":{"start_line":510,"start_character":0,"end_line":511,"end_character":15},"updated":"2018-05-30 22:29:17.000000000","message":"This reads kind of funny since the scheduler shouldn\u0027t be aware of the virt drivers, I think you just mean that the numa filter will only work with these types of requests when using the libvirt driver since that\u0027s the only driver that will report the new information required for properly scheduling these types of requests.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":507,"context_line":"Developer impact"},{"line_number":508,"context_line":"----------------"},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"This change will require changes to the scheduler but will only target the"},{"line_number":511,"context_line":"libvirt driver. Additional drivers do not currently provide sufficient"},{"line_number":512,"context_line":"awareness of host NUMA information for this change to be incorporated in a"},{"line_number":513,"context_line":"meaningful manner. If this changes in the future, the drivers will need to be"},{"line_number":514,"context_line":"extended to consume the configuration option and request spec attributes."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_f0d6504f","line":511,"range":{"start_line":510,"start_character":0,"end_line":511,"end_character":15},"in_reply_to":"5f7c97a3_cf9ffc4d","updated":"2018-06-01 13:01:31.000000000","message":"Yup, that makes sense. Cleared this up now.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":539,"context_line":"#. Add support for the new configuration option and dynamic configuration"},{"line_number":540,"context_line":"   groups."},{"line_number":541,"context_line":""},{"line_number":542,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":543,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":544,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":545,"context_line":"   scheduling and claiming, respectively."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_0fb3b4ea","line":542,"range":{"start_line":542,"start_character":3,"end_line":542,"end_character":54},"updated":"2018-05-30 22:29:17.000000000","message":"L470 is a bit different than this now - does this line need to be updated?","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":539,"context_line":"#. Add support for the new configuration option and dynamic configuration"},{"line_number":540,"context_line":"   groups."},{"line_number":541,"context_line":""},{"line_number":542,"context_line":"#. Add a ``networks`` field to the ``NUMACell`` object, which will be used to"},{"line_number":543,"context_line":"   store the configuration provided in ``nova.conf``. This is required so"},{"line_number":544,"context_line":"   *nova-scheduler* and *nova-conductor* can access this information during"},{"line_number":545,"context_line":"   scheduling and claiming, respectively."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_30b8480f","line":542,"range":{"start_line":542,"start_character":3,"end_line":542,"end_character":54},"in_reply_to":"5f7c97a3_0fb3b4ea","updated":"2018-06-01 13:01:31.000000000","message":"Yup, it should read \u0027network_info\u0027. Fixed.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"bc53fa9863d3c9ff2b6d8d33d4976891472322be","unresolved":false,"context_lines":[{"line_number":560,"context_line":"   request."},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"#. Modify how claims are done to account for network info when building the"},{"line_number":563,"context_line":"   instance\u0027s NUMA topology."},{"line_number":564,"context_line":""},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_0fe894b0","line":563,"updated":"2018-05-30 22:29:17.000000000","message":"You need another work item to store the network information (physnet/tunneled?) about the ports attached to an instance wrt the numa stuff, into the instance info_cache which gets built by the neutron API client code. That\u0027s how we\u0027ll be able to get the information per-instance if necessary.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"e61c8247a3202b1e7603bfe76029ba54319deb79","unresolved":false,"context_lines":[{"line_number":560,"context_line":"   request."},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"#. Modify how claims are done to account for network info when building the"},{"line_number":563,"context_line":"   instance\u0027s NUMA topology."},{"line_number":564,"context_line":""},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f7c97a3_f06410bb","line":563,"in_reply_to":"5f7c97a3_0fe894b0","updated":"2018-06-01 13:01:31.000000000","message":"Yup, done.","commit_id":"28ae402ce1b6bbbbfabb833e9c339772f57fb5f7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"1698561dcfd91a0e0550b0c97d337ea2bc097417","unresolved":false,"context_lines":[{"line_number":370,"context_line":"performance. This exact situation can arise in SR-IOV too and generally occurs"},{"line_number":371,"context_line":"because:"},{"line_number":372,"context_line":""},{"line_number":373,"context_line":"- The operator has misconfigured something"},{"line_number":374,"context_line":"- The operator does not care about optimal performance"},{"line_number":375,"context_line":"- The operator wants to provide resiliency by spreading work across NICs and"},{"line_number":376,"context_line":"  cannot or will not use two NICs from the same NUMA node"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_ae8b821c","line":373,"updated":"2018-06-01 12:59:33.000000000","message":"I\u0027ll ask this again:\n\nWhat happens if we get a request for one of these types of ports during server create, but there aren\u0027t any compute hosts which will satisfy the request, either because they are back-level computes (Queens) or they aren\u0027t using the libvirt driver - will scheduling fail with NoValidHost because numa_fit_instance_to_host will reject it?","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"4cd8a6d1d9bd97a213f43e7bf672d5d2c6b7c3d7","unresolved":false,"context_lines":[{"line_number":370,"context_line":"performance. This exact situation can arise in SR-IOV too and generally occurs"},{"line_number":371,"context_line":"because:"},{"line_number":372,"context_line":""},{"line_number":373,"context_line":"- The operator has misconfigured something"},{"line_number":374,"context_line":"- The operator does not care about optimal performance"},{"line_number":375,"context_line":"- The operator wants to provide resiliency by spreading work across NICs and"},{"line_number":376,"context_line":"  cannot or will not use two NICs from the same NUMA node"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_6e0b2a7c","line":373,"in_reply_to":"5f7c97a3_ae8b821c","updated":"2018-06-01 13:17:06.000000000","message":"I forgot to submit my comments to the earlier revision. Here\u0027s my reply, for posterity:\n\n\u003e No, it will not reject it as there will be no NUMA affinity defined for those hosts. This is opt-in rather than opt-out: if a user does not define NUMA affinity for a network (or can\u0027t, because of old versions or different virt drivers) then we assume lack of affinity is ok.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"63e4a2997f89fdb4df125b894430836ba0626f5c","unresolved":false,"context_lines":[{"line_number":370,"context_line":"performance. This exact situation can arise in SR-IOV too and generally occurs"},{"line_number":371,"context_line":"because:"},{"line_number":372,"context_line":""},{"line_number":373,"context_line":"- The operator has misconfigured something"},{"line_number":374,"context_line":"- The operator does not care about optimal performance"},{"line_number":375,"context_line":"- The operator wants to provide resiliency by spreading work across NICs and"},{"line_number":376,"context_line":"  cannot or will not use two NICs from the same NUMA node"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_0e2716c9","line":373,"in_reply_to":"5f7c97a3_ae8b821c","updated":"2018-06-01 13:16:04.000000000","message":"Nevermind, I see your response now.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"00dc781d2c1ff403ee8c439884663b617185054c","unresolved":false,"context_lines":[{"line_number":400,"context_line":""},{"line_number":401,"context_line":"The fourth and final situation is interfaces that are attached after creating"},{"line_number":402,"context_line":"the instance. Interfaces for which NUMA affinity is required must be requested"},{"line_number":403,"context_line":"when creating instances and any interfaces attached after this will not have"},{"line_number":404,"context_line":"NUMA affinity applied to them. This is required to limit the scope of this"},{"line_number":405,"context_line":"feature and will be resolved in a future spec."},{"line_number":406,"context_line":""},{"line_number":407,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_2edc52e2","line":404,"range":{"start_line":403,"start_character":24,"end_line":404,"end_character":30},"updated":"2018-06-01 13:07:47.000000000","message":"Per our IRC conversation, it sounds like the attached interfaces would have NUMA affinity applied if the instance is then moved to another host, like during a resize.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"63e4a2997f89fdb4df125b894430836ba0626f5c","unresolved":false,"context_lines":[{"line_number":400,"context_line":""},{"line_number":401,"context_line":"The fourth and final situation is interfaces that are attached after creating"},{"line_number":402,"context_line":"the instance. Interfaces for which NUMA affinity is required must be requested"},{"line_number":403,"context_line":"when creating instances and any interfaces attached after this will not have"},{"line_number":404,"context_line":"NUMA affinity applied to them. This is required to limit the scope of this"},{"line_number":405,"context_line":"feature and will be resolved in a future spec."},{"line_number":406,"context_line":""},{"line_number":407,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_ce1efe6e","line":404,"range":{"start_line":403,"start_character":24,"end_line":404,"end_character":30},"in_reply_to":"5f7c97a3_2edc52e2","updated":"2018-06-01 13:16:04.000000000","message":"If that\u0027s a workaround, we could note it here.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"fca5c8973b56ff1c0715ccc295ccf825e0cfae8c","unresolved":false,"context_lines":[{"line_number":400,"context_line":""},{"line_number":401,"context_line":"The fourth and final situation is interfaces that are attached after creating"},{"line_number":402,"context_line":"the instance. Interfaces for which NUMA affinity is required must be requested"},{"line_number":403,"context_line":"when creating instances and any interfaces attached after this will not have"},{"line_number":404,"context_line":"NUMA affinity applied to them. This is required to limit the scope of this"},{"line_number":405,"context_line":"feature and will be resolved in a future spec."},{"line_number":406,"context_line":""},{"line_number":407,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_49c2e886","line":404,"range":{"start_line":403,"start_character":24,"end_line":404,"end_character":30},"in_reply_to":"5f7c97a3_ce1efe6e","updated":"2018-06-01 13:32:04.000000000","message":"Done","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"1698561dcfd91a0e0550b0c97d337ea2bc097417","unresolved":false,"context_lines":[{"line_number":476,"context_line":"against the requested networks for one or more instances during scheduling and"},{"line_number":477,"context_line":"claiming."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":480,"context_line":"about whether the networks requested for or attached to an instance were"},{"line_number":481,"context_line":"*physnets* or *tunneled networks*."},{"line_number":482,"context_line":""}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_ce935e4e","line":479,"range":{"start_line":479,"start_character":57,"end_line":479,"end_character":62},"updated":"2018-06-01 12:59:33.000000000","message":"We aren\u0027t actually going to store this new object, are we? In the \"Virtualization driver\" section above, you say we just use this object to convert the things from the request spec or the limits dict to pass to numa_fit_instance_to_host.\n\nThe actual physnet/trusted information about a given network for the ports attached to an instance will be stored in the instance.info_cache.network_info.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"63e4a2997f89fdb4df125b894430836ba0626f5c","unresolved":false,"context_lines":[{"line_number":476,"context_line":"against the requested networks for one or more instances during scheduling and"},{"line_number":477,"context_line":"claiming."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":480,"context_line":"about whether the networks requested for or attached to an instance were"},{"line_number":481,"context_line":"*physnets* or *tunneled networks*."},{"line_number":482,"context_line":""}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_ee041a1a","line":479,"range":{"start_line":479,"start_character":57,"end_line":479,"end_character":62},"in_reply_to":"5f7c97a3_ce935e4e","updated":"2018-06-01 13:16:04.000000000","message":"I might just be getting hung up on your usage of the word \"store\" here and below.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"4cd8a6d1d9bd97a213f43e7bf672d5d2c6b7c3d7","unresolved":false,"context_lines":[{"line_number":476,"context_line":"against the requested networks for one or more instances during scheduling and"},{"line_number":477,"context_line":"claiming."},{"line_number":478,"context_line":""},{"line_number":479,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":480,"context_line":"about whether the networks requested for or attached to an instance were"},{"line_number":481,"context_line":"*physnets* or *tunneled networks*."},{"line_number":482,"context_line":""}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_6e800ac0","line":479,"range":{"start_line":479,"start_character":57,"end_line":479,"end_character":62},"in_reply_to":"5f7c97a3_ce935e4e","updated":"2018-06-01 13:17:06.000000000","message":"If you mean it\u0027s not going to be persisted anywhere then no, we\u0027re not going to store this. This will only be used to store information in the ``limits`` dictionary, similar to what we do with NUMATopologyLimits at the moment","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"1698561dcfd91a0e0550b0c97d337ea2bc097417","unresolved":false,"context_lines":[{"line_number":507,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":508,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"Other deployer impact"},{"line_number":511,"context_line":"---------------------"},{"line_number":512,"context_line":""},{"line_number":513,"context_line":"Deployment tooling will need to be enhanced to configure the new configuration"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_4ecd0e5b","line":510,"updated":"2018-06-01 12:59:33.000000000","message":"I don\u0027t see anything in the spec about live migration. I thought about this because we don\u0027t do resource claims during live migration, so how do we know we\u0027re going to land on a host that can provide these types of resources during a live migration, besides the filtering done in the scheduler? And even then, I\u0027m not sure what we\u0027re \"claiming\" from the host so that it\u0027s inventory changes for later scheduling requests. I guess the host NUMACell.network_info information should change?\n\nI\u0027m just wondering if this exposes another gap for these types of instances that won\u0027t work for live migration, like the existing numa-aware live migration spec that artom is working on.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"e0bf1baf63f00c88bddff2ab01d68b3d59d39ff0","unresolved":false,"context_lines":[{"line_number":507,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":508,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"Other deployer impact"},{"line_number":511,"context_line":"---------------------"},{"line_number":512,"context_line":""},{"line_number":513,"context_line":"Deployment tooling will need to be enhanced to configure the new configuration"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_ae15c29d","line":510,"in_reply_to":"5f7c97a3_4ecd0e5b","updated":"2018-06-01 13:01:53.000000000","message":"Also, since we don\u0027t persist the requested_networks in the request spec, during a live migration for filtering to work properly, we\u0027d have to modify the request spec to include the information from the instance info cache....","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"4cd8a6d1d9bd97a213f43e7bf672d5d2c6b7c3d7","unresolved":false,"context_lines":[{"line_number":507,"context_line":"on account of the additional NUMA affinity checks. However, network performance"},{"line_number":508,"context_line":"of some vSwitch solutions (OVS-DPDK, for example) will increase by up to 100%."},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"Other deployer impact"},{"line_number":511,"context_line":"---------------------"},{"line_number":512,"context_line":""},{"line_number":513,"context_line":"Deployment tooling will need to be enhanced to configure the new configuration"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_6906c41e","line":510,"in_reply_to":"5f7c97a3_4ecd0e5b","updated":"2018-06-01 13:17:06.000000000","message":"As discussed on IRC, any instances with NUMA and CPU pinning are currently not live-migratable. This should be resolved with artom\u0027s spec so I guess I just need to call this out.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"63e4a2997f89fdb4df125b894430836ba0626f5c","unresolved":false,"context_lines":[{"line_number":543,"context_line":"Other contributors:"},{"line_number":544,"context_line":"  sean-k-mooney"},{"line_number":545,"context_line":""},{"line_number":546,"context_line":"Work Items"},{"line_number":547,"context_line":"----------"},{"line_number":548,"context_line":""},{"line_number":549,"context_line":"#. Add support for the new configuration option and dynamic configuration"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_a9005c02","line":546,"updated":"2018-06-01 13:16:04.000000000","message":"Please add a work item that during any move operation, conductor will have to modify the request spec to include the proper information from the instance info cache for the numa-aware interfaces so the NUMATopologyFilter can properly filter destination hosts.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"fca5c8973b56ff1c0715ccc295ccf825e0cfae8c","unresolved":false,"context_lines":[{"line_number":543,"context_line":"Other contributors:"},{"line_number":544,"context_line":"  sean-k-mooney"},{"line_number":545,"context_line":""},{"line_number":546,"context_line":"Work Items"},{"line_number":547,"context_line":"----------"},{"line_number":548,"context_line":""},{"line_number":549,"context_line":"#. Add support for the new configuration option and dynamic configuration"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_098ab062","line":546,"in_reply_to":"5f7c97a3_a9005c02","updated":"2018-06-01 13:32:04.000000000","message":"Done","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"1698561dcfd91a0e0550b0c97d337ea2bc097417","unresolved":false,"context_lines":[{"line_number":573,"context_line":"   request. This field will not be persisted as it is only needed by the"},{"line_number":574,"context_line":"   scheduler when scheduling new instances."},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"#. Create a ``InstanceNUMANetworks`` object. This will be used to store a"},{"line_number":577,"context_line":"   simplified version of the ``NetworkRequestList`` object, listing only any"},{"line_number":578,"context_line":"   *physnets* or *tunneled* attributes of the networks requested for the"},{"line_number":579,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_eea97a7b","line":576,"range":{"start_line":576,"start_character":66,"end_line":576,"end_character":71},"updated":"2018-06-01 12:59:33.000000000","message":"Again, just to be clear, we aren\u0027t actually persisting this, right? This is just the object used to pass into numa_fit_instance_to_host, yes?","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"4cd8a6d1d9bd97a213f43e7bf672d5d2c6b7c3d7","unresolved":false,"context_lines":[{"line_number":573,"context_line":"   request. This field will not be persisted as it is only needed by the"},{"line_number":574,"context_line":"   scheduler when scheduling new instances."},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"#. Create a ``InstanceNUMANetworks`` object. This will be used to store a"},{"line_number":577,"context_line":"   simplified version of the ``NetworkRequestList`` object, listing only any"},{"line_number":578,"context_line":"   *physnets* or *tunneled* attributes of the networks requested for the"},{"line_number":579,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_29144c43","line":576,"range":{"start_line":576,"start_character":66,"end_line":576,"end_character":71},"in_reply_to":"5f7c97a3_eea97a7b","updated":"2018-06-01 13:17:06.000000000","message":"Correct","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"1698561dcfd91a0e0550b0c97d337ea2bc097417","unresolved":false,"context_lines":[{"line_number":590,"context_line":""},{"line_number":591,"context_line":"#. Modify the serialized models stored in ``Instance.info_cache.network_info``"},{"line_number":592,"context_line":"   to include something akin to the ``physnet`` and ``tunneled`` fields for"},{"line_number":593,"context_line":"   each network. This is required for migration and unshelve claims."},{"line_number":594,"context_line":""},{"line_number":595,"context_line":"#. Modify how claims are done to account for network info from the ``limits``"},{"line_number":596,"context_line":"   when building the instance\u0027s NUMA topology."}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_eec2da33","line":593,"range":{"start_line":593,"start_character":17,"end_line":593,"end_character":68},"updated":"2018-06-01 12:59:33.000000000","message":"nit: evacuate also does claims, but I guess that could be considered like a cold migration (except the original disk is lost if not using shared storage).","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"4cd8a6d1d9bd97a213f43e7bf672d5d2c6b7c3d7","unresolved":false,"context_lines":[{"line_number":590,"context_line":""},{"line_number":591,"context_line":"#. Modify the serialized models stored in ``Instance.info_cache.network_info``"},{"line_number":592,"context_line":"   to include something akin to the ``physnet`` and ``tunneled`` fields for"},{"line_number":593,"context_line":"   each network. This is required for migration and unshelve claims."},{"line_number":594,"context_line":""},{"line_number":595,"context_line":"#. Modify how claims are done to account for network info from the ``limits``"},{"line_number":596,"context_line":"   when building the instance\u0027s NUMA topology."}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_e9197479","line":593,"range":{"start_line":593,"start_character":17,"end_line":593,"end_character":68},"in_reply_to":"5f7c97a3_eec2da33","updated":"2018-06-01 13:17:06.000000000","message":"Done","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"38a737ab021485105868a1d95dab5ede8f9d6c29","unresolved":false,"context_lines":[{"line_number":599,"context_line":"Dependencies"},{"line_number":600,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":601,"context_line":""},{"line_number":602,"context_line":"None."},{"line_number":603,"context_line":""},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"Testing"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_ae4dc29a","line":602,"updated":"2018-06-01 13:12:33.000000000","message":"Do we depend on https://blueprints.launchpad.net/nova/+spec/numa-aware-live-migration for live migration support for these types of instances? If so, call that out.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"fca5c8973b56ff1c0715ccc295ccf825e0cfae8c","unresolved":false,"context_lines":[{"line_number":599,"context_line":"Dependencies"},{"line_number":600,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":601,"context_line":""},{"line_number":602,"context_line":"None."},{"line_number":603,"context_line":""},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"Testing"}],"source_content_type":"text/x-rst","patch_set":15,"id":"5f7c97a3_69a244fe","line":602,"in_reply_to":"5f7c97a3_ae4dc29a","updated":"2018-06-01 13:32:04.000000000","message":"No, it needs that spec for live migration to work, but so does everything that involves NUMA and CPU pinning (seeing as they\u0027re very much tied). That spec should fix all use cases in this area including this particular one. I\u0027ve noted that here.","commit_id":"a3def8bcc8846f9763a031cd4b7a45265160603a"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"dfe149ad248cc74bc3b666106b445d36539b380f","unresolved":false,"context_lines":[{"line_number":345,"context_line":"``NUMATopologyFilter`` to include information about the networks. This is"},{"line_number":346,"context_line":"necessary so we have something to reference during the *claim* stage. In both"},{"line_number":347,"context_line":"cases, the cached information will need to be updated in the event of a move"},{"line_number":348,"context_line":"operation."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Virtualization driver"},{"line_number":351,"context_line":"~~~~~~~~~~~~~~~~~~~~~"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_499f484f","line":348,"updated":"2018-06-01 13:53:15.000000000","message":"I still wish we could have something in here about _what_ the scheduler is trying to do. When I go to review the code, I won\u0027t really be able to look at the filter changes and refer to the spec about the intent.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d1977c2191298da0a7cc34c95588ac62251e25d4","unresolved":false,"context_lines":[{"line_number":345,"context_line":"``NUMATopologyFilter`` to include information about the networks. This is"},{"line_number":346,"context_line":"necessary so we have something to reference during the *claim* stage. In both"},{"line_number":347,"context_line":"cases, the cached information will need to be updated in the event of a move"},{"line_number":348,"context_line":"operation."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Virtualization driver"},{"line_number":351,"context_line":"~~~~~~~~~~~~~~~~~~~~~"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_6911e45c","line":348,"in_reply_to":"5f7c97a3_499f484f","updated":"2018-06-01 14:11:51.000000000","message":"Yeah the scheduler bits are sprinkled throughout, like in the data model impact section.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"dfe149ad248cc74bc3b666106b445d36539b380f","unresolved":false,"context_lines":[{"line_number":464,"context_line":"Data model impact"},{"line_number":465,"context_line":"-----------------"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"The ``NetworkRequest`` object will gain two new fields, a list of strings"},{"line_number":468,"context_line":"field called ``physnet`` and a boolean field called ``tunneled``. These will"},{"line_number":469,"context_line":"store additional information about a given network to avoid having to fetch"},{"line_number":470,"context_line":"this information repeatedly. They will be populated by a more generic version"},{"line_number":471,"context_line":"of the ``create_pci_requests_for_sriov_ports`` function. They will not be"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_09d39002","line":468,"range":{"start_line":467,"start_character":0,"end_line":468,"end_character":65},"updated":"2018-06-01 13:53:15.000000000","message":"I thought we agreed not to put non-request information into the request object? This seems like a shortcut to get these fields to the scheduler, but it kinda sucks to pollute this object for just that reason.\n\nEverywhere else that we use this object will have no need for these fields, right?","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d1977c2191298da0a7cc34c95588ac62251e25d4","unresolved":false,"context_lines":[{"line_number":464,"context_line":"Data model impact"},{"line_number":465,"context_line":"-----------------"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"The ``NetworkRequest`` object will gain two new fields, a list of strings"},{"line_number":468,"context_line":"field called ``physnet`` and a boolean field called ``tunneled``. These will"},{"line_number":469,"context_line":"store additional information about a given network to avoid having to fetch"},{"line_number":470,"context_line":"this information repeatedly. They will be populated by a more generic version"},{"line_number":471,"context_line":"of the ``create_pci_requests_for_sriov_ports`` function. They will not be"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_492188b3","line":468,"range":{"start_line":467,"start_character":0,"end_line":468,"end_character":65},"in_reply_to":"5f7c97a3_09d39002","updated":"2018-06-01 14:11:51.000000000","message":"\u003e This seems like a shortcut to get these fields to the scheduler\n\nIt totally is, yes, and follows what gibi is doing in the bandwidth-aware scheduling series.\n\n\u003e Everywhere else that we use this object will have no need for these fields, right?\n\nI guess, but we\u0027ll still need to re-populate this in conductor for every move operation as well. So it\u0027s either that, or select_destinations grows a new parameter.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8c30a390640cab010bf93c5d431e0d0632bcfddf","unresolved":false,"context_lines":[{"line_number":464,"context_line":"Data model impact"},{"line_number":465,"context_line":"-----------------"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"The ``NetworkRequest`` object will gain two new fields, a list of strings"},{"line_number":468,"context_line":"field called ``physnet`` and a boolean field called ``tunneled``. These will"},{"line_number":469,"context_line":"store additional information about a given network to avoid having to fetch"},{"line_number":470,"context_line":"this information repeatedly. They will be populated by a more generic version"},{"line_number":471,"context_line":"of the ``create_pci_requests_for_sriov_ports`` function. They will not be"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_e4a99b7b","line":468,"range":{"start_line":467,"start_character":0,"end_line":468,"end_character":65},"in_reply_to":"5f7c97a3_492188b3","updated":"2018-06-05 09:56:47.000000000","message":"Aye, this is the closest thing we have to cache of neutron network info at the scheduling phase. I either have to use this or I have to make additional requests to neutron.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"dfe149ad248cc74bc3b666106b445d36539b380f","unresolved":false,"context_lines":[{"line_number":476,"context_line":"networks associated with a host NUMA node. This will be mapped to a parent"},{"line_number":477,"context_line":"``NUMACell`` object using a ``network_info`` field and will be used to compare"},{"line_number":478,"context_line":"against the requested networks for one or more instances during scheduling and"},{"line_number":479,"context_line":"claiming."},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":482,"context_line":"about whether the networks requested for or attached to an instance were"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_890ec089","line":479,"updated":"2018-06-01 13:53:15.000000000","message":"So this goes to the scheduler and compute but is never persisted?","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d1977c2191298da0a7cc34c95588ac62251e25d4","unresolved":false,"context_lines":[{"line_number":476,"context_line":"networks associated with a host NUMA node. This will be mapped to a parent"},{"line_number":477,"context_line":"``NUMACell`` object using a ``network_info`` field and will be used to compare"},{"line_number":478,"context_line":"against the requested networks for one or more instances during scheduling and"},{"line_number":479,"context_line":"claiming."},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":482,"context_line":"about whether the networks requested for or attached to an instance were"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_e432fbf1","line":479,"in_reply_to":"5f7c97a3_890ec089","updated":"2018-06-01 14:11:51.000000000","message":"I assume it will be, because it goes into NUMACell which is in NUMAToplogy which is stored in the ComputeNode object.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8c30a390640cab010bf93c5d431e0d0632bcfddf","unresolved":false,"context_lines":[{"line_number":476,"context_line":"networks associated with a host NUMA node. This will be mapped to a parent"},{"line_number":477,"context_line":"``NUMACell`` object using a ``network_info`` field and will be used to compare"},{"line_number":478,"context_line":"against the requested networks for one or more instances during scheduling and"},{"line_number":479,"context_line":"claiming."},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":482,"context_line":"about whether the networks requested for or attached to an instance were"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_8484670c","line":479,"in_reply_to":"5f7c97a3_e432fbf1","updated":"2018-06-05 09:56:47.000000000","message":"This is persisted for things like building the ComputeNode/HostState objects.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"dfe149ad248cc74bc3b666106b445d36539b380f","unresolved":false,"context_lines":[{"line_number":480,"context_line":""},{"line_number":481,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":482,"context_line":"about whether the networks requested for or attached to an instance were"},{"line_number":483,"context_line":"*physnets* or *tunneled networks*."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"REST API impact"},{"line_number":486,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_09ecf0b8","line":483,"updated":"2018-06-01 13:53:15.000000000","message":"Why not build a list of these and pass them through the ReqSpec to the scheduler for each network in requested networks?\n\nAlso, does this ever get persisted?","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d1977c2191298da0a7cc34c95588ac62251e25d4","unresolved":false,"context_lines":[{"line_number":480,"context_line":""},{"line_number":481,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":482,"context_line":"about whether the networks requested for or attached to an instance were"},{"line_number":483,"context_line":"*physnets* or *tunneled networks*."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"REST API impact"},{"line_number":486,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_2961ac05","line":483,"in_reply_to":"5f7c97a3_09ecf0b8","updated":"2018-06-01 14:11:51.000000000","message":"\u003e Why not build a list of these and pass them through the ReqSpec to the scheduler for each network in requested networks?\n\nI just thought about the same thing - rather than modify NetworkRequest(List) at all, we can just use this new object for both the RequestSpec (to get it to the scheduler) and for the limits (to get it from the scheduler to the compute). That would be better than doing:\n\napi -\u003e NetworkRequestList in RequestSpec -\u003e scheduler -\u003e InstanceNUMANetworks in limits -\u003e compute\n\n\u003e Also, does this ever get persisted?\n\nNo.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"1c15186cf4ce16000b534cfb91685e39d020501a","unresolved":false,"context_lines":[{"line_number":480,"context_line":""},{"line_number":481,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":482,"context_line":"about whether the networks requested for or attached to an instance were"},{"line_number":483,"context_line":"*physnets* or *tunneled networks*."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"REST API impact"},{"line_number":486,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_7f28b631","line":483,"in_reply_to":"5f7c97a3_2961ac05","updated":"2018-06-01 15:09:34.000000000","message":"\u003e I just thought about the same thing - rather than modify\n \u003e NetworkRequest(List) at all, we can just use this new object for\n \u003e both the RequestSpec (to get it to the scheduler) and for the\n \u003e limits (to get it from the scheduler to the compute). That would be\n \u003e better than doing:\n\nRight that avoids mixing non-request stuff into NetworkRequest as well.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8c30a390640cab010bf93c5d431e0d0632bcfddf","unresolved":false,"context_lines":[{"line_number":480,"context_line":""},{"line_number":481,"context_line":"A new object, ``InstanceNUMANetworks``, will be added to store information"},{"line_number":482,"context_line":"about whether the networks requested for or attached to an instance were"},{"line_number":483,"context_line":"*physnets* or *tunneled networks*."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"REST API impact"},{"line_number":486,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_e2bab86c","line":483,"in_reply_to":"5f7c97a3_7f28b631","updated":"2018-06-05 09:56:47.000000000","message":"Done","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d1977c2191298da0a7cc34c95588ac62251e25d4","unresolved":false,"context_lines":[{"line_number":568,"context_line":"   to fetch most of the information we need to populate the new"},{"line_number":569,"context_line":"   ``NetworkRequest`` fields."},{"line_number":570,"context_line":""},{"line_number":571,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":572,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":573,"context_line":"   ``NetworkRequestList`` object. This information will be used by the"},{"line_number":574,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"},{"line_number":575,"context_line":"   request. This field will not be persisted as it is only needed by the"},{"line_number":576,"context_line":"   scheduler when scheduling new instances."}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_8470e7a5","line":573,"range":{"start_line":571,"start_character":3,"end_line":573,"end_character":33},"updated":"2018-06-01 14:11:51.000000000","message":"Can this just be an instance of InstanceNUMANetworks instead?","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8c30a390640cab010bf93c5d431e0d0632bcfddf","unresolved":false,"context_lines":[{"line_number":568,"context_line":"   to fetch most of the information we need to populate the new"},{"line_number":569,"context_line":"   ``NetworkRequest`` fields."},{"line_number":570,"context_line":""},{"line_number":571,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":572,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":573,"context_line":"   ``NetworkRequestList`` object. This information will be used by the"},{"line_number":574,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"},{"line_number":575,"context_line":"   request. This field will not be persisted as it is only needed by the"},{"line_number":576,"context_line":"   scheduler when scheduling new instances."}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_c4a57fb6","line":573,"range":{"start_line":571,"start_character":3,"end_line":573,"end_character":33},"in_reply_to":"5f7c97a3_8470e7a5","updated":"2018-06-05 09:56:47.000000000","message":"I has thought gibi would need more of this information than just physnets/tunneled but apparently not [1]. I can change this.\n\n[1] https://specs.openstack.org/openstack/nova-specs/specs/rocky/approved/bandwidth-resource-provider.html#resource-request-in-the-port","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d1977c2191298da0a7cc34c95588ac62251e25d4","unresolved":false,"context_lines":[{"line_number":585,"context_line":"   ``InstanceNUMANetworks``, and consider these when building NUMA topologies."},{"line_number":586,"context_line":""},{"line_number":587,"context_line":"#. Modify the ``NUMATopologyFilter`` to populate a ``network_requests`` field"},{"line_number":588,"context_line":"   for the ``limits`` dictionary. This will be populated with an instance of"},{"line_number":589,"context_line":"   ``InstanceNUMANetworks`` built using the value of"},{"line_number":590,"context_line":"   ``RequestSpec.requested_networks``. Pass ``InstanceNUMANetworks`` object to"},{"line_number":591,"context_line":"   ``numa_fit_instance_to_host``."},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"#. Modify the serialized models stored in ``Instance.info_cache.network_info``"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_c47a5f86","line":590,"range":{"start_line":588,"start_character":34,"end_line":590,"end_character":37},"updated":"2018-06-01 14:11:51.000000000","message":"As noted above, can we just skip modifying NetworkRequestList and just make RequestSpec.requested_networks an instance of InstanceNUMANetworks? Then we can pass that through during scheduling w/o any object translation, and put it into the limits returned from the filter to the compute as well.","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8c30a390640cab010bf93c5d431e0d0632bcfddf","unresolved":false,"context_lines":[{"line_number":585,"context_line":"   ``InstanceNUMANetworks``, and consider these when building NUMA topologies."},{"line_number":586,"context_line":""},{"line_number":587,"context_line":"#. Modify the ``NUMATopologyFilter`` to populate a ``network_requests`` field"},{"line_number":588,"context_line":"   for the ``limits`` dictionary. This will be populated with an instance of"},{"line_number":589,"context_line":"   ``InstanceNUMANetworks`` built using the value of"},{"line_number":590,"context_line":"   ``RequestSpec.requested_networks``. Pass ``InstanceNUMANetworks`` object to"},{"line_number":591,"context_line":"   ``numa_fit_instance_to_host``."},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"#. Modify the serialized models stored in ``Instance.info_cache.network_info``"}],"source_content_type":"text/x-rst","patch_set":16,"id":"5f7c97a3_02bc946a","line":590,"range":{"start_line":588,"start_character":34,"end_line":590,"end_character":37},"in_reply_to":"5f7c97a3_c47a5f86","updated":"2018-06-05 09:56:47.000000000","message":"Done","commit_id":"657a5af75c15496edc6a99cc8a64de900262de18"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"b449fdea8931513cc00572045a7c5cbf0626a77c","unresolved":false,"context_lines":[{"line_number":322,"context_line":"about *physnets* and VNIC types of each port requested when booting an instance"},{"line_number":323,"context_line":"for the purpose of creating PCI requests for SRIOV ports. This will be made"},{"line_number":324,"context_line":"more generic and extended to get information about the tunneled status of any"},{"line_number":325,"context_line":"requested networks. This information will need in a new object,"},{"line_number":326,"context_line":"``InstanceNUMANetworks``, for later consumption."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_25d8b51c","line":325,"range":{"start_line":325,"start_character":42,"end_line":325,"end_character":46},"updated":"2018-06-05 13:58:20.000000000","message":"be stored (but not persisted) in a new object","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17cb3f7498b99ced080f2594f43455c828fd884b","unresolved":false,"context_lines":[{"line_number":322,"context_line":"about *physnets* and VNIC types of each port requested when booting an instance"},{"line_number":323,"context_line":"for the purpose of creating PCI requests for SRIOV ports. This will be made"},{"line_number":324,"context_line":"more generic and extended to get information about the tunneled status of any"},{"line_number":325,"context_line":"requested networks. This information will need in a new object,"},{"line_number":326,"context_line":"``InstanceNUMANetworks``, for later consumption."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_65af8d2b","line":325,"range":{"start_line":325,"start_character":42,"end_line":325,"end_character":46},"in_reply_to":"5f7c97a3_25d8b51c","updated":"2018-06-05 14:09:43.000000000","message":"Done","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"b449fdea8931513cc00572045a7c5cbf0626a77c","unresolved":false,"context_lines":[{"line_number":344,"context_line":"scheduling but it does not need to be stored after this. We also need to extend"},{"line_number":345,"context_line":"the ``limits`` dictionary returned by the ``NUMATopologyFilter`` to include"},{"line_number":346,"context_line":"information about the networks. This is necessary so we have something to"},{"line_number":347,"context_line":"reference during the *claim* stage. In both cases, the cached information will"},{"line_number":348,"context_line":"need to be updated in the event of a move operation."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Virtualization driver"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_65f78d8e","line":347,"range":{"start_line":347,"start_character":29,"end_line":347,"end_character":34},"updated":"2018-06-05 13:58:20.000000000","message":"To be clear about the \u0027claim stage\u0027 I\u0027d say:\n\n\u0027stage on the compute\u0027.","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17cb3f7498b99ced080f2594f43455c828fd884b","unresolved":false,"context_lines":[{"line_number":344,"context_line":"scheduling but it does not need to be stored after this. We also need to extend"},{"line_number":345,"context_line":"the ``limits`` dictionary returned by the ``NUMATopologyFilter`` to include"},{"line_number":346,"context_line":"information about the networks. This is necessary so we have something to"},{"line_number":347,"context_line":"reference during the *claim* stage. In both cases, the cached information will"},{"line_number":348,"context_line":"need to be updated in the event of a move operation."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Virtualization driver"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_85ac2936","line":347,"range":{"start_line":347,"start_character":29,"end_line":347,"end_character":34},"in_reply_to":"5f7c97a3_65f78d8e","updated":"2018-06-05 14:09:43.000000000","message":"Done","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"b449fdea8931513cc00572045a7c5cbf0626a77c","unresolved":false,"context_lines":[{"line_number":563,"context_line":"   to fetch most of the information we need to populate the new"},{"line_number":564,"context_line":"   ``InstanceNUMANetworks`` fields."},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":567,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":568,"context_line":"   ``InstanceNUMANetworks`` object. This information will be used by the"},{"line_number":569,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_653c6d30","line":566,"range":{"start_line":566,"start_character":11,"end_line":566,"end_character":29},"updated":"2018-06-05 13:58:20.000000000","message":"I expect we might want to rename this now, maybe to something like numa_networks? The RequestSpec already has these fields:\n\n        \u0027numa_topology\u0027: fields.ObjectField(\u0027InstanceNUMATopology\u0027,\n                                            nullable\u003dTrue),\n        \u0027pci_requests\u0027: fields.ObjectField(\u0027InstancePCIRequests\u0027,\n                                           nullable\u003dTrue),\n\nSo this name would follow that pattern.","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17cb3f7498b99ced080f2594f43455c828fd884b","unresolved":false,"context_lines":[{"line_number":563,"context_line":"   to fetch most of the information we need to populate the new"},{"line_number":564,"context_line":"   ``InstanceNUMANetworks`` fields."},{"line_number":565,"context_line":""},{"line_number":566,"context_line":"#. Add a ``requested_networks`` field to the ``RequestSpec`` object, which will"},{"line_number":567,"context_line":"   include a list of networks requested by the instance in the form of a"},{"line_number":568,"context_line":"   ``InstanceNUMANetworks`` object. This information will be used by the"},{"line_number":569,"context_line":"   ``NUMATopologyFilter`` to filter out hosts that could not satisfy the"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_e5607d10","line":566,"range":{"start_line":566,"start_character":11,"end_line":566,"end_character":29},"in_reply_to":"5f7c97a3_653c6d30","updated":"2018-06-05 14:09:43.000000000","message":"Done","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"b449fdea8931513cc00572045a7c5cbf0626a77c","unresolved":false,"context_lines":[{"line_number":577,"context_line":"#. Modify the ``NUMATopologyFilter`` to populate a ``network_requests`` field"},{"line_number":578,"context_line":"   for the ``limits`` dictionary. This will be populated with an instance of"},{"line_number":579,"context_line":"   ``InstanceNUMANetworks`` built using the value of"},{"line_number":580,"context_line":"   ``RequestSpec.requested_networks``. Pass ``InstanceNUMANetworks`` object to"},{"line_number":581,"context_line":"   ``numa_fit_instance_to_host``."},{"line_number":582,"context_line":""},{"line_number":583,"context_line":"#. Modify the serialized models stored in ``Instance.info_cache.network_info``"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_251f55b8","line":580,"range":{"start_line":580,"start_character":17,"end_line":580,"end_character":35},"updated":"2018-06-05 13:58:20.000000000","message":"rename","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17cb3f7498b99ced080f2594f43455c828fd884b","unresolved":false,"context_lines":[{"line_number":577,"context_line":"#. Modify the ``NUMATopologyFilter`` to populate a ``network_requests`` field"},{"line_number":578,"context_line":"   for the ``limits`` dictionary. This will be populated with an instance of"},{"line_number":579,"context_line":"   ``InstanceNUMANetworks`` built using the value of"},{"line_number":580,"context_line":"   ``RequestSpec.requested_networks``. Pass ``InstanceNUMANetworks`` object to"},{"line_number":581,"context_line":"   ``numa_fit_instance_to_host``."},{"line_number":582,"context_line":""},{"line_number":583,"context_line":"#. Modify the serialized models stored in ``Instance.info_cache.network_info``"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_a56a85ea","line":580,"range":{"start_line":580,"start_character":17,"end_line":580,"end_character":35},"in_reply_to":"5f7c97a3_251f55b8","updated":"2018-06-05 14:09:43.000000000","message":"Done","commit_id":"a5e8ed19ea424d8968ba894bf9be6f4547cdafd8"}]}
