)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     Artom Lifshitz \u003califshit@redhat.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2020-12-04 10:27:56 -0500"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[WIP] PCI Socket Affnity"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in"},{"line_number":10,"context_line":"the kinds of affinity that it can express. Either a PCI device has"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"d04be63a_671cdc23","line":7,"range":{"start_line":7,"start_character":17,"end_line":7,"end_character":24},"updated":"2020-12-07 15:59:34.000000000","message":"Affinity","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":34537,"name":"Sonemaly Phrasavath","email":"psonemal@amd.com","username":"sphrasavath"},"change_message_id":"b41eb505f7067190d84060521e19f52bee9b85c2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"a98bc278_f48b6486","updated":"2022-02-03 03:12:48.000000000","message":"This is great work.  Thank you.  The socket policy definitely gives more flexible for optimization that could lead to better performance on a single socket and dual socket config with that caveat that a VM could fit nicely into 1 NUMA node, i.e. hw_numa_nodes\u003d1.  Where hw_numa_nodes \u003e 1 and there is only 1 PCI device on a 2 socket system, the possibility of spreading VM assignment across two socket will result in higher latency.  It\u0027s just something that we have to keep in mind when validating for the most optimal HW and SW configs. ","commit_id":"da78f4f4edd289d10b3a6011b13e275ae5ad7f0b"}],"specs/wallaby/approved/pci-socket-policy.rst":[{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"PCI Socket Policy"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/pci-socket-affinity"}],"source_content_type":"text/x-rst","patch_set":1,"id":"32a8679e_c45dc590","line":8,"range":{"start_line":8,"start_character":11,"end_line":8,"end_character":17},"updated":"2020-12-07 15:59:34.000000000","message":"NUMA-affinity Policy ?","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9a7f5935c8e6dcac75537ebaff22d2c9a60fd664","unresolved":true,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"PCI Socket Policy"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/pci-socket-affinity"}],"source_content_type":"text/x-rst","patch_set":1,"id":"55e0f566_72494976","line":8,"range":{"start_line":8,"start_character":11,"end_line":8,"end_character":17},"in_reply_to":"32a8679e_c45dc590","updated":"2020-12-07 17:36:11.000000000","message":"+1","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"30183db4e33e692ab07ba08f8b1a727a22030de3","unresolved":true,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"PCI Socket Policy"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/pci-socket-affinity"}],"source_content_type":"text/x-rst","patch_set":1,"id":"b89d76fa_739cb05e","line":8,"range":{"start_line":8,"start_character":11,"end_line":8,"end_character":17},"in_reply_to":"55e0f566_72494976","updated":"2021-01-12 21:25:29.000000000","message":"I can\u0027t change the blueprint URL because I already filed it, but I can change the title.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/pci-socket-affinity"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in the"},{"line_number":14,"context_line":"kinds of affinity that it can express. Either a PCI device has affnity for a"},{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5a44086a_2f8b9f3f","line":14,"range":{"start_line":14,"start_character":63,"end_line":14,"end_character":70},"updated":"2020-12-07 15:59:34.000000000","message":"affinity","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"30183db4e33e692ab07ba08f8b1a727a22030de3","unresolved":false,"context_lines":[{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/pci-socket-affinity"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in the"},{"line_number":14,"context_line":"kinds of affinity that it can express. Either a PCI device has affnity for a"},{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"6735d0d7_8759d616","line":14,"range":{"start_line":14,"start_character":63,"end_line":14,"end_character":70},"in_reply_to":"5a44086a_2f8b9f3f","updated":"2021-01-12 21:25:29.000000000","message":"Done","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in the"},{"line_number":14,"context_line":"kinds of affinity that it can express. Either a PCI device has affnity for a"},{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9b7e1724_3f4a4b63","line":15,"range":{"start_line":15,"start_character":60,"end_line":15,"end_character":72},"updated":"2020-12-07 15:59:34.000000000","message":"processors\u0027","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"30183db4e33e692ab07ba08f8b1a727a22030de3","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in the"},{"line_number":14,"context_line":"kinds of affinity that it can express. Either a PCI device has affnity for a"},{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"}],"source_content_type":"text/x-rst","patch_set":1,"id":"a1f720f0_a513d0a7","line":15,"range":{"start_line":15,"start_character":60,"end_line":15,"end_character":72},"in_reply_to":"9b7e1724_3f4a4b63","updated":"2021-01-12 21:25:29.000000000","message":"Done","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":13,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in the"},{"line_number":14,"context_line":"kinds of affinity that it can express. Either a PCI device has affnity for a"},{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"},{"line_number":19,"context_line":"installed into a single physical socket. The internal processor layout makes it"}],"source_content_type":"text/x-rst","patch_set":1,"id":"63f86a0b_77acbdea","line":16,"updated":"2020-12-07 15:59:34.000000000","message":"Not really true. Cluster-on-Die (CoD) has been around for a long time but the lower number of NUMA nodes (meaning more CPUs and memory per node) meant sticking with strict NUMA affinity was still a reasonable proposition. Also, I think (from talking to Sean) the penalties on disabling CoD on these platforms and making multiple NUMA nodes appear as one wasn\u0027t as severe as the penalty we get with AMD Zen 2.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9a7f5935c8e6dcac75537ebaff22d2c9a60fd664","unresolved":true,"context_lines":[{"line_number":13,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in the"},{"line_number":14,"context_line":"kinds of affinity that it can express. Either a PCI device has affnity for a"},{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"},{"line_number":19,"context_line":"installed into a single physical socket. The internal processor layout makes it"}],"source_content_type":"text/x-rst","patch_set":1,"id":"d0518d54_5d3f8204","line":16,"in_reply_to":"63f86a0b_77acbdea","updated":"2020-12-07 17:36:11.000000000","message":"actuly it was about 70-80% of the penaltiy as going across socket but when intel moved to the mesh archatechture instead of rings buses it was reduced significantly.\n\nit is still a resonable performace hit for itenl cpus even with a mesh however.\nwhat has really changed is its now more apprent that this will afect multiple vendors and possible multiple architure so this is a more generic problem where as before it was niche to jsut intel in one specific configurtin that was disbaled by default.\n\ncluster on die or sub numa clustering as it becamse was really aimed at hpc workloads but its also relevent for telcos and with amd zen basced chip its more releveln for more workloads effectivly all numa workloads.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"30183db4e33e692ab07ba08f8b1a727a22030de3","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Nova\u0027s current support for NUMA affinity for PCI devices is limited in the"},{"line_number":14,"context_line":"kinds of affinity that it can express. Either a PCI device has affnity for a"},{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"},{"line_number":19,"context_line":"installed into a single physical socket. The internal processor layout makes it"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9f72ca50_29df61d4","line":16,"in_reply_to":"d0518d54_5d3f8204","updated":"2021-01-12 21:25:29.000000000","message":"Done","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"},{"line_number":19,"context_line":"installed into a single physical socket. The internal processor layout makes it"},{"line_number":20,"context_line":"so that PCI devices have affinity to a specific NUMA node, despite all the"},{"line_number":21,"context_line":"nodes sharing the same socket."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"}],"source_content_type":"text/x-rst","patch_set":1,"id":"01342310_acb9f5b7","line":21,"range":{"start_line":18,"start_character":0,"end_line":21,"end_character":30},"updated":"2020-12-07 15:59:34.000000000","message":"I don\u0027t think this is quite the issue. It\u0027s more that older processors did *not* have more than one NUMA node on a single socket, meaning we could roughly equate a NUMA node to a socket.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9a7f5935c8e6dcac75537ebaff22d2c9a60fd664","unresolved":true,"context_lines":[{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"},{"line_number":19,"context_line":"installed into a single physical socket. The internal processor layout makes it"},{"line_number":20,"context_line":"so that PCI devices have affinity to a specific NUMA node, despite all the"},{"line_number":21,"context_line":"nodes sharing the same socket."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9df8a9b9_66be04a0","line":21,"range":{"start_line":18,"start_character":0,"end_line":21,"end_character":30},"in_reply_to":"01342310_acb9f5b7","updated":"2020-12-07 17:36:11.000000000","message":"if older you mean pre haswell \nwhich was released in 2013 and post nehlem which was released in 2008\n\n\npre nehelm the memocy contoler was on the mother board so all systems were UMA.\nfrom nehlem on the memory contoler moved to the cpu.\n\nintel added cluster on die in 2013 with haswell and they reduced the perfoamcne impact in skylake\nhttps://www.anandtech.com/show/8423/intel-xeon-e5-version-3-up-to-18-haswell-ep-cores-/4\n\nthe haswell chip with more then 10 cores had 2 or even 3 different internal partion which were interconnected with at least 1 memory contoler per ring. with cluster on die off the chip was presented as a singel numa node but with cluster on die on the physical toplogy fo the cpus was exposed to the os with each memory contoler running independely as its own numa region.\n\nin when skylake was intoduced in 2017 it intoduced the mesh architrue\nand also change the cache heriacy.\nhttps://en.wikichip.org/wiki/intel/microarchitectures/skylake_(server)#Memory_Hierarchy\ncluster on die was rebarnded as sub numa clustering\nhttps://en.wikichip.org/wiki/intel/microarchitectures/skylake_(server)#Sub-NUMA_Clustering\nwith some other enhancements.\n\nwith the mesh toplogy the latency between cores/memoyc contoler was significantly reduced which masked the impact of not having socket affinty for singel socket servers as you can enable multiple numa nodes in the 1 socket and use the perfer policy. however since we dont have socket affinity currently it means that on multi socket systems using prefer could result in a pci device form a different socket which will have a signicant penelty due to needing to cross the QPI/UPI bus.\n\nso this has been a issue for 7+ year although some hardware optimisation have hiddine it recently on the intel side it does affect both amd and intel to different degrees depending on the generation.\n\nthis likely also holds true for non x86 achitrues too.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"30183db4e33e692ab07ba08f8b1a727a22030de3","unresolved":true,"context_lines":[{"line_number":15,"context_line":"NUMA node, or no affinity at all. This is modelled on older processors\u0027s"},{"line_number":16,"context_line":"layout, and has worked fine until now."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Older processors package multiple NUMA nodes into a single package, which is"},{"line_number":19,"context_line":"installed into a single physical socket. The internal processor layout makes it"},{"line_number":20,"context_line":"so that PCI devices have affinity to a specific NUMA node, despite all the"},{"line_number":21,"context_line":"nodes sharing the same socket."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ce2530ca_96b41728","line":21,"range":{"start_line":18,"start_character":0,"end_line":21,"end_character":30},"in_reply_to":"9df8a9b9_66be04a0","updated":"2021-01-12 21:25:29.000000000","message":"Yeah, not sure what I was thinking writing this. Changed.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":20,"context_line":"so that PCI devices have affinity to a specific NUMA node, despite all the"},{"line_number":21,"context_line":"nodes sharing the same socket."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"},{"line_number":25,"context_line":"generation AMD EPYC processors have a central IO chiplet per socket, with a"},{"line_number":26,"context_line":"configurable number of NUMA nodes on top. PCI devices are affined to this"}],"source_content_type":"text/x-rst","patch_set":1,"id":"57f29c43_8fb55831","line":23,"range":{"start_line":23,"start_character":43,"end_line":23,"end_character":46},"updated":"2020-12-07 15:59:34.000000000","message":"As above, this also impacts Intel processors with Cluster on Die functionality.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"30183db4e33e692ab07ba08f8b1a727a22030de3","unresolved":false,"context_lines":[{"line_number":20,"context_line":"so that PCI devices have affinity to a specific NUMA node, despite all the"},{"line_number":21,"context_line":"nodes sharing the same socket."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"},{"line_number":25,"context_line":"generation AMD EPYC processors have a central IO chiplet per socket, with a"},{"line_number":26,"context_line":"configurable number of NUMA nodes on top. PCI devices are affined to this"}],"source_content_type":"text/x-rst","patch_set":1,"id":"c0729d96_600c49d3","line":23,"range":{"start_line":23,"start_character":43,"end_line":23,"end_character":46},"in_reply_to":"57f29c43_8fb55831","updated":"2021-01-12 21:25:29.000000000","message":"Done","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"547a39427c22e77299b7321de81fa8e4af534a50","unresolved":true,"context_lines":[{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"},{"line_number":25,"context_line":"generation AMD EPYC processors have a central IO chiplet per socket, with a"},{"line_number":26,"context_line":"configurable number of NUMA nodes on top. PCI devices are affined to this"},{"line_number":27,"context_line":"central chiplet, as opposed to individual NUMA nodes. In other words, Nova"},{"line_number":28,"context_line":"needs a way to express PCI affinity to a socket. This spec proposes a way to"},{"line_number":29,"context_line":"implement that."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dbbc5776_6a7296dc","line":27,"range":{"start_line":26,"start_character":42,"end_line":27,"end_character":53},"updated":"2020-12-07 15:59:34.000000000","message":"Is that true? I thought they were still affined to a single node, or at least that was how Linux would present them? The actual difference was that latency between nodes on the same socket was _significantly_ lower than latency between nodes on different sockets, so we wanted a way to say we could accept the former but not the latter.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"30183db4e33e692ab07ba08f8b1a727a22030de3","unresolved":true,"context_lines":[{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"},{"line_number":25,"context_line":"generation AMD EPYC processors have a central IO chiplet per socket, with a"},{"line_number":26,"context_line":"configurable number of NUMA nodes on top. PCI devices are affined to this"},{"line_number":27,"context_line":"central chiplet, as opposed to individual NUMA nodes. In other words, Nova"},{"line_number":28,"context_line":"needs a way to express PCI affinity to a socket. This spec proposes a way to"},{"line_number":29,"context_line":"implement that."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"a42260fb_1c9d0412","line":27,"range":{"start_line":26,"start_character":42,"end_line":27,"end_character":53},"in_reply_to":"2a533170_06627bf9","updated":"2021-01-12 21:25:29.000000000","message":"So my understanding is that they\u0027re electrically wired to that central chiplet, but the memory mapping is per memory controller, as Sean says. In any case you\u0027re right, what we want is to stay within the socket, because the penalty for going cross-socket is way worse than cross-memory-controller. I\u0027ve left that out of he problem description in my next revision, but I\u0027ll mention it in the proposed chance in a subsequent revision.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9a7f5935c8e6dcac75537ebaff22d2c9a60fd664","unresolved":true,"context_lines":[{"line_number":23,"context_line":"More recent processors, in particular from AMD, have a different internal"},{"line_number":24,"context_line":"layout. This makes Nova\u0027s current NUMA node affinity model insufficient. Second"},{"line_number":25,"context_line":"generation AMD EPYC processors have a central IO chiplet per socket, with a"},{"line_number":26,"context_line":"configurable number of NUMA nodes on top. PCI devices are affined to this"},{"line_number":27,"context_line":"central chiplet, as opposed to individual NUMA nodes. In other words, Nova"},{"line_number":28,"context_line":"needs a way to express PCI affinity to a socket. This spec proposes a way to"},{"line_number":29,"context_line":"implement that."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"2a533170_06627bf9","line":27,"range":{"start_line":26,"start_character":42,"end_line":27,"end_character":53},"in_reply_to":"dbbc5776_6a7296dc","updated":"2020-12-07 17:36:11.000000000","message":"yes stephen is correct.\n\nthe mmio region for the pci bars will still be mapped to an adress space ownwed by one fo the memory\ncontoler on the socket and that is what give the pci device a numa affinity\n\nbut the latency between cores onthe same socket to the memroy contoler is much less then form cores on a different socket.\n\nthat is why socket affintiy is a nice middel ground between stict affinity(same numa node) and no affinity.","commit_id":"8a39acc86ed01140d7464910677f8a516f1edd8c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"540a34671fb12c75482e2237437269f18f4bb163","unresolved":true,"context_lines":[{"line_number":46,"context_line":"minimal performance penalties."},{"line_number":47,"context_line":""},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Proposed change"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Here is where you cover the change you propose to make in detail. How do you"},{"line_number":53,"context_line":"propose to solve this problem?"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"If this is one part of a larger effort make it clear where this piece ends. In"},{"line_number":56,"context_line":"other words, what\u0027s the scope of this effort?"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"At this point, if you would like to just get feedback on if the problem and"},{"line_number":59,"context_line":"proposed change fit in nova, you can stop here and post this for review to get"},{"line_number":60,"context_line":"preliminary feedback. If so please say:"},{"line_number":61,"context_line":"Posting to get preliminary feedback on the scope of this spec."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"Alternatives"},{"line_number":64,"context_line":"------------"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"What other ways could we do this thing? Why aren\u0027t we using those? This doesn\u0027t"},{"line_number":67,"context_line":"have to be a full literature review, but it should demonstrate that thought has"},{"line_number":68,"context_line":"been put into why the proposed solution is an appropriate one."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Data model impact"},{"line_number":71,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"c5e11956_408fea5c","line":68,"range":{"start_line":49,"start_character":0,"end_line":68,"end_character":62},"updated":"2021-01-12 21:26:56.000000000","message":"This is still missing, and is where I\u0027ll discuss the whole central IO chiplet vs memory controller thing.","commit_id":"9008385debf6339c49068825aa4ad1bbbbe08efc"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"540a34671fb12c75482e2237437269f18f4bb163","unresolved":true,"context_lines":[{"line_number":121,"context_line":"Upgrade impact"},{"line_number":122,"context_line":"--------------"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Only Wallaby compute hosts will be able to honor the new `socket` NUMA-affinity"},{"line_number":125,"context_line":"policy."},{"line_number":126,"context_line":""},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":2,"id":"db603919_56ddfd4d","line":125,"range":{"start_line":124,"start_character":0,"end_line":125,"end_character":7},"updated":"2021-01-12 21:26:56.000000000","message":"Need to expand on this.","commit_id":"9008385debf6339c49068825aa4ad1bbbbe08efc"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"`socket` PCI NUMA-affinity Policy"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/pci-socket-affinity"}],"source_content_type":"text/x-rst","patch_set":3,"id":"8b87b41b_4f05d818","line":8,"range":{"start_line":8,"start_character":17,"end_line":8,"end_character":18},"updated":"2021-01-14 17:56:45.000000000","message":"I think you can drop this (you\u0027re switching between including it and dropping it throughout)","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"`socket` PCI NUMA-affinity Policy"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/pci-socket-affinity"}],"source_content_type":"text/x-rst","patch_set":3,"id":"47eccc40_9be4bf13","line":8,"range":{"start_line":8,"start_character":17,"end_line":8,"end_character":18},"in_reply_to":"8b87b41b_4f05d818","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":15,"context_line":"NUMA node, or no affinity at all. This makes one of two assumptions about the"},{"line_number":16,"context_line":"underlying host NUMA topology. Either there is only a single NUMA node per"},{"line_number":17,"context_line":"socket, or - for cluster on die topologies with multiple nodes per socket -"},{"line_number":18,"context_line":"there are enough CPUs in each NUMA node to fit reasonnably large VMs that"},{"line_number":19,"context_line":"require strict PCI NUMA affinity. The latter assumption is no longer true, and"},{"line_number":20,"context_line":"Nova needs a more nuanced way to express PCI NUMA affinity."},{"line_number":21,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"8e6184f4_e4ff0e38","line":18,"range":{"start_line":18,"start_character":52,"end_line":18,"end_character":53},"updated":"2021-01-14 17:56:45.000000000","message":"drop","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":15,"context_line":"NUMA node, or no affinity at all. This makes one of two assumptions about the"},{"line_number":16,"context_line":"underlying host NUMA topology. Either there is only a single NUMA node per"},{"line_number":17,"context_line":"socket, or - for cluster on die topologies with multiple nodes per socket -"},{"line_number":18,"context_line":"there are enough CPUs in each NUMA node to fit reasonnably large VMs that"},{"line_number":19,"context_line":"require strict PCI NUMA affinity. The latter assumption is no longer true, and"},{"line_number":20,"context_line":"Nova needs a more nuanced way to express PCI NUMA affinity."},{"line_number":21,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"fd54cce4_66a17041","line":18,"range":{"start_line":18,"start_character":52,"end_line":18,"end_character":53},"in_reply_to":"8e6184f4_e4ff0e38","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":37,"context_line":".. seealso:: Zen2 BIOSes have a L3AsNUMA configration option, which creates a"},{"line_number":38,"context_line":"   NUMA node for every level 3 cache. There can be up to 4 cores sharing an L3"},{"line_number":39,"context_line":"   cache, and 2 SMT threads per core. This is how the number 8 was arrived at."},{"line_number":40,"context_line":"   See [zen2-topology]_ for more details."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Use Cases"},{"line_number":43,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"659cd643_fdd8e203","line":40,"range":{"start_line":40,"start_character":3,"end_line":40,"end_character":40},"updated":"2021-01-14 17:56:45.000000000","message":"This will render as \n\n  See [zen2-topology] for more details.\n\nin the doc. Perhaps something like:\n\n  See the AMD Developer Documentation [1]_ for more details.\n\n?","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":37,"context_line":".. seealso:: Zen2 BIOSes have a L3AsNUMA configration option, which creates a"},{"line_number":38,"context_line":"   NUMA node for every level 3 cache. There can be up to 4 cores sharing an L3"},{"line_number":39,"context_line":"   cache, and 2 SMT threads per core. This is how the number 8 was arrived at."},{"line_number":40,"context_line":"   See [zen2-topology]_ for more details."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Use Cases"},{"line_number":43,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"496d4f00_1ef93e9d","line":40,"range":{"start_line":40,"start_character":3,"end_line":40,"end_character":40},"in_reply_to":"659cd643_fdd8e203","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":42,"context_line":"Use Cases"},{"line_number":43,"context_line":"---------"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"As an NFV cloud operator, I want to make full use of my Zen2 hardware with"},{"line_number":46,"context_line":"minimal performance penalties."},{"line_number":47,"context_line":""},{"line_number":48,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"d50b0fcd_823ed552","line":45,"range":{"start_line":45,"start_character":56,"end_line":45,"end_character":60},"updated":"2021-01-14 17:56:45.000000000","message":"strike - this isn\u0027t specific to Zen2. Certain Intel processors can exhibit the same issues unless steps are taken (namely, disabling cluser-on-die)","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Use Cases"},{"line_number":43,"context_line":"---------"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"As an NFV cloud operator, I want to make full use of my Zen2 hardware with"},{"line_number":46,"context_line":"minimal performance penalties."},{"line_number":47,"context_line":""},{"line_number":48,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"930d651a_526ccdfe","line":45,"range":{"start_line":45,"start_character":56,"end_line":45,"end_character":60},"in_reply_to":"d50b0fcd_823ed552","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":49,"context_line":"Proposed change"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"This spec proposes a new value for the `hw:pci_numa_affinity_policy` (and the"},{"line_number":53,"context_line":"`hw_pci_numa_affinity_policy` image property). The value is `socket`, and it"},{"line_number":54,"context_line":"indicates that the instance\u0027s PCI device has to be affined to the same socket"},{"line_number":55,"context_line":"as the host CPUs that it is pinned to. If no such devices are available on any"}],"source_content_type":"text/x-rst","patch_set":3,"id":"4aebc526_ca3e4207","line":52,"range":{"start_line":52,"start_character":39,"end_line":52,"end_character":40},"updated":"2021-01-14 17:56:45.000000000","message":"This needs to be doubled for rST. Single backticks means \"default role\", which iirc is italics but can be configured\n\nHere and elsewhere","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":49,"context_line":"Proposed change"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"This spec proposes a new value for the `hw:pci_numa_affinity_policy` (and the"},{"line_number":53,"context_line":"`hw_pci_numa_affinity_policy` image property). The value is `socket`, and it"},{"line_number":54,"context_line":"indicates that the instance\u0027s PCI device has to be affined to the same socket"},{"line_number":55,"context_line":"as the host CPUs that it is pinned to. If no such devices are available on any"}],"source_content_type":"text/x-rst","patch_set":3,"id":"b2e6fb33_773cd3d2","line":52,"range":{"start_line":52,"start_character":39,"end_line":52,"end_character":40},"in_reply_to":"4aebc526_ca3e4207","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":54,"context_line":"indicates that the instance\u0027s PCI device has to be affined to the same socket"},{"line_number":55,"context_line":"as the host CPUs that it is pinned to. If no such devices are available on any"},{"line_number":56,"context_line":"compute hosts, the instance fails to schedule. In that sense, `socket` is the"},{"line_number":57,"context_line":"same as `require`, except the PCI device must \"come from\" the same socket,"},{"line_number":58,"context_line":"rather than the same NUMA node."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The implementation requires tracking what socket host CPUs and PCI devices"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1b763a5e_e18acac2","line":57,"range":{"start_line":57,"start_character":47,"end_line":57,"end_character":56},"updated":"2021-01-14 17:56:45.000000000","message":"must \"belong\" to ?","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":54,"context_line":"indicates that the instance\u0027s PCI device has to be affined to the same socket"},{"line_number":55,"context_line":"as the host CPUs that it is pinned to. If no such devices are available on any"},{"line_number":56,"context_line":"compute hosts, the instance fails to schedule. In that sense, `socket` is the"},{"line_number":57,"context_line":"same as `require`, except the PCI device must \"come from\" the same socket,"},{"line_number":58,"context_line":"rather than the same NUMA node."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The implementation requires tracking what socket host CPUs and PCI devices"}],"source_content_type":"text/x-rst","patch_set":3,"id":"2313480d_dae9fd83","line":57,"range":{"start_line":57,"start_character":47,"end_line":57,"end_character":56},"in_reply_to":"1b763a5e_e18acac2","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2d7d597a9a6f5ce039515ccf1b69573a07cba64d","unresolved":true,"context_lines":[{"line_number":56,"context_line":"compute hosts, the instance fails to schedule. In that sense, `socket` is the"},{"line_number":57,"context_line":"same as `require`, except the PCI device must \"come from\" the same socket,"},{"line_number":58,"context_line":"rather than the same NUMA node."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The implementation requires tracking what socket host CPUs and PCI devices"},{"line_number":61,"context_line":"are affined to. The libvirt driver obtains that information from libvirt\u0027s host"},{"line_number":62,"context_line":"capabilities and nodedev XML, respectively, and saves it in the `PciDevice` and"}],"source_content_type":"text/x-rst","patch_set":3,"id":"84bebb2d_cf640b6c","line":59,"updated":"2021-01-15 11:16:16.000000000","message":"Does hw:pci_numa_affinity_policy\u003dsocket affects how the vCPUs will be pinned? Based on the name I would not expect that. So I would expect that with numa_nodes\u003d1 the vCPUs are pinned to a single numa node\u0027s pCPUs. BUT it allows that the PCI device comes from another numa node until that numa node still belongs to the same socket as the numa node the vCPUs are using. \n\nAlso with numa_nodes\u003d2 the vCPUs will pinned to pCPUs from two numa nodes might be on two different sockets and in this case. The PCI device can come from the all the numa nodes of these two sockets.\n\nIs it a correct understanding?","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"852eebf41f5d7c79e2a6d5bb904770a47868e863","unresolved":true,"context_lines":[{"line_number":56,"context_line":"compute hosts, the instance fails to schedule. In that sense, `socket` is the"},{"line_number":57,"context_line":"same as `require`, except the PCI device must \"come from\" the same socket,"},{"line_number":58,"context_line":"rather than the same NUMA node."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The implementation requires tracking what socket host CPUs and PCI devices"},{"line_number":61,"context_line":"are affined to. The libvirt driver obtains that information from libvirt\u0027s host"},{"line_number":62,"context_line":"capabilities and nodedev XML, respectively, and saves it in the `PciDevice` and"}],"source_content_type":"text/x-rst","patch_set":3,"id":"2e1340b0_f0f9347b","line":59,"in_reply_to":"1ef223b4_a09e17b1","updated":"2021-01-19 14:02:12.000000000","message":"thanks. I refined my question based on your example.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":true,"context_lines":[{"line_number":56,"context_line":"compute hosts, the instance fails to schedule. In that sense, `socket` is the"},{"line_number":57,"context_line":"same as `require`, except the PCI device must \"come from\" the same socket,"},{"line_number":58,"context_line":"rather than the same NUMA node."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The implementation requires tracking what socket host CPUs and PCI devices"},{"line_number":61,"context_line":"are affined to. The libvirt driver obtains that information from libvirt\u0027s host"},{"line_number":62,"context_line":"capabilities and nodedev XML, respectively, and saves it in the `PciDevice` and"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1ef223b4_a09e17b1","line":59,"in_reply_to":"84bebb2d_cf640b6c","updated":"2021-01-18 19:13:19.000000000","message":"I\u0027ve expanded this section with diagrams, I hope they\u0027ll help :)","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"06e91e2390522ca0435908ad59ab28f29b08aedc","unresolved":true,"context_lines":[{"line_number":69,"context_line":""},{"line_number":70,"context_line":"There are no alternative with a similar level of simplicity. A more complex"},{"line_number":71,"context_line":"model could include numeric NUMA distances and/or PCI root complex electrical"},{"line_number":72,"context_line":"connection vs memory mapping affinity. The potential guest performance gains"},{"line_number":73,"context_line":"are not worth the massive complexity."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Data model impact"},{"line_number":76,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"a1216bd7_d03abde7","line":73,"range":{"start_line":72,"start_character":39,"end_line":73,"end_character":37},"updated":"2021-01-14 18:22:20.000000000","message":"well they might be but not untill we rule out that the simple case is not good enough in partice.\nwe may eventuallly go ther but we dont have to go there now untill we show the simple socket affinity case does not meet the performance requirements.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":true,"context_lines":[{"line_number":69,"context_line":""},{"line_number":70,"context_line":"There are no alternative with a similar level of simplicity. A more complex"},{"line_number":71,"context_line":"model could include numeric NUMA distances and/or PCI root complex electrical"},{"line_number":72,"context_line":"connection vs memory mapping affinity. The potential guest performance gains"},{"line_number":73,"context_line":"are not worth the massive complexity."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Data model impact"},{"line_number":76,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"898d8a3f_a8fe3f75","line":73,"range":{"start_line":72,"start_character":39,"end_line":73,"end_character":37},"in_reply_to":"a1216bd7_d03abde7","updated":"2021-01-18 19:13:19.000000000","message":"OK, I\u0027ll just drop this sentence then.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":75,"context_line":"Data model impact"},{"line_number":76,"context_line":"-----------------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"* A `socket` integer field is added to the `PciDevice` object and the"},{"line_number":79,"context_line":"  corresponding `pci_device` table."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"* A `socket` integer field is added to the `NUMACell` object. No database"},{"line_number":82,"context_line":"  changes are necessary here, as the object is stored as a JSON blob."}],"source_content_type":"text/x-rst","patch_set":3,"id":"083d1cfb_a42cba68","line":79,"range":{"start_line":78,"start_character":0,"end_line":79,"end_character":35},"updated":"2021-01-14 17:56:45.000000000","message":"Note that we probably don\u0027t strictly need to do this. We have can pull the NUMA node of the PCI device from the \u0027PCIDevice.numa_node\u0027 field, and we can then look at the \u0027NUMACell\u0027 object whose \u0027id\u0027 corresponds to this to find the socket. That would be a little more normalized, but is possibly more work.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"06e91e2390522ca0435908ad59ab28f29b08aedc","unresolved":true,"context_lines":[{"line_number":75,"context_line":"Data model impact"},{"line_number":76,"context_line":"-----------------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"* A `socket` integer field is added to the `PciDevice` object and the"},{"line_number":79,"context_line":"  corresponding `pci_device` table."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"* A `socket` integer field is added to the `NUMACell` object. No database"},{"line_number":82,"context_line":"  changes are necessary here, as the object is stored as a JSON blob."}],"source_content_type":"text/x-rst","patch_set":3,"id":"492d2cc7_1402c9ae","line":79,"range":{"start_line":78,"start_character":0,"end_line":79,"end_character":35},"updated":"2021-01-14 18:22:20.000000000","message":"there is an extra_info blob inside which we store the pci device cablitiys so if you add teh socket to that you wont need a database schema change for this either since that is also just a json blob.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":true,"context_lines":[{"line_number":75,"context_line":"Data model impact"},{"line_number":76,"context_line":"-----------------"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"* A `socket` integer field is added to the `PciDevice` object and the"},{"line_number":79,"context_line":"  corresponding `pci_device` table."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"* A `socket` integer field is added to the `NUMACell` object. No database"},{"line_number":82,"context_line":"  changes are necessary here, as the object is stored as a JSON blob."}],"source_content_type":"text/x-rst","patch_set":3,"id":"c40108da_15cd2c2e","line":79,"range":{"start_line":78,"start_character":0,"end_line":79,"end_character":35},"in_reply_to":"083d1cfb_a42cba68","updated":"2021-01-18 19:13:19.000000000","message":"I was on the fence about this, as both approaches have pros and cons.\n\ndatabase pros:\n* calculate and save the socket ID once, access anywhere\n\ndatabase cons:\n* It\u0027s a database migration\n\ncalculate on the fly pros:\n* No database migration!\n* More flexibility for the future\n\ncalculate on the fly cons:\n* Have to loop up the socket every time\n\nI think with Sean\u0027s comments that we might extend/change this in the future, I\u0027d rather not impose a database migration on everyone, and stick to calculating it on the fly.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2d7d597a9a6f5ce039515ccf1b69573a07cba64d","unresolved":true,"context_lines":[{"line_number":88,"context_line":""},{"line_number":89,"context_line":"No API changes per se, and definitely no new microversion. A new `socket` value"},{"line_number":90,"context_line":"is added to the list of possible values for the `hw:pci_numa_affinity_policy`"},{"line_number":91,"context_line":"flavor extra spec and the `hw_pci_numa_affinity_policy` image property."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Security impact"},{"line_number":94,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"53643000_ed800c1c","line":91,"updated":"2021-01-15 11:16:16.000000000","message":"and extending the extra_spec validation to allow the new value","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":88,"context_line":""},{"line_number":89,"context_line":"No API changes per se, and definitely no new microversion. A new `socket` value"},{"line_number":90,"context_line":"is added to the list of possible values for the `hw:pci_numa_affinity_policy`"},{"line_number":91,"context_line":"flavor extra spec and the `hw_pci_numa_affinity_policy` image property."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Security impact"},{"line_number":94,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"f92e8507_6aa6e853","line":91,"in_reply_to":"53643000_ed800c1c","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"06e91e2390522ca0435908ad59ab28f29b08aedc","unresolved":true,"context_lines":[{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Only Wallaby compute hosts will be able to honor the new `socket` NUMA-affinity"},{"line_number":130,"context_line":"policy. However, the scheduler uses the same `numa_fit_instance_to_host()`"},{"line_number":131,"context_line":"entry point that this spec will modify. Therefore, in a rolling upgrade"},{"line_number":132,"context_line":"situation, only Wallaby compute hosts will pass scheduling."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":3,"id":"a5d4ef9c_769a8ab0","line":132,"range":{"start_line":131,"start_character":40,"end_line":132,"end_character":59},"updated":"2021-01-14 18:22:20.000000000","message":"so you inded to make it fail if the socket info is not present to facialitate this?\nthat should work we could also use a trait+prefilter but the existing numa toplogy filter can work too.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":true,"context_lines":[{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Only Wallaby compute hosts will be able to honor the new `socket` NUMA-affinity"},{"line_number":130,"context_line":"policy. However, the scheduler uses the same `numa_fit_instance_to_host()`"},{"line_number":131,"context_line":"entry point that this spec will modify. Therefore, in a rolling upgrade"},{"line_number":132,"context_line":"situation, only Wallaby compute hosts will pass scheduling."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":3,"id":"c8186e5d_2f7dca70","line":132,"range":{"start_line":131,"start_character":40,"end_line":132,"end_character":59},"in_reply_to":"a5d4ef9c_769a8ab0","updated":"2021-01-18 19:13:19.000000000","message":"Yeah, that\u0027d be a way to short-circuit the scheduling, although I\u0027m not sure how clean it would be: to really short-circuit it we would have to check the PCI policy close to the very beginning of numa_fit_instance_to_host(), whereas currently all the PCI policy stuff is done (rightly so) in pci/stats.py.\n\nI figure since the scheduler will be running the new Wallaby code (err, including the hardware.py and pci/stats.py stuff, right? You\u0027re making be doubt myself) it\u0027ll automagically exclude hosts that don\u0027t report CPU socket affinity.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a32c0cefcbdad45aa0d7198d6ea524f02b7f373c","unresolved":true,"context_lines":[{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Only Wallaby compute hosts will be able to honor the new `socket` NUMA-affinity"},{"line_number":130,"context_line":"policy. However, the scheduler uses the same `numa_fit_instance_to_host()`"},{"line_number":131,"context_line":"entry point that this spec will modify. Therefore, in a rolling upgrade"},{"line_number":132,"context_line":"situation, only Wallaby compute hosts will pass scheduling."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":3,"id":"72e91cd4_ddc8c1f4","line":132,"range":{"start_line":131,"start_character":40,"end_line":132,"end_character":59},"in_reply_to":"c8186e5d_2f7dca70","updated":"2021-01-19 13:11:47.000000000","message":"Right, finally caught up with what you had in mind - spec amended.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2d7d597a9a6f5ce039515ccf1b69573a07cba64d","unresolved":true,"context_lines":[{"line_number":139,"context_line":"-----------"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Primary assignee:"},{"line_number":142,"context_line":"  notartom"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Feature Liaison"},{"line_number":145,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3bdb7ad1_e87d976e","line":142,"updated":"2021-01-15 11:16:16.000000000","message":"then who?","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":true,"context_lines":[{"line_number":139,"context_line":"-----------"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Primary assignee:"},{"line_number":142,"context_line":"  notartom"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Feature Liaison"},{"line_number":145,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"79e076ab_3eec44b2","line":142,"in_reply_to":"3bdb7ad1_e87d976e","updated":"2021-01-18 19:13:19.000000000","message":"Oy ;) https://launchpad.net/~notartom","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"852eebf41f5d7c79e2a6d5bb904770a47868e863","unresolved":false,"context_lines":[{"line_number":139,"context_line":"-----------"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Primary assignee:"},{"line_number":142,"context_line":"  notartom"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Feature Liaison"},{"line_number":145,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"d3bafbb9_0092d794","line":142,"in_reply_to":"79e076ab_3eec44b2","updated":"2021-01-19 14:02:12.000000000","message":"hah lol.","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"2d7d597a9a6f5ce039515ccf1b69573a07cba64d","unresolved":true,"context_lines":[{"line_number":156,"context_line":"* Start populating the new `socket` fields"},{"line_number":157,"context_line":"* Modify `PciDeviceStats._filter_pools()`, as called by"},{"line_number":158,"context_line":"  `PciDeviceStats.support_requests()`, to support the new `socket`"},{"line_number":159,"context_line":"  NUMA-affinity policy."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":3,"id":"eed81025_1d896dd0","line":159,"updated":"2021-01-15 11:16:16.000000000","message":"* extend flavor extra_spec validation to allow the new value","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":156,"context_line":"* Start populating the new `socket` fields"},{"line_number":157,"context_line":"* Modify `PciDeviceStats._filter_pools()`, as called by"},{"line_number":158,"context_line":"  `PciDeviceStats.support_requests()`, to support the new `socket`"},{"line_number":159,"context_line":"  NUMA-affinity policy."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bc931ada_cd96e715","line":159,"in_reply_to":"eed81025_1d896dd0","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"f0a8e719f21e4b5bee19dd20052b59187247a3c7","unresolved":true,"context_lines":[{"line_number":168,"context_line":"Testing"},{"line_number":169,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"While there are aspirations for AMD Zen2 hardware in a third pary CI, that is"},{"line_number":172,"context_line":"too far in the future to have any impact on this spec. Functional tests will"},{"line_number":173,"context_line":"have to do."},{"line_number":174,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"30fbe2c8_20d88cd5","line":171,"range":{"start_line":171,"start_character":61,"end_line":171,"end_character":65},"updated":"2021-01-14 17:56:45.000000000","message":"party","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"40bf14efc581d4037b7bfb70aaf39290ed44fc23","unresolved":false,"context_lines":[{"line_number":168,"context_line":"Testing"},{"line_number":169,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"While there are aspirations for AMD Zen2 hardware in a third pary CI, that is"},{"line_number":172,"context_line":"too far in the future to have any impact on this spec. Functional tests will"},{"line_number":173,"context_line":"have to do."},{"line_number":174,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"b41bf56b_98f03103","line":171,"range":{"start_line":171,"start_character":61,"end_line":171,"end_character":65},"in_reply_to":"30fbe2c8_20d88cd5","updated":"2021-01-18 19:13:19.000000000","message":"Done","commit_id":"a3b0e415895819f5ea91aa6d2ad0a83805cada42"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"852eebf41f5d7c79e2a6d5bb904770a47868e863","unresolved":true,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"But it *cannot* be pinned to either:"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* NUMA node N0 and N2"},{"line_number":83,"context_line":"* NUMA node N0 and N3"},{"line_number":84,"context_line":"* NUMA node N1 and N2"},{"line_number":85,"context_line":"* NUMA node N1 and N3"}],"source_content_type":"text/x-rst","patch_set":5,"id":"4673620f_68756147","line":82,"range":{"start_line":82,"start_character":12,"end_line":82,"end_character":21},"updated":"2021-01-19 14:02:12.000000000","message":"But N0 and N2 would be allowed by hw_numa_nodes\u003d2 alone as that allows splitting CPUs between two numa nodes. \n\nDo you mean that if there is a single PCI device requested and hw_pci_numa_affinity_policy\u003dsocket then N0 and N2 is not allowed as the single PCI is either close to N0 close to N2 but not close to both of them.\n\nIf yes, then would the above request with 2 PCI devices could succeed as if the two PCI devices also split between the N0,N1 and N2,N3 a same way as the CPUs are split between these nodes?","commit_id":"a475771be07c1e83e2435b0147a9eed879bcd95b"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"9d156515603d5b043e6d040e37c94d724a591af2","unresolved":true,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"But it *cannot* be pinned to either:"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* NUMA node N0 and N2"},{"line_number":83,"context_line":"* NUMA node N0 and N3"},{"line_number":84,"context_line":"* NUMA node N1 and N2"},{"line_number":85,"context_line":"* NUMA node N1 and N3"}],"source_content_type":"text/x-rst","patch_set":5,"id":"e3b89748_559fb0e4","line":82,"range":{"start_line":82,"start_character":12,"end_line":82,"end_character":21},"in_reply_to":"4673620f_68756147","updated":"2021-01-19 14:28:46.000000000","message":"Good questions...\n\nSince the purpose of ``socket`` is to prevent crossing the socket barrier to access the PCI device, with hw_numa_nodes\u003d2 and policy\u003dsocket, both guest NUMA nodes would have to be on the same host socket (so your second paragraph is correct).\n\nIn fact, the presence of a single PCI device with policy\u003dsocket effectively forces all your guest NUMA nodes to come from the same host socket, regardless of how many there are. If there\u0027s a second PCI device with a different policy, nothing changes in how it\u0027s treated, *once* all non-same-socket host NUMA nodes have been eliminated from the possibilities.\n\nLemme amend the spec :)","commit_id":"a475771be07c1e83e2435b0147a9eed879bcd95b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3e65095c1ab0f349360ad0c8928e8020a4bdfe29","unresolved":true,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"But it *cannot* be pinned to either:"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* NUMA node N0 and N2"},{"line_number":83,"context_line":"* NUMA node N0 and N3"},{"line_number":84,"context_line":"* NUMA node N1 and N2"},{"line_number":85,"context_line":"* NUMA node N1 and N3"}],"source_content_type":"text/x-rst","patch_set":5,"id":"16c64678_a85ee466","line":82,"range":{"start_line":82,"start_character":12,"end_line":82,"end_character":21},"in_reply_to":"4673620f_68756147","updated":"2021-01-19 14:39:40.000000000","message":"i agree with gibi here this is inconsitent withthe existing behavior of multiple numa node and it is not the semantics of the socket affinity policy that we discussed befoer.\n\nhw_numa_nodes\u003d2 with hw_pci_numa_affinity_policy\u003drequire has always ment that re require strict affinity between the numa node of the pci device and any virtual numa node of the\nguest.\n\ne.g. if you had hw_numa_nodes\u003d2 with hw_pci_numa_affinity_policy\u003drequire N1 and N2 is valid today.\n\nwhen we have hw_numa_nodes\u003d2 with hw_pci_numa_affinity_policy\u003dsocket then the following are valid\n\nN0 and N1\nN0 and N2\nN0 and N3\nN1 and N2\nN1 and N3\ni am ignroing mirroings\n\nwitht eh above host toplogy with only 1 pci device attach to socket 0\nN2 and N3 is not valid as it does not meet the socket affintiy requiremnt since the pci device is connected to a host socket that is remote to both of the guest numa nodes.\n\n\nhw_pci_numa_affinity_policy\u003dsocket does not mean that all the guests numa nodes will come form the same socket.\n\nit means that at least 1 of the virtual numa nodes  of the guest is mapped to the same socket on the host as the pci device.","commit_id":"a475771be07c1e83e2435b0147a9eed879bcd95b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3e65095c1ab0f349360ad0c8928e8020a4bdfe29","unresolved":true,"context_lines":[{"line_number":93,"context_line":"  +----------+         +----------+"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"In other words, ``hw_pci_numa_affinity_policy\u003dsocket`` prevents crossing the"},{"line_number":96,"context_line":"socket boundary to access the PCI device. While there is a performance penalty"},{"line_number":97,"context_line":"for crossing the NUMA node boundary, it is smaller than crossing the socket"},{"line_number":98,"context_line":"boundary. It is this latter performance penalty that this spec addresses."},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"cc8805d4_a100bc45","line":96,"updated":"2021-01-19 14:39:40.000000000","message":"correct but it does not mean the vm wont span sockets if the vm has multiple numa nodes.","commit_id":"a475771be07c1e83e2435b0147a9eed879bcd95b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":63,"context_line":"and ``hw_pci_numa_affinity_policy\u003dsocket`` can be pinned to NUMA node N0 or N1,"},{"line_number":64,"context_line":"but not N2 or N3"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"::"},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"  +----------+         +----------+"},{"line_number":69,"context_line":"  | N0    N1 |         | N2    N3 |"}],"source_content_type":"text/x-rst","patch_set":6,"id":"9b86e15e_ce120117","line":66,"updated":"2021-01-19 15:09:40.000000000","message":"see https://review.opendev.org/c/openstack/nova-specs/+/765551/5/specs/wallaby/approved/pci-socket-policy.rst#82 this is incorrect","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a1c183263efcad05f47757f8f82f0fc408a0d088","unresolved":true,"context_lines":[{"line_number":63,"context_line":"and ``hw_pci_numa_affinity_policy\u003dsocket`` can be pinned to NUMA node N0 or N1,"},{"line_number":64,"context_line":"but not N2 or N3"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"::"},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"  +----------+         +----------+"},{"line_number":69,"context_line":"  | N0    N1 |         | N2    N3 |"}],"source_content_type":"text/x-rst","patch_set":6,"id":"2ebd28c8_f6d7b288","line":66,"in_reply_to":"9b86e15e_ce120117","updated":"2021-01-19 16:05:23.000000000","message":"We discussed this in IRC. While we can make ``socket`` mean whatever we like, it\u0027s a PCI policy, not a NUMA policy. If we want all guest NUMA nodes to be on the same host socket, we can add a different extra spec for that. Changed the spec to explain the revised behaviour.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":97,"context_line":"for crossing the NUMA node boundary, it is smaller than crossing the socket"},{"line_number":98,"context_line":"boundary. It is this latter performance penalty that this spec addresses."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"This means that setting ``hw_pci_numa_affinity_policy\u003dsocket`` on any of an"},{"line_number":101,"context_line":"instance\u0027s PCI devices effectively forces all the guest NUMA nodes (regardless"},{"line_number":102,"context_line":"of how many there are) to be pinned to the same host socket. Other PCI devices"},{"line_number":103,"context_line":"(if there are any) with different affinity policies are treated as they"}],"source_content_type":"text/x-rst","patch_set":6,"id":"0d499c50_5fe300e0","line":100,"updated":"2021-01-19 15:09:40.000000000","message":"this is wrong and not what we discussed previously.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a1c183263efcad05f47757f8f82f0fc408a0d088","unresolved":true,"context_lines":[{"line_number":97,"context_line":"for crossing the NUMA node boundary, it is smaller than crossing the socket"},{"line_number":98,"context_line":"boundary. It is this latter performance penalty that this spec addresses."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"This means that setting ``hw_pci_numa_affinity_policy\u003dsocket`` on any of an"},{"line_number":101,"context_line":"instance\u0027s PCI devices effectively forces all the guest NUMA nodes (regardless"},{"line_number":102,"context_line":"of how many there are) to be pinned to the same host socket. Other PCI devices"},{"line_number":103,"context_line":"(if there are any) with different affinity policies are treated as they"}],"source_content_type":"text/x-rst","patch_set":6,"id":"99be3fc6_f848d483","line":100,"in_reply_to":"0d499c50_5fe300e0","updated":"2021-01-19 16:05:23.000000000","message":"Yep, see above.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":107,"context_line":"devices. For CPUs, the libvirt driver obtains that information from libvirt\u0027s"},{"line_number":108,"context_line":"host capabilities and saves it in a new field in the ``NUMACell`` object. For"},{"line_number":109,"context_line":"PCI devices, the existing ``PCIDevice.numa_node`` field can be used to look up"},{"line_number":110,"context_line":"the corresponding ``NUMACell`` object and obtain its socket affinity."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"The socket affinity information is then used in"},{"line_number":113,"context_line":"hardware.py\u0027s ``numa_fit_instance_to_host()``, specifically when it calls down"}],"source_content_type":"text/x-rst","patch_set":6,"id":"0fd510b8_4b00ef4b","line":110,"updated":"2021-01-19 15:09:40.000000000","message":"ya this works.\n\nas i noted you could also stach it in the extra_info blob where we store network_capabliies today\nyou can just add a socket_id filed to that json blob if you want without db changes.\nboth work in the extra info case you would just heal it on device load by doing the lookup via the numa node so you dont actully need a dedicated online data mighration nessisarlly.\n\n\nyou should also keep in mind that the lookup will have to be dont in both the scudler and comptue node.\nso you need to ensure that we only use the info avaiable in the db in that code via the host numa toplogy blob.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":123,"context_line":"At the implementation level, an alternative to looking up the PCI device socket"},{"line_number":124,"context_line":"affinity every time is to save it in a new field in the ``PCIDevice`` object."},{"line_number":125,"context_line":"This is ruled out because it adds a database migration, and is less flexible"},{"line_number":126,"context_line":"and future-proof."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Data model impact"},{"line_number":129,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"ab4985a0_4fc84a9f","line":126,"updated":"2021-01-19 15:09:40.000000000","message":"see my alternitve to the db migration above.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a1c183263efcad05f47757f8f82f0fc408a0d088","unresolved":true,"context_lines":[{"line_number":123,"context_line":"At the implementation level, an alternative to looking up the PCI device socket"},{"line_number":124,"context_line":"affinity every time is to save it in a new field in the ``PCIDevice`` object."},{"line_number":125,"context_line":"This is ruled out because it adds a database migration, and is less flexible"},{"line_number":126,"context_line":"and future-proof."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Data model impact"},{"line_number":129,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"19cd4626_11153490","line":126,"in_reply_to":"ab4985a0_4fc84a9f","updated":"2021-01-19 16:05:23.000000000","message":"extra_info is a bit of a cop-out though :) We specifically added objects because we didn\u0027t want unversioned dicts flying around over the wire, and now we\u0027re stashing said dicts inside the objects. I understand if there\u0027s info that\u0027s only applicable to certain devices (in this specific case), but socket affinity goes for all devices (though it can remain None/unreported)","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":130,"context_line":""},{"line_number":131,"context_line":"* A ``socket`` integer field is added to the ``NUMACell`` object. No database"},{"line_number":132,"context_line":"  changes are necessary here, as the object is stored as a JSON blob. The field"},{"line_number":133,"context_line":"  is populated at runtime by the libvirt driver."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"REST API impact"},{"line_number":136,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"ee488d41_9393f658","line":133,"updated":"2021-01-19 15:09:40.000000000","message":"there will be an object change however so the version sent over rpc calls will be different and the ovo version number will be bumpted.\n\nthese cahgne are being made to the host numa cell object not the instance numa cell object\nas a result there will be no change to version notification sicne the host cell object should not be present in those.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":140,"context_line":"``hw:pci_numa_affinity_policy`` flavor extra spec and the"},{"line_number":141,"context_line":"``hw_pci_numa_affinity_policy`` image property. The flavor extra spec"},{"line_number":142,"context_line":"validation logic is extended to support the new value."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Security impact"},{"line_number":145,"context_line":"---------------"},{"line_number":146,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"73dd3664_4414697a","line":143,"updated":"2021-01-19 15:09:40.000000000","message":"just a related note the change being discusssed here for the numa cell object are the host numa cell object\nso there is no interaction with https://specs.openstack.org/openstack/nova-specs/specs/train/implemented/show-server-numa-topology.html \n\nthat api only reflects the guest numa toplogy info and host mapping if you are an admin\n\nit willl not be altered as part of this spec.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a1c183263efcad05f47757f8f82f0fc408a0d088","unresolved":true,"context_lines":[{"line_number":140,"context_line":"``hw:pci_numa_affinity_policy`` flavor extra spec and the"},{"line_number":141,"context_line":"``hw_pci_numa_affinity_policy`` image property. The flavor extra spec"},{"line_number":142,"context_line":"validation logic is extended to support the new value."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Security impact"},{"line_number":145,"context_line":"---------------"},{"line_number":146,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf9f069c_82942ebc","line":143,"in_reply_to":"73dd3664_4414697a","updated":"2021-01-19 16:05:23.000000000","message":"Yeah, and really if we\u0027re taking objects that we know can can evolve and blindly dumping them to the API it\u0027s an implementation mistake - we should be picking which specific fields we\u0027re showing.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":179,"context_line":""},{"line_number":180,"context_line":"The current (pre-Wallaby) implementation of ``_filter_pools_for_numa_cells()``"},{"line_number":181,"context_line":"recognizes ``required``, ``legacy`` and ``preferred`` as values for"},{"line_number":182,"context_line":"``hw_pci_numa_affinity_policy``, with the latter being the catch-all default."},{"line_number":183,"context_line":"Therefore, instances with ``hw_pci_numa_affinity_policy\u003dsocket`` cannot be"},{"line_number":184,"context_line":"permitted to land on pre-Wallaby compute hosts: the ``socket`` value would not"},{"line_number":185,"context_line":"be recognized, and they would be incorrectly treated as having the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"a1e9da8a_5cb124cf","line":182,"range":{"start_line":182,"start_character":33,"end_line":182,"end_character":76},"updated":"2021-01-19 15:09:40.000000000","message":"incorrect the default is legacy if not set.\n\nnit technically if its alias bassed passthough then those pci devices defult to legacy also but it can be overrinde in the alias definition in the nova config.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a1c183263efcad05f47757f8f82f0fc408a0d088","unresolved":true,"context_lines":[{"line_number":179,"context_line":""},{"line_number":180,"context_line":"The current (pre-Wallaby) implementation of ``_filter_pools_for_numa_cells()``"},{"line_number":181,"context_line":"recognizes ``required``, ``legacy`` and ``preferred`` as values for"},{"line_number":182,"context_line":"``hw_pci_numa_affinity_policy``, with the latter being the catch-all default."},{"line_number":183,"context_line":"Therefore, instances with ``hw_pci_numa_affinity_policy\u003dsocket`` cannot be"},{"line_number":184,"context_line":"permitted to land on pre-Wallaby compute hosts: the ``socket`` value would not"},{"line_number":185,"context_line":"be recognized, and they would be incorrectly treated as having the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5bce636b_f07026d8","line":182,"range":{"start_line":182,"start_character":33,"end_line":182,"end_character":76},"in_reply_to":"a1e9da8a_5cb124cf","updated":"2021-01-19 16:05:23.000000000","message":"Doh, yeah, we set a default here [1] - I was confused because we end on preferred here [2], but only if we failed the previous legacy check here [3]\n\n[1] https://opendev.org/openstack/nova/src/branch/master/nova/pci/stats.py#L282\n[2] https://opendev.org/openstack/nova/src/branch/master/nova/pci/stats.py#L321\n[3] https://opendev.org/openstack/nova/src/branch/master/nova/pci/stats.py#L317","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":183,"context_line":"Therefore, instances with ``hw_pci_numa_affinity_policy\u003dsocket`` cannot be"},{"line_number":184,"context_line":"permitted to land on pre-Wallaby compute hosts: the ``socket`` value would not"},{"line_number":185,"context_line":"be recognized, and they would be incorrectly treated as having the"},{"line_number":186,"context_line":"``preferred`` value."},{"line_number":187,"context_line":""},{"line_number":188,"context_line":"To ensure that only Wallaby compute hosts receive instances with"},{"line_number":189,"context_line":"``hw_pci_numa_affinity_policy\u003dsocket``, the scheduler short-circuits scheduling"}],"source_content_type":"text/x-rst","patch_set":6,"id":"d9d4471b_6359c897","line":186,"range":{"start_line":186,"start_character":2,"end_line":186,"end_character":11},"updated":"2021-01-19 15:09:40.000000000","message":"nope it would be treated as legacy.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a1c183263efcad05f47757f8f82f0fc408a0d088","unresolved":false,"context_lines":[{"line_number":183,"context_line":"Therefore, instances with ``hw_pci_numa_affinity_policy\u003dsocket`` cannot be"},{"line_number":184,"context_line":"permitted to land on pre-Wallaby compute hosts: the ``socket`` value would not"},{"line_number":185,"context_line":"be recognized, and they would be incorrectly treated as having the"},{"line_number":186,"context_line":"``preferred`` value."},{"line_number":187,"context_line":""},{"line_number":188,"context_line":"To ensure that only Wallaby compute hosts receive instances with"},{"line_number":189,"context_line":"``hw_pci_numa_affinity_policy\u003dsocket``, the scheduler short-circuits scheduling"}],"source_content_type":"text/x-rst","patch_set":6,"id":"4b410d98_31332e5c","line":186,"range":{"start_line":186,"start_character":2,"end_line":186,"end_character":11},"in_reply_to":"d9d4471b_6359c897","updated":"2021-01-19 16:05:23.000000000","message":"Done","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"79ee6d63a6a3fdc4db22168adf5bf9a943c26add","unresolved":true,"context_lines":[{"line_number":188,"context_line":"To ensure that only Wallaby compute hosts receive instances with"},{"line_number":189,"context_line":"``hw_pci_numa_affinity_policy\u003dsocket``, the scheduler short-circuits scheduling"},{"line_number":190,"context_line":"for hosts that do not report a ``socket`` field in their ``NUMACell`` objects."},{"line_number":191,"context_line":"This is simpler than a new trait to filter our pre-Wallaby compute hosts."},{"line_number":192,"context_line":""},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":6,"id":"7d19bc5a_1305ec52","line":191,"updated":"2021-01-19 15:09:40.000000000","message":"it is although a trait would be nice.\nit would preform better too but im not going to push to say we must have a new trait.\n\nif we did it would proably be COMPUTE_NUMA_SOCKET_AFFINITY sicne the ablity to do affintiy is an atribute of the virtdriver not the hardware.\nwhich you could define here https://github.com/openstack/os-traits/blob/master/os_traits/compute/__init__.py","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"a1c183263efcad05f47757f8f82f0fc408a0d088","unresolved":true,"context_lines":[{"line_number":188,"context_line":"To ensure that only Wallaby compute hosts receive instances with"},{"line_number":189,"context_line":"``hw_pci_numa_affinity_policy\u003dsocket``, the scheduler short-circuits scheduling"},{"line_number":190,"context_line":"for hosts that do not report a ``socket`` field in their ``NUMACell`` objects."},{"line_number":191,"context_line":"This is simpler than a new trait to filter our pre-Wallaby compute hosts."},{"line_number":192,"context_line":""},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":6,"id":"814c810a_492bbeab","line":191,"in_reply_to":"7d19bc5a_1305ec52","updated":"2021-01-19 16:05:23.000000000","message":"Yeah, trait would be faster, since it eliminates old and/or non-libvirt computes from the placement results right away. I\u0027ll add it.","commit_id":"d973d5fd8544ebef8a43917503216171746aceba"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"c59dc0fdaea202277119db9931785e5e275a0665","unresolved":true,"context_lines":[{"line_number":26,"context_line":"Consider a guest with 16 CPUs and a PCI device, and a `require` PCI"},{"line_number":27,"context_line":"NUMA affinity policy. Such a policy requires the guest to \"fit\" entirely into"},{"line_number":28,"context_line":"the host NUMA node to which the PCI device is affined. Until recently, this was"},{"line_number":29,"context_line":"a reasonnable expectation: more than 16 CPUs per NUMA node was the norm, even"},{"line_number":30,"context_line":"in hosts with multiple NUMA nodes per socket."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"With more recent hardware like AMD\u0027s Zen2 architecture, this is no longer the"}],"source_content_type":"text/x-rst","patch_set":7,"id":"77232f40_8937a4dd","line":29,"range":{"start_line":29,"start_character":2,"end_line":29,"end_character":13},"updated":"2021-01-20 11:29:12.000000000","message":"reasonable","commit_id":"da78f4f4edd289d10b3a6011b13e275ae5ad7f0b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"16a14ecbdab9474901b29f592132c513111af42d","unresolved":true,"context_lines":[{"line_number":57,"context_line":"is the same as ``require``, except the PCI device must belong to the same"},{"line_number":58,"context_line":"socket, rather than the same host NUMA node. In the case of multiple NUMA"},{"line_number":59,"context_line":"nodes, the PCI device must belong to the same socket as *one* of the NUMA"},{"line_number":60,"context_line":"nodes."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"To better understand the new policy, consider some examples."},{"line_number":63,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"720ea1fc_956bb755","line":60,"updated":"2021-01-20 10:29:54.000000000","message":"yeah, I guess this is what I was after","commit_id":"da78f4f4edd289d10b3a6011b13e275ae5ad7f0b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"16a14ecbdab9474901b29f592132c513111af42d","unresolved":true,"context_lines":[{"line_number":113,"context_line":"the original purposes of Nova objects was to avoid unversioned dicts flying"},{"line_number":114,"context_line":"over the wire. Relying on JSON blobs inside objects goes against this. In"},{"line_number":115,"context_line":"addition, socket affinity is applicable to all PCI devices, and so does not"},{"line_number":116,"context_line":"belong in a device-specific ``extra_info`` dict."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Data model impact"},{"line_number":119,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"58b1e084_d9c11815","line":116,"updated":"2021-01-20 10:29:54.000000000","message":"+1 for not using random dicts if possible.","commit_id":"da78f4f4edd289d10b3a6011b13e275ae5ad7f0b"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"c59dc0fdaea202277119db9931785e5e275a0665","unresolved":true,"context_lines":[{"line_number":204,"context_line":"* Modify ``PciDeviceStats._filter_pools()``, as called by"},{"line_number":205,"context_line":"  ``PciDeviceStats.support_requests()``, to support the new ``socket``"},{"line_number":206,"context_line":"  NUMA affinity policy."},{"line_number":207,"context_line":"* Add COMPUTE_SOCKET_NUMA_AFFINITY trait (name can be adjusted during"},{"line_number":208,"context_line":"  implementation) and corresponding pre-filter."},{"line_number":209,"context_line":"* Extend the flavor extra spec validation to allow the new ``socket`` value."},{"line_number":210,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a3a6bb4_bf26cc41","line":207,"range":{"start_line":207,"start_character":6,"end_line":207,"end_character":34},"updated":"2021-01-20 11:29:12.000000000","message":"``COMPUTE_SOCKET_NUMA_AFFINITY``","commit_id":"da78f4f4edd289d10b3a6011b13e275ae5ad7f0b"}]}