)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d98976d2220328bb295ad49a27d39862ffc6481b","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Jing Zhang \u003cjing.c.zhang@nokia.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2019-10-01 21:36:51 -0400"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Force small pages accounting for CPU pinned VMs so that memory can be reserved"},{"line_number":8,"context_line":"on NUMA 0 for host processes."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_f0c9255a","line":8,"range":{"start_line":7,"start_character":0,"end_line":8,"end_character":29},"updated":"2019-10-02 07:05:46.000000000","message":"commit titile are ment to be 1 line and no more the 72 coloums so i would re word this to\nDefault mem_page_size to small for pinned instnaces\"\n\nalso numa node 0 is not really relevent here.\n\nwhat you are want in the ability to reserve mempages of a specific size on a specific host numa node.\n\nyou could elect to run your host processes on numa node 1.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Jing Zhang \u003cjing.c.zhang@nokia.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2019-10-01 21:36:51 -0400"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Force small pages accounting for CPU pinned VMs so that memory can be reserved"},{"line_number":8,"context_line":"on NUMA 0 for host processes."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_2211a554","line":8,"range":{"start_line":7,"start_character":0,"end_line":8,"end_character":29},"in_reply_to":"3fa7e38b_f0c9255a","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d98976d2220328bb295ad49a27d39862ffc6481b","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Force small pages accounting for CPU pinned VMs so that memory can be reserved"},{"line_number":8,"context_line":"on NUMA 0 for host processes."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_70b695df","line":10,"range":{"start_line":10,"start_character":0,"end_line":10,"end_character":67},"updated":"2019-10-02 07:05:46.000000000","message":"that is missleading and incorrect.\n\nas you point out belove you can use\nhw:mem_page_size\u003dsmall, plus reserved_huge_pages\n\nreserved_huge_pages \u003d node:0,size:4k,count:262144\n\nto reserve 10GB of 4k pages on host numa node 0\n\nand you can set hw:mem_page_size\u003dsmall as an operator in the flavor or hw_mem_page_size\u003dsmall as a tenant in the image.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Force small pages accounting for CPU pinned VMs so that memory can be reserved"},{"line_number":8,"context_line":"on NUMA 0 for host processes."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_8273b9b1","line":10,"range":{"start_line":10,"start_character":0,"end_line":10,"end_character":67},"in_reply_to":"3fa7e38b_70b695df","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d98976d2220328bb295ad49a27d39862ffc6481b","unresolved":false,"context_lines":[{"line_number":9,"context_line":""},{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_50b599cc","line":12,"updated":"2019-10-02 07:05:46.000000000","message":"nit remove the \u003e and no the ram allocation ration is not ignore for pinned instnaces. \nit is only ignored if you set a mem_page_size.\nthat was a regression https://bugs.launchpad.net/nova/+bug/1810977\nintoduced by https://review.opendev.org/#/c/532168/\nand fixed by https://review.opendev.org/#/c/629281/\n\nso by defualt instance with a numa toplogy respect the ram_allocation_ratio unless mem_page_size is set.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":9,"context_line":""},{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_f479f992","line":12,"in_reply_to":"3fa7e38b_50b599cc","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"5bba948ca2b4e4a008e1f7ec58131a24de04b7f2","unresolved":false,"context_lines":[{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_7607631a","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":1},"updated":"2019-10-02 18:07:43.000000000","message":"can\u0027t you use reserved_huge_pages with size 4k and a specified node as per sean\u0027s comment above?","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":10,"context_line":"When VM CPU is pinned, there is no way to reserve memory on NUMA 0:"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_d4995dd0","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":1},"in_reply_to":"3fa7e38b_7607631a","updated":"2019-10-04 00:29:15.000000000","message":"yes, we used it as a walkaround but it affects VMs.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"5bba948ca2b4e4a008e1f7ec58131a24de04b7f2","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_16b4ef34","line":16,"range":{"start_line":15,"start_character":0,"end_line":16,"end_character":11},"updated":"2019-10-02 18:07:43.000000000","message":"There\u0027s also a related problem where cpu pinning is not involved at all:  https://bugs.launchpad.net/nova/+bug/1792985","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d98976d2220328bb295ad49a27d39862ffc6481b","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_b0464dc0","line":16,"range":{"start_line":15,"start_character":1,"end_line":16,"end_character":11},"updated":"2019-10-02 07:05:46.000000000","message":"this has nothing to to do with ram allocation ratio\n\nthe mempage aware code is only activated by seting mem_page_size so while its true that vms without mempage size only consider the global free memory on the host as modifed by the ram_allocation_raitio and and reserved_host_memory,\nit is not true to imply that is becasue of cpu pinning it is becasue you did not enable the feature.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_74cc2961","line":16,"range":{"start_line":15,"start_character":0,"end_line":16,"end_character":11},"in_reply_to":"3fa7e38b_16b4ef34","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae6e54bafa6b111bdc126d33d38ff4787d172d06","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_8281794d","line":16,"range":{"start_line":15,"start_character":0,"end_line":16,"end_character":11},"in_reply_to":"3fa7e38b_16b4ef34","updated":"2019-10-02 20:50:07.000000000","message":"yes that happens either if you mix implcit 4k (non mem_page_size set) with explcit 4k/small pageages on the same host. which is not supported on the same host today.\nor if you set hw:numa_nodes\u003dx without setting hw:mem_page_size.\n\nin the hw:numa_nodes case the issue is the same as for pinning we schedule based on the gloabl memory and restict you to a give numa node. that said since the memory is swappable if you follow the best pratice and allocate total ram * ram_allocation_ratio as swap space then you will have enough space to swap to disk instead of triggering oom events.\n\nbut you already know that. im just expanind on why for those who have not been dealing with this since 2014. \n\nout of interst i know that in titainm cloud windriver had a host agent that repinned floating instnace so they would not float over pinned guests but would float over the non pinned cores. did ye ever try to fix this in titainium cloud or starlingx?","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"b4e2f9ad7b0e5cd5858db1f9fe02626cb69df3b4","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_485b6a86","line":16,"range":{"start_line":15,"start_character":0,"end_line":16,"end_character":11},"in_reply_to":"3fa7e38b_8281794d","updated":"2019-10-07 15:42:28.000000000","message":"\u003e out of interst i know that in titainm cloud windriver had a host\n \u003e agent that repinned floating instnace so they would not float over\n \u003e pinned guests but would float over the non pinned cores. did ye\n \u003e ever try to fix this in titainium cloud or starlingx?\n\nIn Titanium Cloud we tried a couple different things.  Initially we restricted floating instances to a single NUMA node, but eventually we settled on the suggestion I made below to set the NUMA affinity to \"preferred\" instead of \"strict\" for instances with 4K pages only.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":12,"context_line":"\u003e ram_allocation_ratio is ignored by the nova scheduler when VM CPU is pinned"},{"line_number":13,"context_line":"\u003e reserved_host_memory_mb is a global reservation"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This leads to many VMs are scheduled on NUMA 0 while their memory needs are met"},{"line_number":16,"context_line":"\"globally\"."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_54cfad56","line":16,"range":{"start_line":15,"start_character":1,"end_line":16,"end_character":11},"in_reply_to":"3fa7e38b_b0464dc0","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"5bba948ca2b4e4a008e1f7ec58131a24de04b7f2","unresolved":false,"context_lines":[{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"},{"line_number":20,"context_line":"to kill host processes."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"It is agreed that the workaround is to use hw:mem_page_size\u003dsmall, plus"},{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_569347a4","line":23,"range":{"start_line":22,"start_character":0,"end_line":23,"end_character":79},"updated":"2019-10-02 18:07:43.000000000","message":"agreed by who?","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":19,"context_line":"on NUMA 0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"},{"line_number":20,"context_line":"to kill host processes."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"It is agreed that the workaround is to use hw:mem_page_size\u003dsmall, plus"},{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_9433e562","line":23,"range":{"start_line":22,"start_character":0,"end_line":23,"end_character":79},"in_reply_to":"3fa7e38b_569347a4","updated":"2019-10-04 00:29:15.000000000","message":"Agreed from 1844721.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d98976d2220328bb295ad49a27d39862ffc6481b","unresolved":false,"context_lines":[{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_90c1d1e7","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":19},"updated":"2019-10-02 07:05:46.000000000","message":"this should be related bug.\nthe patch you propose is already possible via flavor and config settings today and it does not actully adress the proablem of mixing instnace with numa affined memroy and instnace without numa affied mermroy.\n\nin https://bugs.launchpad.net/nova/+bug/1844721/comments/3\nyou stated that https://bugs.launchpad.net/nova/+bug/1439247\nwas obsessed with numa and non numa isntaces and that that was not the root cause.\n\nwith all due respect your understand of this is incorrect\nhttps://bugs.launchpad.net/nova/+bug/1439247 deals with the more genral porblem which https://bugs.launchpad.net/nova/+bug/1844721 is a special case of.\nso while this patch can be considerd a partal fix of \nhttps://bugs.launchpad.net/nova/+bug/1844721\nit does not close\n\nthis will only fix the special case where all instances on a give host have hw:mem_page_size or hw_mem_page_size defiend.\n\nnot that in train we added the ability to have two cpu pools\nthe cpu_dedicated_set and cpu_share_set. this allows pinned and unpinned guest to share the same host safely but this only works correctly if the mem_page_size is set on all vms on a multi numa host or if its a singel numa host.\n\nthe reason that this has not been closed for 4 years is the only way to fix the genreal proablem is to always do muma aware tracking and asitnment. which is a proablme that we have to adress if we ever want to track mempages in placment\n\nthat problem has 4 posible solutions\n1 require all vms to have a numa toplogy\n2 remove the implict numa toplogy form pinned instances.\n3 staticaly pin the memory of floating instances but do not create a virtual numa toplogy. this could be restated as saying that hw:mem_page_size should not create an\n4 is a combination of 2 and 3 which basicaly means we nolonger create an implict numa toplogy ever meaning if you want a virtual numa toplogy you must use hw:numa_nodes or hw_numa_nodes to define one.\n\n\nif we done do one of those 4 options we will not be able to model mempages in placment and that is why this has not made progress in the last few years.\n\ni have proposed all 4 option in the past and they have been blocked due to backwards comaptaiblety reasons.\n\nim hoping just as modleing PCPU in plamcent was a forcing funtion to break backwards compatibaliy, im hoping the desire to model mempages in placement will be a forcing funtion to finally pick one of those 4 options.\n\ni would prefer options 4 or 1 as they are the simpelet to teach others they all have backwards incompatiable behavior.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_409a88c5","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":19},"in_reply_to":"3fa7e38b_223f65b6","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"b4e2f9ad7b0e5cd5858db1f9fe02626cb69df3b4","unresolved":false,"context_lines":[{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_a8977eeb","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":19},"in_reply_to":"3fa7e38b_223f65b6","updated":"2019-10-07 15:42:28.000000000","message":"Modelling 4K pages in placement isn\u0027t sufficient if we allow instances with 4K pages that are restricted to a NUMA node on the same host as instances with 4K pages that float across the whole host.  (Because then we don\u0027t know which NUMA node the floating instance allocated memory from.)\n\nIn order to have accurate modelling in placement either we treat 4K pages as a whole-host resource rather than a per-NUMA-node one, or else we restrict instances with 4K pages to a single NUMA node.\n\nThe former option has the advantage of being backwards-compatible.  The latter option might offer more performance, but will almost certainly cause problems for someone with existing floating instances that don\u0027t fit on the available resources of a single NUMA node.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae6e54bafa6b111bdc126d33d38ff4787d172d06","unresolved":false,"context_lines":[{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_223f65b6","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":19},"in_reply_to":"3fa7e38b_561d870d","updated":"2019-10-02 20:50:07.000000000","message":"ya that is kind of like option 3 e.g. no implict numa affinity/toplogy for mempages although more target to just 4k\nso in my mind i was kind of grouping them.\n\nthere is the expclit option of actully implematning\nhw:numa_mem_policy\u003dstrict|prefered that was part of the hyperv proposal.\n\nnot that we really need another configuration option for this stuff.\n\nthe problem isn\u0027t really that we dont know how to fix this issue, we know at least 5 way to do it we just dont know what is the correct path forward. i am hoping that we can use the lens of modeling mempages in placment to focus on what element are importent to solving this problem with the view to moveing to placment.\n\nfor your propsoal the placment implciation of that is we only model non 4k mapges as now mempage_$SIZE resource types and continue to ues the MEMORY_MB resocue class inventoy on the compute node RP for 4k memroy. the down side to this would be that also would require that hw:numa_nodes\u003d1 also does not guarentee numa affinity of memroy and cpus if you have 4k pages.\n\nso this is why i want to adress modeling mempages in placmenet as a spec for ussuri like we did with PCPU to cover all the sidefffect of each choice but i dont think we have to wait to create that spec to adress the specal case of this bug.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_3d3695b6","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":19},"in_reply_to":"3fa7e38b_90c1d1e7","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"5bba948ca2b4e4a008e1f7ec58131a24de04b7f2","unresolved":false,"context_lines":[{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_561d870d","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":19},"in_reply_to":"3fa7e38b_90c1d1e7","updated":"2019-10-02 18:07:43.000000000","message":"I think you missed a possible temporary workaround.  For 4k pages change the NUMA affinity from \"strict\" to \"best-effort\".  This would have performance implications, but anyone caring about performance should already be using hugepages.\n\nIt should get rid of the OOM killer problem though.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"a8a14ac242f6616620b8dfcfd268c2f6dddfa488","unresolved":false,"context_lines":[{"line_number":23,"context_line":"reserved_huge_pages set to proper number of small pages on NUMA 0 in nova.conf."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_14a907dc","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":19},"in_reply_to":"3fa7e38b_a8977eeb","updated":"2019-10-07 18:24:58.000000000","message":"yep that is the crux of the debate we have been having for the last few years. \n\ni think we should finally come to a decision on the above and either disable numa affintiy if using 4k pages either explcitly or implcitly or make all instance have a numa toplogy of 1 unless you say otherwise.\n\nform a perfomace stand point i think the numa route is obviously better but i understand the other usecase too.\nthis is not the place to fially decide this but until we do we cannot model numa in placement. at lest not properly since we cant model memory which is kind of important when the topic is non umiform _memory_ architechture modeling.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"5bba948ca2b4e4a008e1f7ec58131a24de04b7f2","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_96445f1d","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":19},"updated":"2019-10-02 18:07:43.000000000","message":"This bug is marked as a dupe of the other bug you list.  I don\u0027t think we normally list dupes here.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d98976d2220328bb295ad49a27d39862ffc6481b","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_10ade130","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":19},"updated":"2019-10-02 07:05:46.000000000","message":"this should be partial-bug the patch you have submitted will solve your special case but its also not required as you can create a flaovr and set the config options today. this would certenly make it less likely that operator will not correctly configure it. so im actully ok with this but it needs a release not as the side effect of this cahnge is that you will not be able to have memroy oversubsciption of pinned instances which is a usecase that people have.\n\nthat said it much more comment for people to use hugepages without cpu pinning then it is to use cpu pinning without hugepages. pepole then to be cpu constrated mroe then memory but its still an upgrade impact.\n\nas this does not adress exsiting instance on a host it should not be closes-bug.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_fd58fdfa","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":19},"in_reply_to":"3fa7e38b_10ade130","updated":"2019-10-04 00:29:15.000000000","message":"This patch is to deal with sites that already have VNFs deployed. By changing and restarting the nova, we can avoid taking down the VMs, i.e. \"in-service\" patching.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"79159f898490ccdef016a676d36ddd5ab1de4b78","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":26,"context_line":"Closes-Bug: 1439247"},{"line_number":27,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_546b2d1e","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":19},"in_reply_to":"3fa7e38b_96445f1d","updated":"2019-10-04 00:29:15.000000000","message":"Done","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"f34a6073d7c5dc4e7c6c602d443bd141bc0dbc07","unresolved":false,"context_lines":[{"line_number":7,"context_line":"[WIP] Default mem_page_size to small for pinned instances"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If VM CPU is pinned but mem_page_size is not set, many VMs can be scheduled on"},{"line_number":10,"context_line":"NUMA 0 while their memory needs are met \"globally\" on the host as modified by"},{"line_number":11,"context_line":"ram_allocation_ratio and reserved_host_memory_mb."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_8f198189","line":10,"range":{"start_line":10,"start_character":0,"end_line":10,"end_character":6},"updated":"2019-10-04 17:14:19.000000000","message":"nit: as i said previously this has noting to do with numa0 \n\nwe select core form numa node 0 first becasue fo how we use itertools.permutations() to generate the set of possible core pinnings but this can happen happen to vms on other numa nodes too.\n\ni woudl change this as follows\n\u0027many VMs can be scheduled on\nNUMA 0 while their memory needs are met \"globally\"\u0027\n-\u003e \n\u0027many VMs can be scheduled to a NUMA node while their memory needs are checked \"globally\"\u0027","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"34460f502ea6363e7866970e39639f703afdaa1c","unresolved":false,"context_lines":[{"line_number":7,"context_line":"[WIP] Default mem_page_size to small for pinned instances"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If VM CPU is pinned but mem_page_size is not set, many VMs can be scheduled on"},{"line_number":10,"context_line":"NUMA 0 while their memory needs are met \"globally\" on the host as modified by"},{"line_number":11,"context_line":"ram_allocation_ratio and reserved_host_memory_mb."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_9901b207","line":10,"range":{"start_line":10,"start_character":0,"end_line":10,"end_character":6},"in_reply_to":"3fa7e38b_8f198189","updated":"2019-10-08 02:18:09.000000000","message":"Done","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"f34a6073d7c5dc4e7c6c602d443bd141bc0dbc07","unresolved":false,"context_lines":[{"line_number":10,"context_line":"NUMA 0 while their memory needs are met \"globally\" on the host as modified by"},{"line_number":11,"context_line":"ram_allocation_ratio and reserved_host_memory_mb."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":14,"context_line":"on NUMA0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"},{"line_number":15,"context_line":"to kill host processes."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"It is agreed that the workaround is to use hw:mem_page_size\u003dsmall, and setting"},{"line_number":18,"context_line":"reserved_huge_pages for proper number of small pages on NUMA 0 in nova.conf."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_5221ec9e","line":15,"range":{"start_line":13,"start_character":0,"end_line":15,"end_character":23},"updated":"2019-10-04 17:14:19.000000000","message":"When the system starts to take load, the allocated memroy for \nthe affined vms can exceed the aviable memroy on a numa node.\nThe OOM reaper runs per numa node so when the numa local memory is exausted it can lead to the kernel killing host processes.","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"34460f502ea6363e7866970e39639f703afdaa1c","unresolved":false,"context_lines":[{"line_number":10,"context_line":"NUMA 0 while their memory needs are met \"globally\" on the host as modified by"},{"line_number":11,"context_line":"ram_allocation_ratio and reserved_host_memory_mb."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When the system starts to take load, VMs\u0027 memory pages start to get allocated"},{"line_number":14,"context_line":"on NUMA0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"},{"line_number":15,"context_line":"to kill host processes."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"It is agreed that the workaround is to use hw:mem_page_size\u003dsmall, and setting"},{"line_number":18,"context_line":"reserved_huge_pages for proper number of small pages on NUMA 0 in nova.conf."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_3908be26","line":15,"range":{"start_line":13,"start_character":0,"end_line":15,"end_character":23},"in_reply_to":"3fa7e38b_5221ec9e","updated":"2019-10-08 02:18:09.000000000","message":"Done","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"f34a6073d7c5dc4e7c6c602d443bd141bc0dbc07","unresolved":false,"context_lines":[{"line_number":14,"context_line":"on NUMA0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"},{"line_number":15,"context_line":"to kill host processes."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"It is agreed that the workaround is to use hw:mem_page_size\u003dsmall, and setting"},{"line_number":18,"context_line":"reserved_huge_pages for proper number of small pages on NUMA 0 in nova.conf."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"This path is to default mem_page_size to small for pinned instances when it is"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_3236b0e9","line":17,"range":{"start_line":17,"start_character":22,"end_line":17,"end_character":32},"updated":"2019-10-04 17:14:19.000000000","message":"nit: its not a workaround this is how it was desgined to work.","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"34460f502ea6363e7866970e39639f703afdaa1c","unresolved":false,"context_lines":[{"line_number":14,"context_line":"on NUMA0 to the extend that memory shortage occurs on NUMA 0 and OOM kicks in"},{"line_number":15,"context_line":"to kill host processes."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"It is agreed that the workaround is to use hw:mem_page_size\u003dsmall, and setting"},{"line_number":18,"context_line":"reserved_huge_pages for proper number of small pages on NUMA 0 in nova.conf."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"This path is to default mem_page_size to small for pinned instances when it is"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_590b3a23","line":17,"range":{"start_line":17,"start_character":22,"end_line":17,"end_character":32},"in_reply_to":"3fa7e38b_3236b0e9","updated":"2019-10-08 02:18:09.000000000","message":"Done","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"f34a6073d7c5dc4e7c6c602d443bd141bc0dbc07","unresolved":false,"context_lines":[{"line_number":22,"context_line":"customer sites."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":25,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_924c4477","line":25,"range":{"start_line":25,"start_character":0,"end_line":25,"end_character":10},"updated":"2019-10-04 17:14:19.000000000","message":"Partial-Bug","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"34460f502ea6363e7866970e39639f703afdaa1c","unresolved":false,"context_lines":[{"line_number":22,"context_line":"customer sites."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Change-Id: I04bed3055de90b2372c2cb67f73090ad5568a52a"},{"line_number":25,"context_line":"Closes-Bug: 1844721"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3fa7e38b_f911c654","line":25,"range":{"start_line":25,"start_character":0,"end_line":25,"end_character":10},"in_reply_to":"3fa7e38b_924c4477","updated":"2019-10-08 02:18:09.000000000","message":"Done","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9eb53602dfc8bc54c7ceb5b418b6c9ddec3f5a92","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[WIP] Default mem_page_size to small for pinned instances"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If VM CPU is pinned but mem_page_size is not set, many VMs can be scheduled"},{"line_number":10,"context_line":"on a NUMA node while their memory needs are met \"globally\" on the host as"},{"line_number":11,"context_line":"modified by ram_allocation_ratio and reserved_host_memory_mb."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When the system starts to take load, the allocated memroy for the affined"},{"line_number":14,"context_line":"VMs can exceed the aviable memroy on a NUMA node. The OOM reaper runs per"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"3fa7e38b_15421769","line":11,"range":{"start_line":9,"start_character":0,"end_line":11,"end_character":61},"updated":"2019-10-11 10:54:18.000000000","message":"im not going to repete it here as i noted this in the release not but this is not actully true i have been digging into this code for a downstream issue yesterday and for instnace with a numa toplogy we  do not look at global memory.\n\nhowever the reserved_host_memory_mb is not considerd.\n\nso to reserve memory for the host you need to use the\nreserved_huge_pages paramater instead if the host will have guests with numa toplogies.","commit_id":"5ad4394726e0838852899209cfeb7a651e71a2a5"}],"nova/virt/hardware.py":[{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"5bba948ca2b4e4a008e1f7ec58131a24de04b7f2","unresolved":false,"context_lines":[{"line_number":1797,"context_line":"        for c in numa_topology.cells:"},{"line_number":1798,"context_line":"            setattr(c, \u0027pagesize\u0027, pagesize)"},{"line_number":1799,"context_line":""},{"line_number":1800,"context_line":"    cpu_policy \u003d get_cpu_policy_constraint(flavor, image_meta)"},{"line_number":1801,"context_line":"    cpu_thread_policy \u003d get_cpu_thread_policy_constraint(flavor, image_meta)"},{"line_number":1802,"context_line":"    rt_mask \u003d _get_realtime_constraint(flavor, image_meta)"},{"line_number":1803,"context_line":"    emu_threads_policy \u003d get_emulator_thread_policy_constraint(flavor)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_d6bf37ce","side":"PARENT","line":1800,"range":{"start_line":1800,"start_character":0,"end_line":1800,"end_character":62},"updated":"2019-10-02 18:07:43.000000000","message":"I think is is an error to remove this line, no?","commit_id":"bf37bec80baa527ac013dfaa7480ef2761ed2cb9"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae6e54bafa6b111bdc126d33d38ff4787d172d06","unresolved":false,"context_lines":[{"line_number":1797,"context_line":"        for c in numa_topology.cells:"},{"line_number":1798,"context_line":"            setattr(c, \u0027pagesize\u0027, pagesize)"},{"line_number":1799,"context_line":""},{"line_number":1800,"context_line":"    cpu_policy \u003d get_cpu_policy_constraint(flavor, image_meta)"},{"line_number":1801,"context_line":"    cpu_thread_policy \u003d get_cpu_thread_policy_constraint(flavor, image_meta)"},{"line_number":1802,"context_line":"    rt_mask \u003d _get_realtime_constraint(flavor, image_meta)"},{"line_number":1803,"context_line":"    emu_threads_policy \u003d get_emulator_thread_policy_constraint(flavor)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_8212d908","side":"PARENT","line":1800,"range":{"start_line":1800,"start_character":0,"end_line":1800,"end_character":62},"in_reply_to":"3fa7e38b_d6bf37ce","updated":"2019-10-02 20:50:07.000000000","message":"no its just move earlier in the fucntion with case the code to endter the if block but the value of cpu_policy will still be the same as if it was defined here.\n\nwith the excption that if you have requested pinned cpus it would not use explcit small pages.","commit_id":"bf37bec80baa527ac013dfaa7480ef2761ed2cb9"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"d98976d2220328bb295ad49a27d39862ffc6481b","unresolved":false,"context_lines":[{"line_number":1770,"context_line":"    pagesize \u003d _get_numa_pagesize_constraint(flavor, image_meta)"},{"line_number":1771,"context_line":"    vpmems \u003d get_vpmems(flavor)"},{"line_number":1772,"context_line":""},{"line_number":1773,"context_line":"    cpu_policy \u003d get_cpu_policy_constraint(flavor, image_meta)"},{"line_number":1774,"context_line":""},{"line_number":1775,"context_line":"    if pagesize \u003d\u003d None and cpu_policy \u003d\u003d fields.CPUAllocationPolicy.DEDICATED:"},{"line_number":1776,"context_line":"        pagesize \u003d MEMPAGES_SMALL"},{"line_number":1777,"context_line":""},{"line_number":1778,"context_line":"    if nodes or pagesize or vpmems:"},{"line_number":1779,"context_line":"        nodes \u003d nodes or 1"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_f02285ba","line":1776,"range":{"start_line":1773,"start_character":4,"end_line":1776,"end_character":33},"updated":"2019-10-02 07:05:46.000000000","message":"i actully think this is a good change. when we start tracking numa in plamcent i want to track mempages in placment and i think we should be tracking  floating and pinned instance as using small pages unless they request hugepages.\n\nwith that said this is missing several other changes.\n\nthis will not handel exsiting instance on a host. it will only set the page size for new instaces or when existing instace are migrated or unshelved. while we dont have existing mempages in placment yet and therefor dont have to reshape we do have to think about the instance numa toplogy blob and host numa toplogy blob.\n\nin theory we should do an online data mightration of some kind to to bring the host numa topology blob in sysnc with the actual state of the host and update the intane numa toplogy blob to set the pagesize.\n\nif we dont do that we would still get OOM events as the data in the host numa toplogy blob will not match the actual state on the host.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ae6e54bafa6b111bdc126d33d38ff4787d172d06","unresolved":false,"context_lines":[{"line_number":1770,"context_line":"    pagesize \u003d _get_numa_pagesize_constraint(flavor, image_meta)"},{"line_number":1771,"context_line":"    vpmems \u003d get_vpmems(flavor)"},{"line_number":1772,"context_line":""},{"line_number":1773,"context_line":"    cpu_policy \u003d get_cpu_policy_constraint(flavor, image_meta)"},{"line_number":1774,"context_line":""},{"line_number":1775,"context_line":"    if pagesize \u003d\u003d None and cpu_policy \u003d\u003d fields.CPUAllocationPolicy.DEDICATED:"},{"line_number":1776,"context_line":"        pagesize \u003d MEMPAGES_SMALL"},{"line_number":1777,"context_line":""},{"line_number":1778,"context_line":"    if nodes or pagesize or vpmems:"},{"line_number":1779,"context_line":"        nodes \u003d nodes or 1"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_87ae6b96","line":1776,"range":{"start_line":1773,"start_character":4,"end_line":1776,"end_character":33},"in_reply_to":"3fa7e38b_d692d74b","updated":"2019-10-02 20:50:07.000000000","message":"the \"and cpu_policy ...\" means this page size change will only effect pinned instance which always have a numa toplogy. so not we are not forceing a numa toplogy on floating instance because they will have cpu_policy\u003dshared or None","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"5bba948ca2b4e4a008e1f7ec58131a24de04b7f2","unresolved":false,"context_lines":[{"line_number":1770,"context_line":"    pagesize \u003d _get_numa_pagesize_constraint(flavor, image_meta)"},{"line_number":1771,"context_line":"    vpmems \u003d get_vpmems(flavor)"},{"line_number":1772,"context_line":""},{"line_number":1773,"context_line":"    cpu_policy \u003d get_cpu_policy_constraint(flavor, image_meta)"},{"line_number":1774,"context_line":""},{"line_number":1775,"context_line":"    if pagesize \u003d\u003d None and cpu_policy \u003d\u003d fields.CPUAllocationPolicy.DEDICATED:"},{"line_number":1776,"context_line":"        pagesize \u003d MEMPAGES_SMALL"},{"line_number":1777,"context_line":""},{"line_number":1778,"context_line":"    if nodes or pagesize or vpmems:"},{"line_number":1779,"context_line":"        nodes \u003d nodes or 1"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_d692d74b","line":1776,"range":{"start_line":1773,"start_character":4,"end_line":1776,"end_character":33},"in_reply_to":"3fa7e38b_f02285ba","updated":"2019-10-02 18:07:43.000000000","message":"What about floating instances that don\u0027t have a numa topology at all currently?  By specifying the pagesize here you will be forcing the instance to have a numa topology, which will limit it to a single NUMA node.\n\nThis will have implications about when hosts are \"full\".  You could have enough free memory across the host to fit an instance, but not enough on any given NUMA node...which would be a behaviour change from what happens today and should therefore be documented somewhere.","commit_id":"b840d9a5e20f7eda187be59abf35363fe77a397b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"f34a6073d7c5dc4e7c6c602d443bd141bc0dbc07","unresolved":false,"context_lines":[{"line_number":1766,"context_line":"    \"\"\""},{"line_number":1767,"context_line":"    numa_topology \u003d None"},{"line_number":1768,"context_line":""},{"line_number":1769,"context_line":"    # https://bugs.launchpad.net/nova/+bug/1844721"},{"line_number":1770,"context_line":"    # Default mem_page_size to small for pinned instances"},{"line_number":1771,"context_line":"    #"},{"line_number":1772,"context_line":"    nodes \u003d _get_numa_node_count_constraint(flavor, image_meta)"},{"line_number":1773,"context_line":"    pagesize \u003d _get_numa_pagesize_constraint(flavor, image_meta)"},{"line_number":1774,"context_line":"    vpmems \u003d get_vpmems(flavor)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_d2495c60","line":1771,"range":{"start_line":1769,"start_character":4,"end_line":1771,"end_character":5},"updated":"2019-10-04 17:14:19.000000000","message":"nit: this should really be in the if on line 1778","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":26124,"name":"Jing Zhang","email":"Jing.C.Zhang@Nokia.com","username":"Jing.C.Zhang"},"change_message_id":"34460f502ea6363e7866970e39639f703afdaa1c","unresolved":false,"context_lines":[{"line_number":1766,"context_line":"    \"\"\""},{"line_number":1767,"context_line":"    numa_topology \u003d None"},{"line_number":1768,"context_line":""},{"line_number":1769,"context_line":"    # https://bugs.launchpad.net/nova/+bug/1844721"},{"line_number":1770,"context_line":"    # Default mem_page_size to small for pinned instances"},{"line_number":1771,"context_line":"    #"},{"line_number":1772,"context_line":"    nodes \u003d _get_numa_node_count_constraint(flavor, image_meta)"},{"line_number":1773,"context_line":"    pagesize \u003d _get_numa_pagesize_constraint(flavor, image_meta)"},{"line_number":1774,"context_line":"    vpmems \u003d get_vpmems(flavor)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3fa7e38b_396f5ec6","line":1771,"range":{"start_line":1769,"start_character":4,"end_line":1771,"end_character":5},"in_reply_to":"3fa7e38b_d2495c60","updated":"2019-10-08 02:18:09.000000000","message":"Done","commit_id":"9cb773e48682bc640efb8c118606463e75da452a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9eb53602dfc8bc54c7ceb5b418b6c9ddec3f5a92","unresolved":false,"context_lines":[{"line_number":1770,"context_line":"    pagesize \u003d _get_numa_pagesize_constraint(flavor, image_meta)"},{"line_number":1771,"context_line":"    vpmems \u003d get_vpmems(flavor)"},{"line_number":1772,"context_line":""},{"line_number":1773,"context_line":"    # https://bugs.launchpad.net/nova/+bug/1844721"},{"line_number":1774,"context_line":"    # Default mem_page_size to small for pinned instances"},{"line_number":1775,"context_line":"    #"},{"line_number":1776,"context_line":"    cpu_policy \u003d get_cpu_policy_constraint(flavor, image_meta)"},{"line_number":1777,"context_line":""},{"line_number":1778,"context_line":"    if pagesize is None and cpu_policy \u003d\u003d fields.CPUAllocationPolicy.DEDICATED:"},{"line_number":1779,"context_line":"        pagesize \u003d MEMPAGES_SMALL"},{"line_number":1780,"context_line":""},{"line_number":1781,"context_line":"    if nodes or pagesize or vpmems:"},{"line_number":1782,"context_line":"        nodes \u003d nodes or 1"}],"source_content_type":"text/x-python","patch_set":4,"id":"3fa7e38b_75d82b52","line":1779,"range":{"start_line":1773,"start_character":4,"end_line":1779,"end_character":33},"updated":"2019-10-11 10:54:18.000000000","message":"so looking at this again\nhttps://review.opendev.org/#/c/532168/27/nova/virt/hardware.py\nshould have had the same effect as this much simpler change.\n\nso with no code change at all you might be able to just set\nthe reserved_huge_pages config option to get the desired effect.\n\nin either case it would be better to move this check into \n_get_numa_pagesize_constraint\n\nsince that is what that function should do.","commit_id":"5ad4394726e0838852899209cfeb7a651e71a2a5"}],"releasenotes/notes/bug-1844721-default-to-small-pages-for-pinned-instances-5861e1e00b0e5ca1.yaml":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9eb53602dfc8bc54c7ceb5b418b6c9ddec3f5a92","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"upgrade:"},{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    If VM CPU is pinned but mem_page_size is not set, many VMs can be scheduled"},{"line_number":5,"context_line":"    on a NUMA node while their memory needs are met \"globally\" on the host as"},{"line_number":6,"context_line":"    modified by ram_allocation_ratio and reserved_host_memory_mb."},{"line_number":7,"context_line":""},{"line_number":8,"context_line":"    When the system starts to take load, the allocated memroy for the affined"},{"line_number":9,"context_line":"    VMs can exceed the aviable memroy on a NUMA node. The OOM reaper runs per"}],"source_content_type":"text/x-yaml","patch_set":4,"id":"3fa7e38b_d5f9df7f","line":6,"range":{"start_line":4,"start_character":53,"end_line":6,"end_character":65},"updated":"2019-10-11 10:54:18.000000000","message":"so i though this was still the case but it is not.\n\nwe subtly change this behavior by adding this else\nblock https://github.com/openstack/nova/blob/d3403e52942e74edc1b677e83062bfb2dbcd192d/nova/virt/hardware.py#L1071-L1093\n\nin https://review.opendev.org/#/c/532168/27 a year ago.\n\nif the instance has a numa toplogy but does not have a mempage request then we will set the page size to SMALL\non line 1077\n\nthen we check if the available memory on that numa node of that page size is enough to fit the instnace. note the available memory is not the same as the free memory\nwe allow over subscription.\n\nthe change this patch is making will instead force the code to take the if branch instead of the else\n\nhttps://github.com/openstack/nova/blob/d3403e52942e74edc1b677e83062bfb2dbcd192d/nova/virt/hardware.py#L1055-L1070\nso it will disable over subscription.","commit_id":"5ad4394726e0838852899209cfeb7a651e71a2a5"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"9eb53602dfc8bc54c7ceb5b418b6c9ddec3f5a92","unresolved":false,"context_lines":[{"line_number":10,"context_line":"    NUMA node so when the NUMA local memory is exausted it can lead to the"},{"line_number":11,"context_line":"    kernel killing host processes."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"    The above issue can be mitigated by using hw:mem_page_size\u003dsmall, and"},{"line_number":14,"context_line":"    setting reserved_huge_pages for small pages per NUMA node in nova.conf."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"    A change is made in this release to default mem_page_size to small for"},{"line_number":17,"context_line":"    pinned instances if it is not set, so as to avoid VNF impact and avoid"}],"source_content_type":"text/x-yaml","patch_set":4,"id":"3fa7e38b_1531b759","line":14,"range":{"start_line":13,"start_character":70,"end_line":14,"end_character":75},"updated":"2019-10-11 10:54:18.000000000","message":"i think this is actully the only thing that is required.\nbecause we actully aread are checking the smallest page size reported per numa node already.\n\nhw:mem_page_size\u003dsmall  disable over subsription which is not actully required.","commit_id":"5ad4394726e0838852899209cfeb7a651e71a2a5"}]}