)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"16f71f5158e461e6fc250798d603b8639221d751","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"add libvirt pqos spec"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"- This spec propoposes adding support for"},{"line_number":10,"context_line":"  static allocation of cache and memory bandwidth limits to"},{"line_number":11,"context_line":"  the libvirt virt driver."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"blueprint libvirt-pqos"},{"line_number":14,"context_line":"Change-Id: If2a5ec0e1c08d3a5d86f5cf1ba597611b3139774"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"9fb8cfa7_5b7708a1","line":11,"range":{"start_line":9,"start_character":0,"end_line":11,"end_character":26},"updated":"2019-07-02 13:39:45.000000000","message":"Can you reformat this when you have time?","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"}],"specs/train/approved/libvirt-pqos.rst":[{"author":{"_account_id":18051,"name":"David Shaughnessy","email":"david.shaughnessy@intel.com","username":"davidsha"},"change_message_id":"0a3ea4855f93c848f2f6620553e48cfdfdde3144","unresolved":false,"context_lines":[{"line_number":8,"context_line":"libvirt-platform-quality-of-service"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"To run realtime workloads on commodity hardware effectively requires that the"},{"line_number":12,"context_line":"guest and host be tuned for deterministic low latency and high throughput."},{"line_number":13,"context_line":"Typically this is done using the existing EPA features such as hugepages, CPU"},{"line_number":14,"context_line":"pinning and emulator thread policies on a realtime host kernel. If the host"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_8582d5c7","line":11,"range":{"start_line":11,"start_character":7,"end_line":11,"end_character":15},"updated":"2019-06-04 16:06:02.000000000","message":"I think this should be \"real time\" or \"real-time\".\nI don\u0027t think it\u0027s meant to be one word.\nBut this is a nit.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6e0f035b8a625734d3e3d9c5555dc2ddf20d3cd7","unresolved":false,"context_lines":[{"line_number":8,"context_line":"libvirt-platform-quality-of-service"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"To run realtime workloads on commodity hardware effectively requires that the"},{"line_number":12,"context_line":"guest and host be tuned for deterministic low latency and high throughput."},{"line_number":13,"context_line":"Typically this is done using the existing EPA features such as hugepages, CPU"},{"line_number":14,"context_line":"pinning and emulator thread policies on a realtime host kernel. If the host"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_449a9614","line":11,"range":{"start_line":11,"start_character":7,"end_line":11,"end_character":15},"in_reply_to":"9fb8cfa7_8582d5c7","updated":"2019-06-17 09:33:50.000000000","message":"Done","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"To run realtime workloads on commodity hardware effectively requires that the"},{"line_number":12,"context_line":"guest and host be tuned for deterministic low latency and high throughput."},{"line_number":13,"context_line":"Typically this is done using the existing EPA features such as hugepages, CPU"},{"line_number":14,"context_line":"pinning and emulator thread policies on a realtime host kernel. If the host"},{"line_number":15,"context_line":"also runs non-realtime workloads or unoptimized workloads, the determinism"},{"line_number":16,"context_line":"afforded by the careful tuning of the guest can be undone due to noisy neighbor"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_3ccc7b88","line":13,"range":{"start_line":13,"start_character":42,"end_line":13,"end_character":45},"updated":"2019-06-03 08:49:05.000000000","message":"Enhanced Platform Awareness?","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c36dce444f4eedd9079691f25f032ab3657f837b","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"To run realtime workloads on commodity hardware effectively requires that the"},{"line_number":12,"context_line":"guest and host be tuned for deterministic low latency and high throughput."},{"line_number":13,"context_line":"Typically this is done using the existing EPA features such as hugepages, CPU"},{"line_number":14,"context_line":"pinning and emulator thread policies on a realtime host kernel. If the host"},{"line_number":15,"context_line":"also runs non-realtime workloads or unoptimized workloads, the determinism"},{"line_number":16,"context_line":"afforded by the careful tuning of the guest can be undone due to noisy neighbor"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_27945b44","line":13,"range":{"start_line":13,"start_character":42,"end_line":13,"end_character":45},"in_reply_to":"9fb8cfa7_3ccc7b88","updated":"2019-06-04 14:11:20.000000000","message":"yes","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":18051,"name":"David Shaughnessy","email":"david.shaughnessy@intel.com","username":"davidsha"},"change_message_id":"0a3ea4855f93c848f2f6620553e48cfdfdde3144","unresolved":false,"context_lines":[{"line_number":24,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Realtime workloads rely on deterministic cache latency, memory access time and"},{"line_number":27,"context_line":"bandwidth to fulfill their SLAs. Noisy neighbor effect from non-realtime"},{"line_number":28,"context_line":"workloads co-located on the same host can result in violation of realtime"},{"line_number":29,"context_line":"workloads SLAs, which can be mitigated via static allocation of cache and"},{"line_number":30,"context_line":"memory bandwidth to instances."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_65f6c192","line":27,"range":{"start_line":27,"start_character":33,"end_line":27,"end_character":38},"updated":"2019-06-04 16:06:02.000000000","message":"The noisy","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":49,"context_line":""},{"line_number":50,"context_line":"    Throughout this spec we refer to enabling both static cache allocation and"},{"line_number":51,"context_line":"    memory bandwidth limits. The primary feature is static management of cache"},{"line_number":52,"context_line":"    and memory bandwidth limits is proposed as a stretch goal for Train. It is"},{"line_number":53,"context_line":"    intended that both will be implemented in Train but a phased approach is"},{"line_number":54,"context_line":"    proposed to complete cache allocation first followed by memory bandwidth"},{"line_number":55,"context_line":"    limits."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":".. note::"},{"line_number":58,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_8e2d51ff","line":55,"range":{"start_line":52,"start_character":73,"end_line":55,"end_character":11},"updated":"2019-05-31 11:02:05.000000000","message":"So the discussion of more details of memory bandwidth limits will be kicked off later, right?","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":49,"context_line":""},{"line_number":50,"context_line":"    Throughout this spec we refer to enabling both static cache allocation and"},{"line_number":51,"context_line":"    memory bandwidth limits. The primary feature is static management of cache"},{"line_number":52,"context_line":"    and memory bandwidth limits is proposed as a stretch goal for Train. It is"},{"line_number":53,"context_line":"    intended that both will be implemented in Train but a phased approach is"},{"line_number":54,"context_line":"    proposed to complete cache allocation first followed by memory bandwidth"},{"line_number":55,"context_line":"    limits."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":".. note::"},{"line_number":58,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_3153646d","line":55,"range":{"start_line":52,"start_character":73,"end_line":55,"end_character":11},"in_reply_to":"bfb3d3c7_8e2d51ff","updated":"2019-05-31 12:34:33.000000000","message":"well this spec contains the details need for the implementation of memory bandwidth limit too.\nthe intent of this is that if we only have time to land one of the two feature we complete cache allocation first since we have at least 1 customer asking for that but we have 0 customer asking for memory bandwith limts yet but its a closely related feature and is trivial to add. since we expect that to be there next request once cache allocation is implemeted i think we should also support it now rather then wait.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":"This spec proposes extending the libvirt virt driver to support static"},{"line_number":65,"context_line":"allocation of cacheways and enforcement of maximum memory bandwidth limits."},{"line_number":66,"context_line":"delegation of discovery and enforcement to an external agent (RMD, CYBORG, cron"},{"line_number":67,"context_line":"job) will also be possible. This will be achieved at a high level by"},{"line_number":68,"context_line":"introducing five new standard resource classes: two for the class of service"},{"line_number":69,"context_line":"types and three for L3 cache, to model the capacity of each resource in"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_1ca2d79b","line":66,"range":{"start_line":66,"start_character":0,"end_line":66,"end_character":1},"updated":"2019-06-03 08:49:05.000000000","message":"D","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":87,"context_line":"The detail of the config parameters are specified in the \"Other deployer"},{"line_number":88,"context_line":"impact\" section of this spec."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Two new compute capability traits will be added to indicate if a host support"},{"line_number":91,"context_line":"cache allocation or memory bandwidth limits based in the values of the config"},{"line_number":92,"context_line":"options. These new traits will be used in combination with a new request"},{"line_number":93,"context_line":"prefilter to ensure that we do not mix cache allocated instance and non cache"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_fcc10330","line":90,"range":{"start_line":90,"start_character":70,"end_line":90,"end_character":77},"updated":"2019-06-03 08:49:05.000000000","message":"supports","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":18051,"name":"David Shaughnessy","email":"david.shaughnessy@intel.com","username":"davidsha"},"change_message_id":"0a3ea4855f93c848f2f6620553e48cfdfdde3144","unresolved":false,"context_lines":[{"line_number":88,"context_line":"impact\" section of this spec."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Two new compute capability traits will be added to indicate if a host support"},{"line_number":91,"context_line":"cache allocation or memory bandwidth limits based in the values of the config"},{"line_number":92,"context_line":"options. These new traits will be used in combination with a new request"},{"line_number":93,"context_line":"prefilter to ensure that we do not mix cache allocated instance and non cache"},{"line_number":94,"context_line":"allocated instances on the same host. similarly the same will be done for"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_800f232d","line":91,"range":{"start_line":91,"start_character":64,"end_line":91,"end_character":66},"updated":"2019-06-04 16:06:02.000000000","message":"in","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":18051,"name":"David Shaughnessy","email":"david.shaughnessy@intel.com","username":"davidsha"},"change_message_id":"0a3ea4855f93c848f2f6620553e48cfdfdde3144","unresolved":false,"context_lines":[{"line_number":88,"context_line":"impact\" section of this spec."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Two new compute capability traits will be added to indicate if a host support"},{"line_number":91,"context_line":"cache allocation or memory bandwidth limits based in the values of the config"},{"line_number":92,"context_line":"options. These new traits will be used in combination with a new request"},{"line_number":93,"context_line":"prefilter to ensure that we do not mix cache allocated instance and non cache"},{"line_number":94,"context_line":"allocated instances on the same host. similarly the same will be done for"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_60122f5b","line":91,"range":{"start_line":91,"start_character":50,"end_line":91,"end_character":52},"updated":"2019-06-04 16:06:02.000000000","message":"on","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":90,"context_line":"Two new compute capability traits will be added to indicate if a host support"},{"line_number":91,"context_line":"cache allocation or memory bandwidth limits based in the values of the config"},{"line_number":92,"context_line":"options. These new traits will be used in combination with a new request"},{"line_number":93,"context_line":"prefilter to ensure that we do not mix cache allocated instance and non cache"},{"line_number":94,"context_line":"allocated instances on the same host. similarly the same will be done for"},{"line_number":95,"context_line":"memory limits."},{"line_number":96,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_9cdc8719","line":93,"range":{"start_line":93,"start_character":55,"end_line":93,"end_character":63},"updated":"2019-06-03 08:49:05.000000000","message":"instances","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":124,"context_line":""},{"line_number":125,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":126,"context_line":"    hw:numa_cache\u003d4"},{"line_number":127,"context_line":"    hw:numa_bandwidth\u003d10"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Alternatively, consider a guest with two virtual NUMA nodes, with unbalanced"},{"line_number":130,"context_line":"CPUs, cache and memory::"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_ce9ca96b","line":127,"range":{"start_line":127,"start_character":4,"end_line":127,"end_character":24},"updated":"2019-05-31 11:02:05.000000000","message":"\u0027hw:numa_cache\u0027 represents the total cache ways of the instance, that is, each NUMA node in this case will have 2 cache ways. \nDoes \u0027hw:numa_bandwidth\u0027 means each NUMA node of the instance will be reserved 5% memory bandwidth?","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":124,"context_line":""},{"line_number":125,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":126,"context_line":"    hw:numa_cache\u003d4"},{"line_number":127,"context_line":"    hw:numa_bandwidth\u003d10"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Alternatively, consider a guest with two virtual NUMA nodes, with unbalanced"},{"line_number":130,"context_line":"CPUs, cache and memory::"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_71eb1c77","line":127,"range":{"start_line":127,"start_character":4,"end_line":127,"end_character":24},"in_reply_to":"bfb3d3c7_ce9ca96b","updated":"2019-05-31 12:34:33.000000000","message":"i was thinking of defineing it that it would not be devided but since all the other hw:numa_* values are deived by hw:numa_nodes to determin the amount per numa node its better to be consitent so ill update this to 20 to be consitent.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":148,"context_line":"    ``hw:numa_mem`` options and because if you imbalance CPUs and memory it is"},{"line_number":149,"context_line":"    likely that imbalanced cache and memory bandwidth will also be desired."},{"line_number":150,"context_line":"    It is expected that short form ``hw:numa_cache\u003dX`` and"},{"line_number":151,"context_line":"    ``hw:numa_bandwidth\u003dY`` will be used in most applications. Note the minimum"},{"line_number":152,"context_line":"    allocation size of cache if specified is two cacheways per NUMA node. Each"},{"line_number":153,"context_line":"    instance will consume one cache and memory class of service resource per"},{"line_number":154,"context_line":"    NUMA node if either memory limits or cache allocation are requested. For"},{"line_number":155,"context_line":"    hosts that support code and data partitioning of last level"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_2ea0e58f","line":152,"range":{"start_line":151,"start_character":63,"end_line":152,"end_character":73},"updated":"2019-05-31 11:02:05.000000000","message":"Who introduced the limitation of \u0027the minimum allocation size of cache if specified is two cacheways per NUMA node\u0027? \nHardware, kernel, libvirt or introduced by this spec?","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c36dce444f4eedd9079691f25f032ab3657f837b","unresolved":false,"context_lines":[{"line_number":148,"context_line":"    ``hw:numa_mem`` options and because if you imbalance CPUs and memory it is"},{"line_number":149,"context_line":"    likely that imbalanced cache and memory bandwidth will also be desired."},{"line_number":150,"context_line":"    It is expected that short form ``hw:numa_cache\u003dX`` and"},{"line_number":151,"context_line":"    ``hw:numa_bandwidth\u003dY`` will be used in most applications. Note the minimum"},{"line_number":152,"context_line":"    allocation size of cache if specified is two cacheways per NUMA node. Each"},{"line_number":153,"context_line":"    instance will consume one cache and memory class of service resource per"},{"line_number":154,"context_line":"    NUMA node if either memory limits or cache allocation are requested. For"},{"line_number":155,"context_line":"    hosts that support code and data partitioning of last level"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_7b9f3e50","line":152,"range":{"start_line":151,"start_character":63,"end_line":152,"end_character":73},"in_reply_to":"9fb8cfa7_99e81917","updated":"2019-06-04 14:11:20.000000000","message":"@huachang\nright as i said above in my previous respoce we will use the value from libvirt however we will need to call this out in doucmentation so that operator can understand how to correctly create flavors.\n\n@alex\nright so libvirt does not allows us to directly set the cat mask which is unfortunate as it really is not a good map to hardware. that means that we cannoth deal with fragmentation in nova.\n\nthe only approach i can think of to address the fragmentation would be to limit the min and max unit to be the same value in placement as such each vm would only be able to consume 1 allocation of the minimum available and there for we should not have to deal with fragmentation.\n\nthat is a fairly severe restiction but since we cannot select which cacheways to assign as we would with cpu pinning there is no other way to achive this via the api presented via libvirt.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":148,"context_line":"    ``hw:numa_mem`` options and because if you imbalance CPUs and memory it is"},{"line_number":149,"context_line":"    likely that imbalanced cache and memory bandwidth will also be desired."},{"line_number":150,"context_line":"    It is expected that short form ``hw:numa_cache\u003dX`` and"},{"line_number":151,"context_line":"    ``hw:numa_bandwidth\u003dY`` will be used in most applications. Note the minimum"},{"line_number":152,"context_line":"    allocation size of cache if specified is two cacheways per NUMA node. Each"},{"line_number":153,"context_line":"    instance will consume one cache and memory class of service resource per"},{"line_number":154,"context_line":"    NUMA node if either memory limits or cache allocation are requested. For"},{"line_number":155,"context_line":"    hosts that support code and data partitioning of last level"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_d4b6763e","line":152,"range":{"start_line":151,"start_character":63,"end_line":152,"end_character":73},"in_reply_to":"bfb3d3c7_2ea0e58f","updated":"2019-05-31 12:34:33.000000000","message":"hardware.\ni added this specificaly due to this assertion\nhttps://github.com/intel/intel-cmt-cat/wiki/FAQ#how-fine-grained-can-the-cache-assignment-be-for-each-cos\n\nits seams to match the only examlple libvirt xml i could find with the min allcoation specified also\n\n    \u003ccache\u003e\n      \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270-5\u0027\u003e\n        \u003ccontrol granularity\u003d\u0027768\u0027 min\u003d\u00271536\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u00274\u0027/\u003e\n      \u003c/bank\u003e\n      \u003cbank id\u003d\u00271\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00276-11\u0027\u003e\n        \u003ccontrol granularity\u003d\u0027768\u0027 min\u003d\u00271536\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u00274\u0027/\u003e\n      \u003c/bank\u003e\n\u003c/cache\u003e\n\nhttps://github.com/libvirt/libvirt/blob/0ec6343a069b21178d4580688a8380dbb6d76620/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml#L44-L52","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"3a5815da9d09ea02d34b99bde6ebe395d4b0b4b2","unresolved":false,"context_lines":[{"line_number":148,"context_line":"    ``hw:numa_mem`` options and because if you imbalance CPUs and memory it is"},{"line_number":149,"context_line":"    likely that imbalanced cache and memory bandwidth will also be desired."},{"line_number":150,"context_line":"    It is expected that short form ``hw:numa_cache\u003dX`` and"},{"line_number":151,"context_line":"    ``hw:numa_bandwidth\u003dY`` will be used in most applications. Note the minimum"},{"line_number":152,"context_line":"    allocation size of cache if specified is two cacheways per NUMA node. Each"},{"line_number":153,"context_line":"    instance will consume one cache and memory class of service resource per"},{"line_number":154,"context_line":"    NUMA node if either memory limits or cache allocation are requested. For"},{"line_number":155,"context_line":"    hosts that support code and data partitioning of last level"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_99e81917","line":152,"range":{"start_line":151,"start_character":63,"end_line":152,"end_character":73},"in_reply_to":"bfb3d3c7_d4b6763e","updated":"2019-06-03 07:37:55.000000000","message":"@Sean,\n\nThe minimum CAT allocation size is not always 2 adjacent cache-ways, this value depends on the implementation of CPU architecture. For example, my BDW-E5-2699v4 machine, the libvirt capabilities output is: (no existence of \u0027min\u0027 in \u003cbank\u003e element means the minimal allocation unit is the value of \u0027granularity\u0027.)\n\n\u0027\u0027\u0027\n    \u003ccache\u003e\n      \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002755\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270-21,44-65\u0027\u003e\n        \u003ccontrol granularity\u003d\u00272816\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u002716\u0027/\u003e\n      \u003c/bank\u003e\n      \u003cbank id\u003d\u00271\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002755\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u002722-43,66-87\u0027\u003e\n        \u003ccontrol granularity\u003d\u00272816\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u002716\u0027/\u003e\n      \u003c/bank\u003e\n    \u003c/cache\u003e\n\u0027\u0027\n\nThe minimal cache allocating size is determined by the property value of \u0027min\u0027 of \u003ccache\u003e/\u003cbank\u003e element, *IF* \u0027min\u0027 exists. Otherwise, the minimum allocation is the value of property \u0027granularity\u0027.\n\nWe should not impose a limitation the cache allocation unit is size of two granularity, this value is really hardware implemtation dependent.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"e54333c6a2324db875e337eabe82224dbf1973c4","unresolved":false,"context_lines":[{"line_number":148,"context_line":"    ``hw:numa_mem`` options and because if you imbalance CPUs and memory it is"},{"line_number":149,"context_line":"    likely that imbalanced cache and memory bandwidth will also be desired."},{"line_number":150,"context_line":"    It is expected that short form ``hw:numa_cache\u003dX`` and"},{"line_number":151,"context_line":"    ``hw:numa_bandwidth\u003dY`` will be used in most applications. Note the minimum"},{"line_number":152,"context_line":"    allocation size of cache if specified is two cacheways per NUMA node. Each"},{"line_number":153,"context_line":"    instance will consume one cache and memory class of service resource per"},{"line_number":154,"context_line":"    NUMA node if either memory limits or cache allocation are requested. For"},{"line_number":155,"context_line":"    hosts that support code and data partitioning of last level"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_1af65c91","line":152,"range":{"start_line":151,"start_character":63,"end_line":152,"end_character":73},"in_reply_to":"bfb3d3c7_d4b6763e","updated":"2019-06-03 02:58:24.000000000","message":"@Sean, there also have fragmentation issue. I confirm with Huaqiang, the libvirt doesn\u0027t resolve the fragmentation issue.\n\nSo If that is true, we have to deal with that in the Nova. one of way is that we can only allow allocating the fixed size cache, just like vpmem.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":154,"context_line":"    NUMA node if either memory limits or cache allocation are requested. For"},{"line_number":155,"context_line":"    hosts that support code and data partitioning of last level"},{"line_number":156,"context_line":"    cache, ``hw:numa_code_cache`` and ``hw:numa_data_cache`` will be used"},{"line_number":157,"context_line":"    instead of ``hw:numa_cache``. As code and data partitioning is a"},{"line_number":158,"context_line":"    system-wide BIOS configuration, hosts with cache allocation enabled will"},{"line_number":159,"context_line":"    either report inventories of ``L3_CACHE`` or inventories of"},{"line_number":160,"context_line":"    ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":161,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_ae0b758b","line":158,"range":{"start_line":157,"start_character":37,"end_line":158,"end_character":34},"updated":"2019-05-31 11:02:05.000000000","message":"For Intel CPU with CAT\u0026CDP, the CDP could be enabled/disable at the mounting stage:\n # mount -t resctrl resctrl [-o cdp] /sys/fs/resctrl\n\nRelated doc:\nhttps://www.kernel.org/doc/Documentation/x86/resctrl_ui.txt","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":154,"context_line":"    NUMA node if either memory limits or cache allocation are requested. For"},{"line_number":155,"context_line":"    hosts that support code and data partitioning of last level"},{"line_number":156,"context_line":"    cache, ``hw:numa_code_cache`` and ``hw:numa_data_cache`` will be used"},{"line_number":157,"context_line":"    instead of ``hw:numa_cache``. As code and data partitioning is a"},{"line_number":158,"context_line":"    system-wide BIOS configuration, hosts with cache allocation enabled will"},{"line_number":159,"context_line":"    either report inventories of ``L3_CACHE`` or inventories of"},{"line_number":160,"context_line":"    ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":161,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_74cd4ab5","line":158,"range":{"start_line":157,"start_character":37,"end_line":158,"end_character":34},"in_reply_to":"bfb3d3c7_ae0b758b","updated":"2019-05-31 12:34:33.000000000","message":"ah you are correct i will update this although we do not expect(intend to support) this to change at runtime while so from the perspective of this spec it will either be enabled or disabled.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":177,"context_line":"    bandwidth are inherently linked to the memory subsystem on a host and"},{"line_number":178,"context_line":"    therefore are significantly impacted by NUMA affinity. For this reason, and"},{"line_number":179,"context_line":"    the accuracy limitation of the hardware, PQoS will only be supported for"},{"line_number":180,"context_line":"    instances with ``hw:cpu_policy\u003ddedicated``."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"Alternatives"},{"line_number":183,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_f1c2ac52","line":180,"range":{"start_line":180,"start_character":19,"end_line":180,"end_character":45},"updated":"2019-05-31 11:02:05.000000000","message":"+1\nSince we are seeking a way to improve performance, this limitation is reasonable.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"e54333c6a2324db875e337eabe82224dbf1973c4","unresolved":false,"context_lines":[{"line_number":177,"context_line":"    bandwidth are inherently linked to the memory subsystem on a host and"},{"line_number":178,"context_line":"    therefore are significantly impacted by NUMA affinity. For this reason, and"},{"line_number":179,"context_line":"    the accuracy limitation of the hardware, PQoS will only be supported for"},{"line_number":180,"context_line":"    instances with ``hw:cpu_policy\u003ddedicated``."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"Alternatives"},{"line_number":183,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_13cc0f88","line":180,"range":{"start_line":180,"start_character":19,"end_line":180,"end_character":45},"in_reply_to":"bfb3d3c7_f1c2ac52","updated":"2019-06-03 02:58:24.000000000","message":"+1 too, yes, it only makes sense for the dedicated instance.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":197,"context_line":"Data model impact"},{"line_number":198,"context_line":"-----------------"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"The libvirt driver will be extended to create one nested RP per cache region."},{"line_number":201,"context_line":"On all current hardware that supports CAT there is a 1:1 mapping between cache"},{"line_number":202,"context_line":"region per NUMA node for L3 cache."},{"line_number":203,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_eef64d5d","line":200,"range":{"start_line":200,"start_character":64,"end_line":200,"end_character":76},"updated":"2019-05-31 11:02:05.000000000","message":"\u0027cache region\u0027 is a little confusing, I think you mean the last level sharing cache that CAT is working on, right?","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":197,"context_line":"Data model impact"},{"line_number":198,"context_line":"-----------------"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"The libvirt driver will be extended to create one nested RP per cache region."},{"line_number":201,"context_line":"On all current hardware that supports CAT there is a 1:1 mapping between cache"},{"line_number":202,"context_line":"region per NUMA node for L3 cache."},{"line_number":203,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_b40bc28b","line":200,"range":{"start_line":200,"start_character":64,"end_line":200,"end_character":76},"in_reply_to":"bfb3d3c7_eef64d5d","updated":"2019-05-31 12:34:33.000000000","message":"not quite.\ni am refering to the cache id of the l3 cache segment.\n\non all existing hardware at least up to cascadelake the cache region id of the l3 cache on a specific cpu package\ni dont have cascade lake cpus to had so this is an example form an ivybridge system\n\n[sean@workstation ~]$ cat /sys/bus/cpu/devices/cpu47/cache/index3/id \n1\n ls -al /sys/bus/cpu/devices/cpu47/node1\nlrwxrwxrwx 1 root root 0 May 31 12:34 /sys/bus/cpu/devices/cpu47/node1 -\u003e ../../node/node1\n\nif you are on haswell or later and enable cluster on die\nsuch that each meory contoler/home agent on the physical cpu package works as an independt numa node then the l3 cache is similarly reported as 2 seperate cache regions so \n\nso while on my stytem the cache region id matches teh phyical\n_package_id that is not always the case.\n\n[sean@workstation ~]$ cat /sys/bus/cpu/devices/cpu47/cache/index3/id \n1\n[sean@workstation ~]$ cat /sys/bus/cpu/devices/cpu47/topology/physical_package_id\n1\n\nthe cache region id however always matches on the numa node on all shiping hardware.\n\nhowever since the intel silocon folk i was working with inisisted that they wanted to keep the flexablity to change this if needed at some point in the future i have tried to use cache region where it matters to refer to the cache id rather then numa node.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":198,"context_line":"-----------------"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"The libvirt driver will be extended to create one nested RP per cache region."},{"line_number":201,"context_line":"On all current hardware that supports CAT there is a 1:1 mapping between cache"},{"line_number":202,"context_line":"region per NUMA node for L3 cache."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"The libvirt driver will also create one RP per memory controller with an"},{"line_number":205,"context_line":"inventory of bandwidth."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_eea40d4d","line":202,"range":{"start_line":201,"start_character":53,"end_line":202,"end_character":34},"updated":"2019-05-31 11:02:05.000000000","message":"Not clear. What kind of \u00271:1\u0027 mapping?","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"3a5815da9d09ea02d34b99bde6ebe395d4b0b4b2","unresolved":false,"context_lines":[{"line_number":198,"context_line":"-----------------"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"The libvirt driver will be extended to create one nested RP per cache region."},{"line_number":201,"context_line":"On all current hardware that supports CAT there is a 1:1 mapping between cache"},{"line_number":202,"context_line":"region per NUMA node for L3 cache."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"The libvirt driver will also create one RP per memory controller with an"},{"line_number":205,"context_line":"inventory of bandwidth."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_792c4547","line":202,"range":{"start_line":201,"start_character":53,"end_line":202,"end_character":34},"in_reply_to":"bfb3d3c7_74b30aea","updated":"2019-06-03 07:37:55.000000000","message":"OK","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":198,"context_line":"-----------------"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"The libvirt driver will be extended to create one nested RP per cache region."},{"line_number":201,"context_line":"On all current hardware that supports CAT there is a 1:1 mapping between cache"},{"line_number":202,"context_line":"region per NUMA node for L3 cache."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"The libvirt driver will also create one RP per memory controller with an"},{"line_number":205,"context_line":"inventory of bandwidth."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_74b30aea","line":202,"range":{"start_line":201,"start_character":53,"end_line":202,"end_character":34},"in_reply_to":"bfb3d3c7_eea40d4d","updated":"2019-05-31 12:34:33.000000000","message":"see answer above.\n\nthe resouce providers wont technically correspond to NUMA nodes but in practice it always does at least today so\n\nill update this slightly.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":226,"context_line":""},{"line_number":227,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":228,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":229,"context_line":"1-1.2 MBs. A typical processor will have one cache way per physical core."},{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_ce058915","line":229,"range":{"start_line":229,"start_character":10,"end_line":229,"end_character":73},"updated":"2019-05-31 11:02:05.000000000","message":"Disagree.\nThe number of cacheway for Intel latest CPU should be a fixed number and regardless of the number of core number as I know.\nFor example, the E5-2600v4(Broadwell) has 20 cache ways, and Skylake-SP \u0026 Casladelake-SP has 11 cacheways.\n\nThe total number of processor cache ways is indicated by kernel \u0027/sys/fs/resctrl/info/L3/cbm_mask\u0027","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":226,"context_line":""},{"line_number":227,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":228,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":229,"context_line":"1-1.2 MBs. A typical processor will have one cache way per physical core."},{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_b4cc2235","line":229,"range":{"start_line":229,"start_character":10,"end_line":229,"end_character":73},"in_reply_to":"bfb3d3c7_ce058915","updated":"2019-05-31 12:34:33.000000000","message":"ok ill remove this its not really relevent i was just trying to provide context for peopel. it will not affect openstack one way or another as we will just be using the values from libvirt.\n\n   \u003ccache\u003e\n      \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002733\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270\u0027\u003e\n        \u003ccontrol granularity\u003d\u00273\u0027 unit\u003d\u0027MiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u002716\u0027/\u003e\n      \u003c/bank\u003e\n\u003c/cache\u003e","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":227,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":228,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":229,"context_line":"1-1.2 MBs. A typical processor will have one cache way per physical core."},{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"},{"line_number":233,"context_line":"all cache ways on the processor. CAT has a minimum requirement of two cache"},{"line_number":234,"context_line":"ways and a granularity thereafter of one cache way."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_5107f818","line":231,"range":{"start_line":230,"start_character":0,"end_line":231,"end_character":51},"updated":"2019-05-31 11:02:05.000000000","message":"Seems E5-2600v4, Skylake-SP and Cascadelake-SP are not working in this way.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"632b892d700b1945c682a67905b82ace76e1c010","unresolved":false,"context_lines":[{"line_number":227,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":228,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":229,"context_line":"1-1.2 MBs. A typical processor will have one cache way per physical core."},{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"},{"line_number":233,"context_line":"all cache ways on the processor. CAT has a minimum requirement of two cache"},{"line_number":234,"context_line":"ways and a granularity thereafter of one cache way."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_57e8487a","line":231,"range":{"start_line":230,"start_character":0,"end_line":231,"end_character":51},"in_reply_to":"bfb3d3c7_14416ea9","updated":"2019-05-31 12:53:12.000000000","message":"actuly this is a better quote form the same section\n\n\"When CDP is enabled, the existing mask space for CAT-only operation is split. As an example if the system supports 16 CAT-only COS, when CDP is enabled the same MSR interfaces are used, however half of the masks correspond to code, half correspond to data, and the effective number of COS is reduced by half. Code/Data masks are defined per-COS and interleaved in the MSR space as described in subsequent sections\"\n\nso you are right the numer of cacheways does not change but the granularity of how we generate masks does.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":227,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":228,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":229,"context_line":"1-1.2 MBs. A typical processor will have one cache way per physical core."},{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"},{"line_number":233,"context_line":"all cache ways on the processor. CAT has a minimum requirement of two cache"},{"line_number":234,"context_line":"ways and a granularity thereafter of one cache way."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_14416ea9","line":231,"range":{"start_line":230,"start_character":0,"end_line":231,"end_character":51},"in_reply_to":"bfb3d3c7_5107f818","updated":"2019-05-31 12:34:33.000000000","message":"that should have said enabled.\n\nagain this the number of cache ways of each type will be reported in libvirt like this\n\n    \u003ccache\u003e\n      \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270-5\u0027\u003e\n        \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027code\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n        \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027data\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n      \u003c/bank\u003e\n      \u003cbank id\u003d\u00271\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00276-11\u0027\u003e\n        \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027code\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n        \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027data\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n      \u003c/bank\u003e\n\u003c/cache\u003e\n\nso we will actully be useing the values form libvirt when reporting the inventory\n\nhttps://github.com/libvirt/libvirt/blob/0ec6343a069b21178d4580688a8380dbb6d76620/tests/vircaps2xmldata/vircaps-x86_64-resctrl-cdp.xml\n\nill just replace this with explit examples rather then trying to simplfy with a generaliation that when you enable\ncode and data priortisation you get an equal numabe of code and data cache segments that are half the size of the partitioned cacheways.\n\ni was basing it on the intel archtecture guide\nhttps://software.intel.com/sites/default/files/managed/39/c5/325462-sdm-vol-1-2abcd-3abcd.pdf#G55.133315\n\n\"An example of the operating mode of CDP is shown in Figure 17-29. Shown at the top are traditional CAT usage models where capacity masks map 1:1 with a COS number to enable control over the cache space which a given COS (and thus applications, threads or VMs) may occupy. Shown at the bottom are example mask configurations where CDP is enabled, and each COS number maps 1:2 to two masks, one for code and one for data. This enables code and data to be either overlapped or isolated to varying degrees either globally or on a per-COS basis, depending on application and system needs\"\n\n\ni guess i could have missread that and the number of cacheways does not change but rather the granlarity of what infomation can be sotred in each cacheway has jsut been increased.\n\nso you still have 20 cache ways but for each cos policy\nyou can state if a cache way can be written to to store instruction or data.\n\nill carify this in the next version","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"},{"line_number":233,"context_line":"all cache ways on the processor. CAT has a minimum requirement of two cache"},{"line_number":234,"context_line":"ways and a granularity thereafter of one cache way."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"In addition to the resource classes, above two additional resources class will"},{"line_number":237,"context_line":"be added to model the number of \"Class of Service (COS)\" or \"Quality of Service"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_b152f411","line":234,"range":{"start_line":233,"start_character":33,"end_line":234,"end_character":4},"updated":"2019-05-31 11:02:05.000000000","message":"Current Intel CPUs do not have such kind of limitation. Is this introduced by AMD CPUs?\nYou can assign only one cache way to the instance, and following \u0027schemata\u0027 is valid:\n\n[root@c200 p0]# echo \"L3:0\u003d1;1\u003d1\" \u003eschemata \n[root@c200 p0]# cat schemata \n    L3:0\u003d00001;1\u003d00001\n[root@c200 p0]# pwd\n/sys/fs/resctrl/p0","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"},{"line_number":233,"context_line":"all cache ways on the processor. CAT has a minimum requirement of two cache"},{"line_number":234,"context_line":"ways and a granularity thereafter of one cache way."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"In addition to the resource classes, above two additional resources class will"},{"line_number":237,"context_line":"be added to model the number of \"Class of Service (COS)\" or \"Quality of Service"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_71a6bc11","line":234,"range":{"start_line":234,"start_character":9,"end_line":234,"end_character":51},"updated":"2019-05-31 11:02:05.000000000","message":"This information should be gotten through parsing kernel interface \u0027/sys/fs/resctrl/info/L3/min_cbm_bits\u0027, instead of hard coded here.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"},{"line_number":233,"context_line":"all cache ways on the processor. CAT has a minimum requirement of two cache"},{"line_number":234,"context_line":"ways and a granularity thereafter of one cache way."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"In addition to the resource classes, above two additional resources class will"},{"line_number":237,"context_line":"be added to model the number of \"Class of Service (COS)\" or \"Quality of Service"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_d4a99623","line":234,"range":{"start_line":233,"start_character":33,"end_line":234,"end_character":4},"in_reply_to":"bfb3d3c7_b152f411","updated":"2019-05-31 12:34:33.000000000","message":"no this information should be gotten by parsing the libvirt capability xml which actually parses resctrul filesystem under the hood.\n\nwhen we create the inventories of cache ways in placemetn we can the reported minium allcoation size and granualrity\nas the min allocation and set size respectivly.\n\nthe limitation of need a min of 2 cacheways was coming form the intel cmt tool faq but if that is incorrect ill just remove that limitation form the spec. that makes life a lost simpeler.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"3a5815da9d09ea02d34b99bde6ebe395d4b0b4b2","unresolved":false,"context_lines":[{"line_number":230,"context_line":"If code and data prioritization is disabled it will have two cache ways, one of"},{"line_number":231,"context_line":"each type per physical core that are half the size. PQoS and CAT in particular"},{"line_number":232,"context_line":"works by restricting the cache way a process can write to but allows reads from"},{"line_number":233,"context_line":"all cache ways on the processor. CAT has a minimum requirement of two cache"},{"line_number":234,"context_line":"ways and a granularity thereafter of one cache way."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"In addition to the resource classes, above two additional resources class will"},{"line_number":237,"context_line":"be added to model the number of \"Class of Service (COS)\" or \"Quality of Service"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_f969158b","line":234,"range":{"start_line":233,"start_character":33,"end_line":234,"end_character":4},"in_reply_to":"bfb3d3c7_d4a99623","updated":"2019-06-03 07:37:55.000000000","message":"As I replied in above comment, the minimum cacheways exists, but not always be 2 cacheways. \n\nWe still consider there is the constrain of the minimum cacheways (more precisely, the cache allocation size). The minimum cache allocation size is given by the value \u0027min\u0027 of \u003ccache\u003e/\u003cbank\u003e (of result of command \u0027virsh capabilities\u0027).\n\nIf \u0027min\u0027 does not appear in \u003ccache\u003e/\u003c/bank\u003e section, then the minimum allocation size is size of \"granularity\".","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"756ea5ed3d4ddf38ab5d8bb0e4ba7b07650b776c","unresolved":false,"context_lines":[{"line_number":237,"context_line":"be added to model the number of \"Class of Service (COS)\" or \"Quality of Service"},{"line_number":238,"context_line":"(QoS)\" policies supported by the CPU."},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"- ``CACHE_COS``"},{"line_number":241,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":244,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_9107b0ca","line":241,"range":{"start_line":240,"start_character":0,"end_line":241,"end_character":26},"updated":"2019-05-31 11:02:05.000000000","message":"Do we need two \u0027COS\u0027 resource class? Kernel only export the minimal value of cache_closid and mba_closid, the interface \u0027/sys/fs/resctrl/info/L3/num_closids\u0027.\n\nOnce MBA and CAT are all enabled, the closid value will likly be 8.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5b865474f58dccb3c80c7f1ef5d1b6cbab23af2f","unresolved":false,"context_lines":[{"line_number":237,"context_line":"be added to model the number of \"Class of Service (COS)\" or \"Quality of Service"},{"line_number":238,"context_line":"(QoS)\" policies supported by the CPU."},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"- ``CACHE_COS``"},{"line_number":241,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":244,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bfb3d3c7_97eb2011","line":241,"range":{"start_line":240,"start_character":0,"end_line":241,"end_character":26},"in_reply_to":"bfb3d3c7_9107b0ca","updated":"2019-05-31 12:34:33.000000000","message":"yes we shoudl keep them seperate.\n\nlibvirt report them septeratly\n\nhttps://github.com/libvirt/libvirt/blob/76be4f5ddac608873378e5bc43eb12731f7ddcf2/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml#L44-L66\n\nso we shoudl track them seperatly here too.\n\nif the COS is a shared resouce that means that libvirt is incorrectly reporting this.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"3a5815da9d09ea02d34b99bde6ebe395d4b0b4b2","unresolved":false,"context_lines":[{"line_number":237,"context_line":"be added to model the number of \"Class of Service (COS)\" or \"Quality of Service"},{"line_number":238,"context_line":"(QoS)\" policies supported by the CPU."},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"- ``CACHE_COS``"},{"line_number":241,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":244,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_79d0c551","line":241,"range":{"start_line":240,"start_character":0,"end_line":241,"end_character":26},"in_reply_to":"bfb3d3c7_97eb2011","updated":"2019-06-03 07:37:55.000000000","message":"After reading the SDM and the libvirt/kernel implementation. \nNow I agree that we should keep these two CLOSes.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":258,"context_line":"to request cache allocation and memory bandwidth limits."},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"``hw:numa_cache``"},{"line_number":261,"context_line":"    Controls allocation of L3 unified cache ways. There is a minimum"},{"line_number":262,"context_line":"    allocation of two cacheways per cache region/NUMA node with a granularity"},{"line_number":263,"context_line":"    of one cache way."},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"``hw:numa_code_cache``"},{"line_number":266,"context_line":"    Controls allocation of L3 code cache ways. There is a minimum allocation of"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_dcaaff17","line":263,"range":{"start_line":261,"start_character":50,"end_line":263,"end_character":21},"updated":"2019-06-03 08:49:05.000000000","message":"As above we can reword this to use the output from Libvirt.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6e0f035b8a625734d3e3d9c5555dc2ddf20d3cd7","unresolved":false,"context_lines":[{"line_number":258,"context_line":"to request cache allocation and memory bandwidth limits."},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"``hw:numa_cache``"},{"line_number":261,"context_line":"    Controls allocation of L3 unified cache ways. There is a minimum"},{"line_number":262,"context_line":"    allocation of two cacheways per cache region/NUMA node with a granularity"},{"line_number":263,"context_line":"    of one cache way."},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"``hw:numa_code_cache``"},{"line_number":266,"context_line":"    Controls allocation of L3 code cache ways. There is a minimum allocation of"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_5efd7e38","line":263,"range":{"start_line":261,"start_character":50,"end_line":263,"end_character":21},"in_reply_to":"9fb8cfa7_dcaaff17","updated":"2019-06-17 09:33:50.000000000","message":"im just going to remove this","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":263,"context_line":"    of one cache way."},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"``hw:numa_code_cache``"},{"line_number":266,"context_line":"    Controls allocation of L3 code cache ways. There is a minimum allocation of"},{"line_number":267,"context_line":"    two cacheways per cache region/NUMA node with a granularity of one cache"},{"line_number":268,"context_line":"    way. This may only be set if ``hw:numa_cache`` is not set."},{"line_number":269,"context_line":""},{"line_number":270,"context_line":"``hw:numa_data_cache``"},{"line_number":271,"context_line":"    Controls allocation of L3 data cache ways. There is a minimum allocation of"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_bc670b26","line":268,"range":{"start_line":266,"start_character":47,"end_line":268,"end_character":8},"updated":"2019-06-03 08:49:05.000000000","message":"As above.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":268,"context_line":"    way. This may only be set if ``hw:numa_cache`` is not set."},{"line_number":269,"context_line":""},{"line_number":270,"context_line":"``hw:numa_data_cache``"},{"line_number":271,"context_line":"    Controls allocation of L3 data cache ways. There is a minimum allocation of"},{"line_number":272,"context_line":"    two cacheways per cache region/NUMA node with a granularity of one cache"},{"line_number":273,"context_line":"    way.This may only be set if ``hw:numa_cache`` is not set."},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"``hw:numa_bandwidth``"},{"line_number":276,"context_line":"    Specifies the maximum memory bandwidth an instance can consume per NUMA"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_7c7193e0","line":273,"range":{"start_line":271,"start_character":47,"end_line":273,"end_character":8},"updated":"2019-06-03 08:49:05.000000000","message":"As above.","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":331,"context_line":"    capability trait and cache resouce classes. This config option controls"},{"line_number":332,"context_line":"    generation of cache allocation element in the domain XML. This value should"},{"line_number":333,"context_line":"    be set to false on hosts that will not be used for cache confined guest or"},{"line_number":334,"context_line":"    when cache allocation is enforce by an external agent."},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"``[libvirt] memory_bandwidth_limiting``"},{"line_number":337,"context_line":"  :Type: Boolean"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_bc506b34","line":334,"range":{"start_line":334,"start_character":29,"end_line":334,"end_character":36},"updated":"2019-06-03 08:49:05.000000000","message":"enforced","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6e0f035b8a625734d3e3d9c5555dc2ddf20d3cd7","unresolved":false,"context_lines":[{"line_number":331,"context_line":"    capability trait and cache resouce classes. This config option controls"},{"line_number":332,"context_line":"    generation of cache allocation element in the domain XML. This value should"},{"line_number":333,"context_line":"    be set to false on hosts that will not be used for cache confined guest or"},{"line_number":334,"context_line":"    when cache allocation is enforce by an external agent."},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"``[libvirt] memory_bandwidth_limiting``"},{"line_number":337,"context_line":"  :Type: Boolean"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_be003a36","line":334,"range":{"start_line":334,"start_character":29,"end_line":334,"end_character":36},"in_reply_to":"9fb8cfa7_bc506b34","updated":"2019-06-17 09:33:50.000000000","message":"Done","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"e9a3d10287abf73c32c390b09fe080a61f8abe02","unresolved":false,"context_lines":[{"line_number":418,"context_line":"- 4.1.0 for the ``cachetune`` element"},{"line_number":419,"context_line":"- 4.7.0 for the ``memorytune`` element"},{"line_number":420,"context_line":""},{"line_number":421,"context_line":"kernel"},{"line_number":422,"context_line":"- 4.10 for Intel CAT"},{"line_number":423,"context_line":"- 4.18 for Intel MBA"},{"line_number":424,"context_line":"- 4.21 for AMD"},{"line_number":425,"context_line":""},{"line_number":426,"context_line":"Live migration support depends on"},{"line_number":427,"context_line":"https://specs.openstack.org/openstack/nova-specs/specs/train/approved/numa-aware-live-migration.html"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_9c19e789","line":424,"range":{"start_line":421,"start_character":0,"end_line":424,"end_character":14},"updated":"2019-06-03 08:49:05.000000000","message":"I assume Libvirt would let us know if these dependencies aren\u0027t met on a compute host?","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6e0f035b8a625734d3e3d9c5555dc2ddf20d3cd7","unresolved":false,"context_lines":[{"line_number":418,"context_line":"- 4.1.0 for the ``cachetune`` element"},{"line_number":419,"context_line":"- 4.7.0 for the ``memorytune`` element"},{"line_number":420,"context_line":""},{"line_number":421,"context_line":"kernel"},{"line_number":422,"context_line":"- 4.10 for Intel CAT"},{"line_number":423,"context_line":"- 4.18 for Intel MBA"},{"line_number":424,"context_line":"- 4.21 for AMD"},{"line_number":425,"context_line":""},{"line_number":426,"context_line":"Live migration support depends on"},{"line_number":427,"context_line":"https://specs.openstack.org/openstack/nova-specs/specs/train/approved/numa-aware-live-migration.html"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9fb8cfa7_7e7de2af","line":424,"range":{"start_line":421,"start_character":0,"end_line":424,"end_character":14},"in_reply_to":"9fb8cfa7_9c19e789","updated":"2019-06-17 09:33:50.000000000","message":"libvirt supports detecting the amount of cache in older version but effectively if the control element is not present\nthen the dependencies are not met.\n\n    \u003ccache\u003e\n        \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002733\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270\u0027\u003e\n            \u003ccontrol granularity\u003d\u00273\u0027 unit\u003d\u0027MiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u002716\u0027/\u003e\n        \u003c/bank\u003e\n    \u003c/cache\u003e\n\n\nif code and data priortisation are enabled then teh contol element will be  of type data and code\n\n    \u003ccache\u003e\n        \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270-5\u0027\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027code\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027data\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n        \u003c/bank\u003e\n        \u003cbank id\u003d\u00271\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00276-11\u0027\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027code\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027data\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n        \u003c/bank\u003e\n    \u003c/cache\u003e","commit_id":"35adfd8722070a56d1c1541afd94e6d145566b0e"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":26,"context_line":"Realtime workloads rely on deterministic cache latency, memory access time and"},{"line_number":27,"context_line":"bandwidth to fulfill their SLAs. Noisy neighbor effects from non-realtime"},{"line_number":28,"context_line":"workloads co-located on the same host can result in violation of realtime"},{"line_number":29,"context_line":"workloads SLAs, which can be mitigated via static allocation of cache and"},{"line_number":30,"context_line":"memory bandwidth to instances."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_97b4f483","line":29,"range":{"start_line":29,"start_character":0,"end_line":29,"end_character":9},"updated":"2019-06-24 18:33:01.000000000","message":"workloads\u0027 (plural possessive)","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":26,"context_line":"Realtime workloads rely on deterministic cache latency, memory access time and"},{"line_number":27,"context_line":"bandwidth to fulfill their SLAs. Noisy neighbor effects from non-realtime"},{"line_number":28,"context_line":"workloads co-located on the same host can result in violation of realtime"},{"line_number":29,"context_line":"workloads SLAs, which can be mitigated via static allocation of cache and"},{"line_number":30,"context_line":"memory bandwidth to instances."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7d010d06","line":29,"range":{"start_line":29,"start_character":0,"end_line":29,"end_character":9},"in_reply_to":"9fb8cfa7_97b4f483","updated":"2019-06-25 21:26:26.000000000","message":"or i can make it singular","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"30666f1382d9c8c48ffdac703d00f0b7bc210f7b","unresolved":false,"context_lines":[{"line_number":32,"context_line":"Use Cases"},{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As an operator, I want to be able to deploy a realtime workload on a host"},{"line_number":36,"context_line":"with other realtime and non-realtime workloads and achieve a deterministic"},{"line_number":37,"context_line":"level of performance."},{"line_number":38,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_682e2893","line":35,"range":{"start_line":35,"start_character":69,"end_line":35,"end_character":73},"updated":"2019-06-25 21:32:19.000000000","message":"i might change this use case to not refer to the same host.\ni would like to be able to definitely support this cache confined and non cache confined instance in the same cloud without the need for host aggregates or special filter configuration.\n\nsupporting it on the same host is possible but rely on behaviour in libvirt that I\u0027m not sure i want to rely on.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As an operator, I want to be able to deploy a realtime workload on a host"},{"line_number":36,"context_line":"with other realtime and non-realtime workloads and achieve a deterministic"},{"line_number":37,"context_line":"level of performance."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"As an operator with a datacenter containing multiple generations of hardware,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_378c28d6","line":36,"range":{"start_line":36,"start_character":61,"end_line":36,"end_character":74},"updated":"2019-06-24 18:33:01.000000000","message":"Well... \"predictable\" might be a better word choice.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As an operator, I want to be able to deploy a realtime workload on a host"},{"line_number":36,"context_line":"with other realtime and non-realtime workloads and achieve a deterministic"},{"line_number":37,"context_line":"level of performance."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"As an operator with a datacenter containing multiple generations of hardware,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_5819b777","line":36,"range":{"start_line":36,"start_character":61,"end_line":36,"end_character":74},"in_reply_to":"9fb8cfa7_378c28d6","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":".. note::"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    This set of features are referred to as RDT or resource directory"},{"line_number":60,"context_line":"    technology by Intel, the generic vendor-independent name is PQoS (Platform"},{"line_number":61,"context_line":"    Quality of Service). The Zen 2-based AMD Epyc CPUs also support PQoS as of"},{"line_number":62,"context_line":"    kernel 4.21. See https://lkml.org/lkml/2018/12/23/69"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_b7b9f879","line":59,"range":{"start_line":59,"start_character":60,"end_line":59,"end_character":69},"updated":"2019-06-24 18:33:01.000000000","message":"director","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":".. note::"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    This set of features are referred to as RDT or resource directory"},{"line_number":60,"context_line":"    technology by Intel, the generic vendor-independent name is PQoS (Platform"},{"line_number":61,"context_line":"    Quality of Service). The Zen 2-based AMD Epyc CPUs also support PQoS as of"},{"line_number":62,"context_line":"    kernel 4.21. See https://lkml.org/lkml/2018/12/23/69"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_382e4392","line":59,"range":{"start_line":59,"start_character":60,"end_line":59,"end_character":69},"in_reply_to":"9fb8cfa7_b7b9f879","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":62,"context_line":"    kernel 4.21. See https://lkml.org/lkml/2018/12/23/69"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"This spec proposes extending the libvirt virt driver to support static"},{"line_number":65,"context_line":"allocation of cacheways and enforcement of maximum memory bandwidth limits."},{"line_number":66,"context_line":"Delegation of discovery and enforcement to an external agent (RMD, CYBORG, cron"},{"line_number":67,"context_line":"job) will also be possible. This will be achieved at a high level by"},{"line_number":68,"context_line":"introducing five new standard resource classes: two for the class of service"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_57b2fc95","line":65,"range":{"start_line":65,"start_character":14,"end_line":65,"end_character":23},"updated":"2019-06-24 18:33:01.000000000","message":"\"cacheways\" needs to be defined somewhere in this document.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":65,"context_line":"allocation of cacheways and enforcement of maximum memory bandwidth limits."},{"line_number":66,"context_line":"Delegation of discovery and enforcement to an external agent (RMD, CYBORG, cron"},{"line_number":67,"context_line":"job) will also be possible. This will be achieved at a high level by"},{"line_number":68,"context_line":"introducing five new standard resource classes: two for the class of service"},{"line_number":69,"context_line":"types and three for L3 cache, to model the capacity of each resource in"},{"line_number":70,"context_line":"placement. As memory bandwidth limits are a max limit not a min limit no"},{"line_number":71,"context_line":"bandwidth is actually allocated to an instance as such memory bandwidth will"},{"line_number":72,"context_line":"not be tracked as a consumable resource in placement and therefore no resource"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_17a8041e","line":69,"range":{"start_line":68,"start_character":60,"end_line":69,"end_character":28},"updated":"2019-06-24 18:33:01.000000000","message":"eh?\n\n[Later] Okay, this depends on definition provided later for \"class of service\"","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Three new config options will be added to the libvirt virt driver, two to"},{"line_number":77,"context_line":"enable support for memory limits and cache allocation, and one to declare the"},{"line_number":78,"context_line":"reserved NUMA of cacheways for the host per cache region/NUMA node."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":".. code::"},{"line_number":81,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_976954f9","line":78,"range":{"start_line":78,"start_character":9,"end_line":78,"end_character":13},"updated":"2019-06-24 18:33:01.000000000","message":"number?","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Three new config options will be added to the libvirt virt driver, two to"},{"line_number":77,"context_line":"enable support for memory limits and cache allocation, and one to declare the"},{"line_number":78,"context_line":"reserved NUMA of cacheways for the host per cache region/NUMA node."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":".. code::"},{"line_number":81,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_4b16d54b","line":78,"range":{"start_line":78,"start_character":9,"end_line":78,"end_character":13},"in_reply_to":"9fb8cfa7_976954f9","updated":"2019-06-25 21:26:26.000000000","message":"yes","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"923269cfa780c894dba4017a6e1081e3ae52d5a9","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  [libvirt]"},{"line_number":83,"context_line":"  cache_allocation_support\u003dtrue  # true|false default(false)"},{"line_number":84,"context_line":"  memory_bandwidth_limiting\u003dtrue  # true|false default(false)"},{"line_number":85,"context_line":"  reserved_cacheways\u003d2,0  #reserved cacheways per NUMA node"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"The detail of the config parameters are specified in the \"Other deployer"},{"line_number":88,"context_line":"impact\" section of this spec."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_e1c3aa89","line":85,"range":{"start_line":85,"start_character":23,"end_line":85,"end_character":24},"updated":"2019-06-17 09:18:09.000000000","message":"\u00270\u0027 is not a valid value for this case.\nMy understanding of \u0027reserved_cacheways\u0027 is shared by all other Linux processes including kernel. \u00270\u0027 means no cacheway  is reserved for a \u0027resctrl\u0027 allocation specific for non-VM processes. You should at least reserve one cache-way per node for non-VM processes.\n\nTests:\n\n[root@c200 resctrl]# echo \"L3:0\u003d0000f;1\u003d00001\" \u003e schemata #\u003c- OK\n[root@c200 resctrl]# echo \"L3:0\u003d0000f;1\u003d00000\" \u003e schemata #\u003c- Failed\nbash: echo: write error: Invalid argument","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"dadcb05c6f2f8f0f4989b5f9bfc039ac39eb1ea9","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  [libvirt]"},{"line_number":83,"context_line":"  cache_allocation_support\u003dtrue  # true|false default(false)"},{"line_number":84,"context_line":"  memory_bandwidth_limiting\u003dtrue  # true|false default(false)"},{"line_number":85,"context_line":"  reserved_cacheways\u003d2,0  #reserved cacheways per NUMA node"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"The detail of the config parameters are specified in the \"Other deployer"},{"line_number":88,"context_line":"impact\" section of this spec."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_015b5e54","line":85,"range":{"start_line":85,"start_character":23,"end_line":85,"end_character":24},"in_reply_to":"9fb8cfa7_8198ae1b","updated":"2019-06-19 23:05:50.000000000","message":"OK","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6e0f035b8a625734d3e3d9c5555dc2ddf20d3cd7","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  [libvirt]"},{"line_number":83,"context_line":"  cache_allocation_support\u003dtrue  # true|false default(false)"},{"line_number":84,"context_line":"  memory_bandwidth_limiting\u003dtrue  # true|false default(false)"},{"line_number":85,"context_line":"  reserved_cacheways\u003d2,0  #reserved cacheways per NUMA node"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"The detail of the config parameters are specified in the \"Other deployer"},{"line_number":88,"context_line":"impact\" section of this spec."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_8198ae1b","line":85,"range":{"start_line":85,"start_character":23,"end_line":85,"end_character":24},"in_reply_to":"9fb8cfa7_e1c3aa89","updated":"2019-06-17 09:33:50.000000000","message":"0 is a valid value if i have reserved cacheways on onther numa nodes for the kernel to use.\n\nif i am using the isolcpu kernel option or i use systemd to partition the cpus that OS servcies can run at i can ensure that kenels and userspace process that are not vms never get schduled to as specific numa node/cpu socket.\n\nsetting reserved_cacheways will not program the reservation via resctrl so even if all 0 is not valid to program it will not be an issue. \n\nyou are right that in general you should probaly reserve 1 cacheway per numa node for the host but only if you alos resevce at least 1 core per numa node for the host.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":90,"context_line":"Two new compute capability traits will be added to indicate if a host supports"},{"line_number":91,"context_line":"cache allocation or memory bandwidth limits based on the values in the config"},{"line_number":92,"context_line":"options. These new traits will be used in combination with a new request"},{"line_number":93,"context_line":"prefilter to ensure that we do not mix cache allocated instances and non cache"},{"line_number":94,"context_line":"allocated instances on the same host. similarly the same will be done for"},{"line_number":95,"context_line":"memory limits."},{"line_number":96,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_b715784b","line":93,"range":{"start_line":93,"start_character":28,"end_line":93,"end_character":38},"updated":"2019-06-24 18:33:01.000000000","message":"waitwait, L36 implies you want to be able to mix","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":90,"context_line":"Two new compute capability traits will be added to indicate if a host supports"},{"line_number":91,"context_line":"cache allocation or memory bandwidth limits based on the values in the config"},{"line_number":92,"context_line":"options. These new traits will be used in combination with a new request"},{"line_number":93,"context_line":"prefilter to ensure that we do not mix cache allocated instances and non cache"},{"line_number":94,"context_line":"allocated instances on the same host. similarly the same will be done for"},{"line_number":95,"context_line":"memory limits."},{"line_number":96,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_ebd2295c","line":93,"range":{"start_line":93,"start_character":28,"end_line":93,"end_character":38},"in_reply_to":"9fb8cfa7_b715784b","updated":"2019-06-25 21:26:26.000000000","message":"technically we could support mixing but not with the api provided by libvirt. i initially planned to support passing a\nmask of what cache-ways can be used for dedicated cache and second for shared cache similar to what we will be doing for cpu pinning however libvit does not allow use to chose the cache allocations. \n\nwe can still support this use case if we choose as any instance without explicit cache allocation will use the default system cache-ways. \n\nim in two minds if we should allow this as it assumes you have correctly configured the system cache allocation.\n\nsince that is technically required for this feature to work\n(libvirt assume the cache not in use by it cgroup is available for use by vms.) \ni guess its safe to assume that and i can remove this statement.\n\ni was starting to be more conservative when i found libvirt does not provide the level of control i wanted.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":98,"context_line":"driver will be extended to discover the capacity and availability of both"},{"line_number":99,"context_line":"resources by parsing the XML returned by the libvirt capabilities API. The"},{"line_number":100,"context_line":"libvirt driver will also be extended to support generating the domain XML"},{"line_number":101,"context_line":"element to instruct libvirt to allocate cache and specify memory bandwidth"},{"line_number":102,"context_line":"limits via the ``cachetune`` and ``memorytune`` sub-elements of the ``cputune``"},{"line_number":103,"context_line":"element."},{"line_number":104,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9a1e595d","line":101,"range":{"start_line":101,"start_character":31,"end_line":101,"end_character":39},"updated":"2019-06-24 18:33:01.000000000","message":"Let\u0027s use \"assign\" here to disambiguate from what Placement does.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":98,"context_line":"driver will be extended to discover the capacity and availability of both"},{"line_number":99,"context_line":"resources by parsing the XML returned by the libvirt capabilities API. The"},{"line_number":100,"context_line":"libvirt driver will also be extended to support generating the domain XML"},{"line_number":101,"context_line":"element to instruct libvirt to allocate cache and specify memory bandwidth"},{"line_number":102,"context_line":"limits via the ``cachetune`` and ``memorytune`` sub-elements of the ``cputune``"},{"line_number":103,"context_line":"element."},{"line_number":104,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_58045718","line":101,"range":{"start_line":101,"start_character":31,"end_line":101,"end_character":39},"in_reply_to":"9fb8cfa7_9a1e595d","updated":"2019-06-25 21:26:26.000000000","message":"i will reluctantly change this. we usually reserve assign for when we are assign a specific resource such as a specific cpu or pci device to an instance. in this case we are not assigning a cache way as we do not have that level of control and in stead we are allocating a reservation form that amount of cacheways available.\n\ni was trying to disambiguate between instructing libvirt to  allocate cacheways vs direct assignment of specific cacheways to an instance but i am ok with changing to disambiuguate between nova and placement.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"To support external management of cache and memory bandwidth, the libvirt"},{"line_number":106,"context_line":"driver ``update_provider_tree`` method will be modified to only report cache"},{"line_number":107,"context_line":"and memory bandwidth inventories if enable the config."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":".. note ::"},{"line_number":110,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_1a3269f3","line":107,"range":{"start_line":107,"start_character":36,"end_line":107,"end_character":42},"updated":"2019-06-24 18:33:01.000000000","message":"enabled in","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":119,"context_line":"These new extra spec will also support the dot notation supported by the other"},{"line_number":120,"context_line":"``hw:numa_*`` extra specs to allow specifying cache and memory bandwidth per"},{"line_number":121,"context_line":"virtual NUMA node. For example, consider a guest with two virtual NUMA nodes,"},{"line_number":122,"context_line":"two cache ways per NUMA node and a limit per NUMA node of 10% of the host NUMA"},{"line_number":123,"context_line":"nodes bandwidth ::"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"    hw:numa_nodes\u003d2"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_5a6a21b3","line":122,"range":{"start_line":122,"start_character":41,"end_line":122,"end_character":61},"updated":"2019-06-24 18:33:01.000000000","message":"e...","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":123,"context_line":"nodes bandwidth ::"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":126,"context_line":"    hw:numa_cache\u003d4"},{"line_number":127,"context_line":"    hw:numa_bandwidth\u003d20"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Alternatively, consider a guest with two virtual NUMA nodes, with unbalanced"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7aacc5ec","line":126,"range":{"start_line":126,"start_character":12,"end_line":126,"end_character":17},"updated":"2019-06-24 18:33:01.000000000","message":"this should probably be spelled cacheways","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":123,"context_line":"nodes bandwidth ::"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":126,"context_line":"    hw:numa_cache\u003d4"},{"line_number":127,"context_line":"    hw:numa_bandwidth\u003d20"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Alternatively, consider a guest with two virtual NUMA nodes, with unbalanced"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_527af3f0","line":126,"range":{"start_line":126,"start_character":12,"end_line":126,"end_character":17},"in_reply_to":"9fb8cfa7_7aacc5ec","updated":"2019-06-25 21:26:26.000000000","message":"i am okk with doing that for the resurce class name im not conviced it a go desgin for the extra spec unless you expeact to support somthing like hw:numa_cache_mb at some point in the future.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":124,"context_line":""},{"line_number":125,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":126,"context_line":"    hw:numa_cache\u003d4"},{"line_number":127,"context_line":"    hw:numa_bandwidth\u003d20"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Alternatively, consider a guest with two virtual NUMA nodes, with unbalanced"},{"line_number":130,"context_line":"CPUs, cache and memory::"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9a7ad974","line":127,"range":{"start_line":127,"start_character":22,"end_line":127,"end_character":24},"updated":"2019-06-24 18:33:01.000000000","message":"...w\n\nImplicit division of a *percentage* feels really weird to me. I get that we implicitly divide discrete units evenly, but is there a precedent for what we do with percentages?\n\nI\u0027m saying, IMO whatever number is specified here should be the percentage we take out of *each* NUMA node. So if you want 10% each, you put 10 here.\n\nAlso, it should be made clear that we\u0027re talking about *memory* bandwidth here (\"hw:numa_mem_bandwidth\")\n\nAlso, the units should be manifested somewhere, either in the value (\"20%\") or the key (\"hw:numa_mem_bandwidth_percent\")","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":124,"context_line":""},{"line_number":125,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":126,"context_line":"    hw:numa_cache\u003d4"},{"line_number":127,"context_line":"    hw:numa_bandwidth\u003d20"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Alternatively, consider a guest with two virtual NUMA nodes, with unbalanced"},{"line_number":130,"context_line":"CPUs, cache and memory::"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_803cac2f","line":127,"range":{"start_line":127,"start_character":22,"end_line":127,"end_character":24},"in_reply_to":"9fb8cfa7_9a7ad974","updated":"2019-06-25 21:26:26.000000000","message":"ya i had that in the first version and it was pointed out it was not the same as the other resource where we implicitly divide.\n\nim happy to go back to not dividing because initially i felt it was more intuitive too.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":155,"context_line":"    and ``hw:numa_data_cache`` will be used instead of ``hw:numa_cache``. While"},{"line_number":156,"context_line":"    code and data partitioning (CDP) can be enabled at runtime, The state of"},{"line_number":157,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":158,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":159,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":".. note::"},{"line_number":162,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7ac3658a","line":159,"range":{"start_line":158,"start_character":52,"end_line":159,"end_character":78},"updated":"2019-06-24 18:33:01.000000000","message":"nts: come back to this\n\n[Later] So yeah, it\u0027s implied (though never made clear) that compute will report the cache as *either* a unified cache (``L3_CACHE`` is spelled ``L3_UNIFIED_CACHE`` later in the document) or as ``L3_DATA_CACHE`` plus ``L3_CODE_CACHE``. I think this is kind of broken, because it means the flavor has to request its cache in one form or the other; and then will only be able to land on a host that happens to be exposing it in the same fashion.\n\nI\u0027m not sure what the right answer is here. Are hosts that are capable of partitioning also capable of *not* partitioning? If so, the simplest thing to start with might be to simply expose unified cache for everyone.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":155,"context_line":"    and ``hw:numa_data_cache`` will be used instead of ``hw:numa_cache``. While"},{"line_number":156,"context_line":"    code and data partitioning (CDP) can be enabled at runtime, The state of"},{"line_number":157,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":158,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":159,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":".. note::"},{"line_number":162,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_0b053d11","line":159,"range":{"start_line":158,"start_character":52,"end_line":159,"end_character":78},"in_reply_to":"9fb8cfa7_7ac3658a","updated":"2019-06-25 21:26:26.000000000","message":"host that are capably of code and data priortisation/partionioning can be configured either way.\nnot all host that are capable of CAT suppor CDP.\n\nwhen configured for CDP while you can request just \nL3_DATA_CACHE or just L3_CODE_CACHE cache  or you can request both however that required specifying the cache can be used expcitly for code and cache in two bit mask intest of just one.\n\nat the hardware level when cdp is enabled the bit mast form two COS are combied to allow a seperate policy to be specifed for code or data. so requesting l3 unified cache allocation on a host with CDP enabled should be an error.\n\nyou are right that by having  L3_UNIFIED_CACHE and have (``L3_DATA_CACHE`` plus ``L3_CODE_CACHE``) we will need to partition our host and have two different flavours for CDP enabled or CDP disabled. unless we deicide to declare CDP out of scope and not support it initially i think we have to treat them separately and have different resource classes and flavours.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":196,"context_line":"Data model impact"},{"line_number":197,"context_line":"-----------------"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"The libvirt driver will be extended to create one nested RP per l3 cache"},{"line_number":200,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":201,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":202,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":203,"context_line":"Service (CoS) policies which will be allocated to instnaces."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_1aed89ef","line":200,"range":{"start_line":199,"start_character":0,"end_line":200,"end_character":7},"updated":"2019-06-24 18:33:01.000000000","message":"How does this help us (versus splitting per NUMA node) if we have no way of ensuring which package the cpu and memory come from?\n\nEven splitting per NUMA node makes zero sense until we have proper NUMA modeling of cpu/memory.\n\nWe should not try to \"anticipate\" NUMA modeling piecemeal like this.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":196,"context_line":"Data model impact"},{"line_number":197,"context_line":"-----------------"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"The libvirt driver will be extended to create one nested RP per l3 cache"},{"line_number":200,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":201,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":202,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":203,"context_line":"Service (CoS) policies which will be allocated to instnaces."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_c3f7fc8a","line":200,"range":{"start_line":199,"start_character":0,"end_line":200,"end_character":7},"in_reply_to":"9fb8cfa7_1aed89ef","updated":"2019-06-25 21:26:26.000000000","message":"this is not anticipating numa modelling.\n\nnuma support is required for this feature to proceed but numa suport in nova does not require numa support in placement. since i am requiring cpu pinning for this feature it means that all instance will have a numa topology.\n\nin the libvirt xml we map a cache policy to specific virtual cpus which are pinned to a specific cache regions. \nlibvirt does not allow us to specify the mask within that region but we can spcify the amount of cache within a region to allocate.\n\nWhen we define an instance domain we can map specific set of guest cpu to a host cache region.\n\nfor example for a vm with 4 cores and 2 numa nodes we can map vm core 0 and 1 to host cache region 0 and vm core 2 and 3 to cache region 1.\n\n  \u003ccachetune vcpus\u003d\u00270-1\u0027\u003e\n      \u003ccache id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u00273\u0027 unit\u003d\u0027MiB\u0027/\u003e\n    \u003c/cachetune\u003e\n\n    \u003ccachetune vcpus\u003d\u00272-3\u0027\u003e \n\u003ccache id\u003d\u00271\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u00273\u0027 unit\u003d\u0027MiB\u0027/\u003e\n    \u003c/cachetune\u003e\n\nfrom the virsh capablity infomation libvirt  will report the invetorys of the caceh per cache region and the mapping of each cache region to the phyical cores on the host\n\n   \u003ccache\u003e\n        \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270-5\u0027\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027code\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027data\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n        \u003c/bank\u003e\n        \u003cbank id\u003d\u00271\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002715\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00276-11\u0027\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027code\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n            \u003ccontrol granularity\u003d\u0027768\u0027 unit\u003d\u0027KiB\u0027 type\u003d\u0027data\u0027 maxAllocs\u003d\u00278\u0027/\u003e\n        \u003c/bank\u003e\n    \u003c/cache\u003e\n\n\nso since we are pinning the cores and all pinned instance are numa affiend we can ensure the cache region we allocate from is affinites tot he same numa node as the cpus we pin too.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":200,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":201,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":202,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":203,"context_line":"Service (CoS) policies which will be allocated to instnaces."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"On all current intel hardware that supports CAT there is a 1:1 mapping between"},{"line_number":206,"context_line":"cache region per NUMA node for L3 cache."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_fa0af5cf","line":203,"range":{"start_line":203,"start_character":50,"end_line":203,"end_character":59},"updated":"2019-06-24 18:33:01.000000000","message":"instances","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":200,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":201,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":202,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":203,"context_line":"Service (CoS) policies which will be allocated to instnaces."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"On all current intel hardware that supports CAT there is a 1:1 mapping between"},{"line_number":206,"context_line":"cache region per NUMA node for L3 cache."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9e9877c0","line":203,"range":{"start_line":203,"start_character":50,"end_line":203,"end_character":59},"in_reply_to":"9fb8cfa7_fa0af5cf","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":207,"context_line":""},{"line_number":208,"context_line":".. note::"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    A L3 cache region corresponds to a phyical cache segemt identifed by the"},{"line_number":211,"context_line":"    cache id ``cat /sys/bus/cpu/devices/cpu*/cache/index3/id``. On older cpus"},{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_faa675c1","line":210,"range":{"start_line":210,"start_character":39,"end_line":210,"end_character":46},"updated":"2019-06-24 18:33:01.000000000","message":"physical","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":207,"context_line":""},{"line_number":208,"context_line":".. note::"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    A L3 cache region corresponds to a phyical cache segemt identifed by the"},{"line_number":211,"context_line":"    cache id ``cat /sys/bus/cpu/devices/cpu*/cache/index3/id``. On older cpus"},{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_ba10fd01","line":210,"range":{"start_line":210,"start_character":53,"end_line":210,"end_character":59},"updated":"2019-06-24 18:33:01.000000000","message":"segment","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":207,"context_line":""},{"line_number":208,"context_line":".. note::"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    A L3 cache region corresponds to a phyical cache segemt identifed by the"},{"line_number":211,"context_line":"    cache id ``cat /sys/bus/cpu/devices/cpu*/cache/index3/id``. On older cpus"},{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_5e8e7f79","line":210,"range":{"start_line":210,"start_character":53,"end_line":210,"end_character":59},"in_reply_to":"9fb8cfa7_ba10fd01","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":207,"context_line":""},{"line_number":208,"context_line":".. note::"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    A L3 cache region corresponds to a phyical cache segemt identifed by the"},{"line_number":211,"context_line":"    cache id ``cat /sys/bus/cpu/devices/cpu*/cache/index3/id``. On older cpus"},{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7e8b4367","line":210,"range":{"start_line":210,"start_character":39,"end_line":210,"end_character":46},"in_reply_to":"9fb8cfa7_faa675c1","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    A L3 cache region corresponds to a phyical cache segemt identifed by the"},{"line_number":211,"context_line":"    cache id ``cat /sys/bus/cpu/devices/cpu*/cache/index3/id``. On older cpus"},{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"},{"line_number":214,"context_line":"    per physical cpu package depends on if Cluster on Die (CoD) is enabled."},{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7af605da","line":212,"range":{"start_line":212,"start_character":10,"end_line":212,"end_character":17},"updated":"2019-06-24 18:33:01.000000000","message":"common?","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    A L3 cache region corresponds to a phyical cache segemt identifed by the"},{"line_number":211,"context_line":"    cache id ``cat /sys/bus/cpu/devices/cpu*/cache/index3/id``. On older cpus"},{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"},{"line_number":214,"context_line":"    per physical cpu package depends on if Cluster on Die (CoD) is enabled."},{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9aa739bd","line":212,"range":{"start_line":212,"start_character":55,"end_line":212,"end_character":62},"updated":"2019-06-24 18:33:01.000000000","message":"physical","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":209,"context_line":""},{"line_number":210,"context_line":"    A L3 cache region corresponds to a phyical cache segemt identifed by the"},{"line_number":211,"context_line":"    cache id ``cat /sys/bus/cpu/devices/cpu*/cache/index3/id``. On older cpus"},{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"},{"line_number":214,"context_line":"    per physical cpu package depends on if Cluster on Die (CoD) is enabled."},{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9eddd771","line":212,"range":{"start_line":212,"start_character":10,"end_line":212,"end_character":17},"in_reply_to":"9fb8cfa7_7af605da","updated":"2019-06-25 21:26:26.000000000","message":"yes :)\nthis is why happens when i use a spell checker to fix some things","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"},{"line_number":214,"context_line":"    per physical cpu package depends on if Cluster on Die (CoD) is enabled."},{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"},{"line_number":216,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":217,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":218,"context_line":"    partition into 1 numa node and 1 cache region per memory controler. As"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_da2e1142","line":215,"range":{"start_line":215,"start_character":43,"end_line":215,"end_character":50},"updated":"2019-06-24 18:33:01.000000000","message":"physical","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":212,"context_line":"    it is comment to have a single l3 cache region per phyical package."},{"line_number":213,"context_line":"    On newer generation of Intel CPUs the number of L3 cache regions reported"},{"line_number":214,"context_line":"    per physical cpu package depends on if Cluster on Die (CoD) is enabled."},{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"},{"line_number":216,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":217,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":218,"context_line":"    partition into 1 numa node and 1 cache region per memory controler. As"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_beda1b75","line":215,"range":{"start_line":215,"start_character":43,"end_line":215,"end_character":50},"in_reply_to":"9fb8cfa7_da2e1142","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"},{"line_number":216,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":217,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":218,"context_line":"    partition into 1 numa node and 1 cache region per memory controler. As"},{"line_number":219,"context_line":"    the 1:1 mapping between cache region and numa node may change in the"},{"line_number":220,"context_line":"    future with new hardware, cache inventories will be tracked per cache"},{"line_number":221,"context_line":"    region."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_5a222154","line":218,"range":{"start_line":218,"start_character":61,"end_line":218,"end_character":70},"updated":"2019-06-24 18:33:01.000000000","message":"controller","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"},{"line_number":216,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":217,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":218,"context_line":"    partition into 1 numa node and 1 cache region per memory controler. As"},{"line_number":219,"context_line":"    the 1:1 mapping between cache region and numa node may change in the"},{"line_number":220,"context_line":"    future with new hardware, cache inventories will be tracked per cache"},{"line_number":221,"context_line":"    region."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_ba29dd37","line":218,"range":{"start_line":218,"start_character":4,"end_line":218,"end_character":13},"updated":"2019-06-24 18:33:01.000000000","message":"partitioned","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"},{"line_number":216,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":217,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":218,"context_line":"    partition into 1 numa node and 1 cache region per memory controler. As"},{"line_number":219,"context_line":"    the 1:1 mapping between cache region and numa node may change in the"},{"line_number":220,"context_line":"    future with new hardware, cache inventories will be tracked per cache"},{"line_number":221,"context_line":"    region."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_52de7311","line":218,"range":{"start_line":218,"start_character":61,"end_line":218,"end_character":70},"in_reply_to":"9fb8cfa7_5a222154","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":215,"context_line":"    When CoD is disabled the cpu cache and phyical memory connected to a cpu"},{"line_number":216,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":217,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":218,"context_line":"    partition into 1 numa node and 1 cache region per memory controler. As"},{"line_number":219,"context_line":"    the 1:1 mapping between cache region and numa node may change in the"},{"line_number":220,"context_line":"    future with new hardware, cache inventories will be tracked per cache"},{"line_number":221,"context_line":"    region."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_72e377da","line":218,"range":{"start_line":218,"start_character":4,"end_line":218,"end_character":13},"in_reply_to":"9fb8cfa7_ba29dd37","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":221,"context_line":"    region."},{"line_number":222,"context_line":""},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":225,"context_line":"with an inventory of CoS policies."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":228,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_baac7d9f","line":225,"range":{"start_line":224,"start_character":0,"end_line":225,"end_character":34},"updated":"2019-06-24 18:33:01.000000000","message":"as above","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":221,"context_line":"    region."},{"line_number":222,"context_line":""},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":225,"context_line":"with an inventory of CoS policies."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":228,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_de56efce","line":225,"range":{"start_line":224,"start_character":0,"end_line":225,"end_character":34},"in_reply_to":"9fb8cfa7_baac7d9f","updated":"2019-06-25 21:26:26.000000000","message":"this tends to be compatible with placement numa support without depending on it which is why i  have specified it like this.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":234,"context_line":"    If the NUMA affinity in placement spec is completed in train  this"},{"line_number":235,"context_line":"    feature will leverage the NUMA modeling support it introduces to track"},{"line_number":236,"context_line":"    cache and memory banding per NUMA node. The approach described above is"},{"line_number":237,"context_line":"    intended to facilitate this feature with our requiring a dependency"},{"line_number":238,"context_line":"    on the new placement feature while also allowing a simple path to adopt the"},{"line_number":239,"context_line":"    new feature in U via a reshape."},{"line_number":240,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_da8a9119","line":237,"range":{"start_line":237,"start_character":40,"end_line":237,"end_character":48},"updated":"2019-06-24 18:33:01.000000000","message":"without","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":235,"context_line":"    feature will leverage the NUMA modeling support it introduces to track"},{"line_number":236,"context_line":"    cache and memory banding per NUMA node. The approach described above is"},{"line_number":237,"context_line":"    intended to facilitate this feature with our requiring a dependency"},{"line_number":238,"context_line":"    on the new placement feature while also allowing a simple path to adopt the"},{"line_number":239,"context_line":"    new feature in U via a reshape."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"This spec proposes introducing three new standard resource classes:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7aa04591","line":238,"range":{"start_line":238,"start_character":44,"end_line":238,"end_character":75},"updated":"2019-06-24 18:33:01.000000000","message":"The described approach doesn\u0027t simplify any aspect of adopting NUMA modeling/affinity via placement in the future, no matter what form that takes. (Same argument at [1].)\n\n[1] https://review.opendev.org/#/c/601596/14/specs/train/approved/virtual-persistent-memory.rst@137","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":235,"context_line":"    feature will leverage the NUMA modeling support it introduces to track"},{"line_number":236,"context_line":"    cache and memory banding per NUMA node. The approach described above is"},{"line_number":237,"context_line":"    intended to facilitate this feature with our requiring a dependency"},{"line_number":238,"context_line":"    on the new placement feature while also allowing a simple path to adopt the"},{"line_number":239,"context_line":"    new feature in U via a reshape."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"This spec proposes introducing three new standard resource classes:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7e982374","line":238,"range":{"start_line":238,"start_character":44,"end_line":238,"end_character":75},"in_reply_to":"9fb8cfa7_7aa04591","updated":"2019-06-25 21:26:26.000000000","message":"that limitation is not listed in the api doc for placement.\nthis still simplfiles numa affinity in nova until we can model it in placemenbt as it allow us to map the placment allocation to the host_numa cell by haveing 1 placemetn RP per cache region which we use the host_numa_cell object to correlate with a numa node.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":240,"context_line":""},{"line_number":241,"context_line":"This spec proposes introducing three new standard resource classes:"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"- ``L3_DATA_CACHE``"},{"line_number":244,"context_line":"- ``L3_CODE_CACHE``"},{"line_number":245,"context_line":"- ``L3_UNIFIED_CACHE``"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":248,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_da2b71e8","line":245,"range":{"start_line":243,"start_character":0,"end_line":245,"end_character":22},"updated":"2019-06-24 18:33:01.000000000","message":"I don\u0027t understand: are these three separate kinds of resources, or is \"unified cache\" a combination of \"data cache\" and \"code cache\" where the host will report it one way or the other?\n\n[Later] See above (L159)","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":240,"context_line":""},{"line_number":241,"context_line":"This spec proposes introducing three new standard resource classes:"},{"line_number":242,"context_line":""},{"line_number":243,"context_line":"- ``L3_DATA_CACHE``"},{"line_number":244,"context_line":"- ``L3_CODE_CACHE``"},{"line_number":245,"context_line":"- ``L3_UNIFIED_CACHE``"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":248,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_f2f7678a","line":245,"range":{"start_line":243,"start_character":0,"end_line":245,"end_character":22},"in_reply_to":"9fb8cfa7_da2b71e8","updated":"2019-06-25 21:26:26.000000000","message":"it is but if the host is configured with CDP so that it repors data cache adn code cache seperate we cannot request an allocation of unified cache and vice versa","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":244,"context_line":"- ``L3_CODE_CACHE``"},{"line_number":245,"context_line":"- ``L3_UNIFIED_CACHE``"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":248,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":249,"context_line":"1-1.2 MBs. When CDP is enabled thw number of cachways is unchanged but the"},{"line_number":250,"context_line":"number of CoS polices is reduced by half, as this is hardware dependent the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7ace055b","line":247,"range":{"start_line":247,"start_character":4,"end_line":247,"end_character":61},"updated":"2019-06-24 18:33:01.000000000","message":"Following convention, the unit should be included in the resource class name. ``L3_DATA_CACHE_WAY`` or similar.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":244,"context_line":"- ``L3_CODE_CACHE``"},{"line_number":245,"context_line":"- ``L3_UNIFIED_CACHE``"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":248,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":249,"context_line":"1-1.2 MBs. When CDP is enabled thw number of cachways is unchanged but the"},{"line_number":250,"context_line":"number of CoS polices is reduced by half, as this is hardware dependent the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_b2424faa","line":247,"range":{"start_line":247,"start_character":4,"end_line":247,"end_character":61},"in_reply_to":"9fb8cfa7_7ace055b","updated":"2019-06-25 21:26:26.000000000","message":"im not sure we have full established that convention but sure we could do that but. its longer then i would like but ill update.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":246,"context_line":""},{"line_number":247,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":248,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":249,"context_line":"1-1.2 MBs. When CDP is enabled thw number of cachways is unchanged but the"},{"line_number":250,"context_line":"number of CoS polices is reduced by half, as this is hardware dependent the"},{"line_number":251,"context_line":"values will be transposed directly form the data reported by libvirt."},{"line_number":252,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_fa0bd58a","line":249,"range":{"start_line":249,"start_character":31,"end_line":249,"end_character":34},"updated":"2019-06-24 18:33:01.000000000","message":"the","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"bd54f5a686c825bbb4b1aa66ecd67e7c461eb0fa","unresolved":false,"context_lines":[{"line_number":248,"context_line":"cache is divided into a number of cache ways each of which is approximately"},{"line_number":249,"context_line":"1-1.2 MBs. When CDP is enabled thw number of cachways is unchanged but the"},{"line_number":250,"context_line":"number of CoS polices is reduced by half, as this is hardware dependent the"},{"line_number":251,"context_line":"values will be transposed directly form the data reported by libvirt."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"In addition to the resource classes, above two additional resources class will"},{"line_number":254,"context_line":"be added to model the number of \"Class of Service (COS)\" or"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_41d4f6bd","line":251,"range":{"start_line":251,"start_character":35,"end_line":251,"end_character":39},"updated":"2019-06-17 09:35:54.000000000","message":"s/form/from/","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"bd54f5a686c825bbb4b1aa66ecd67e7c461eb0fa","unresolved":false,"context_lines":[{"line_number":254,"context_line":"be added to model the number of \"Class of Service (COS)\" or"},{"line_number":255,"context_line":"\"Quality of Service (QoS)\" policies supported by the CPU."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"- ``CACHE_COS``"},{"line_number":258,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":261,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_81666e29","line":258,"range":{"start_line":257,"start_character":0,"end_line":258,"end_character":26},"updated":"2019-06-17 09:35:54.000000000","message":"Should these two resource trackers be the value parsed from libvirt XML minus one? The default \u0027resctrl\u0027 folder, which is \u0027/sys/fs/resctrl\u0027 will occupy one COS, both cache COS and memBW COS, if they are both supported by hardware.\n\n\nTake an example, only for the case of cache COS and memBW COS is similar. If CAT is supported and the hardware determined cache COS is 16, then the available COS to be allocated for VMs is 15.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"bdcc93caeeb44ce03dcb7fc758bc618b0f685e85","unresolved":false,"context_lines":[{"line_number":254,"context_line":"be added to model the number of \"Class of Service (COS)\" or"},{"line_number":255,"context_line":"\"Quality of Service (QoS)\" policies supported by the CPU."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"- ``CACHE_COS``"},{"line_number":258,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":261,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_bcc4f560","line":258,"range":{"start_line":257,"start_character":0,"end_line":258,"end_character":26},"in_reply_to":"9fb8cfa7_81666e29","updated":"2019-06-19 13:38:04.000000000","message":"im not sure. if that is the case then libvirt should be reporting that.\n\ni think we can leave this to the implementation to determine.\n\nwe shoudl always report the capastiy as 16 but we can set the reserved value to 1 if we determin that is need.\n\nfor the CACHE_COS value we would key it off the reserved_cacheways config value.\n\ne.g. we would only","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":254,"context_line":"be added to model the number of \"Class of Service (COS)\" or"},{"line_number":255,"context_line":"\"Quality of Service (QoS)\" policies supported by the CPU."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"- ``CACHE_COS``"},{"line_number":258,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":261,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_bddeb73b","line":258,"range":{"start_line":257,"start_character":0,"end_line":258,"end_character":26},"in_reply_to":"9fb8cfa7_bcc4f560","updated":"2019-06-24 18:33:01.000000000","message":"These need units.\n\nBut I don\u0027t understand from the description what they should be.\n\nAre these the ones that will have percentages? Or is it \"number of policies\"?\n\nAnd is a \"policy\" kind of like the slot/context concept we wound up with for SEV? So like max_unit\u003d1, it just costs us once to enable the thing for a guest, but we can only do that for a limited number of guests?","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":254,"context_line":"be added to model the number of \"Class of Service (COS)\" or"},{"line_number":255,"context_line":"\"Quality of Service (QoS)\" policies supported by the CPU."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"- ``CACHE_COS``"},{"line_number":258,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":261,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_529673f2","line":258,"range":{"start_line":257,"start_character":0,"end_line":258,"end_character":26},"in_reply_to":"9fb8cfa7_bddeb73b","updated":"2019-06-25 21:26:26.000000000","message":"the unit is a COS policy.\nso its a dimensionless number and shoudl not have a unit in the resouce class name.\n\nits like VCPU and PCPU they have no units.\n\nyes a COS policy is a hardware resouce like the sev encrption context. \n\n\ntechnically i does not need to have a max_unit\u003d1 and infact placment should not enforce that. it would be valid to have multiple COS polices for the same vm even if it only has 1 numa node but we wont require more then 1 allocation per inventory for a single instance based on what is enebled in this spec.\n\nmore advanced usecases could require that but i don\u0027t intend to support those in nova in train.\n\nif cache allocation was combinding with intels speed select technology then there are cases where we might want to future enhance this but im trying to keep this topic as simple as posible as its inherit compleity is already high.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"bd54f5a686c825bbb4b1aa66ecd67e7c461eb0fa","unresolved":false,"context_lines":[{"line_number":258,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":261,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"},{"line_number":262,"context_line":"to exhaust the number of COS policies available so they must be tracked as"},{"line_number":263,"context_line":"consumable resources."},{"line_number":264,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_21e8e27a","line":261,"range":{"start_line":261,"start_character":3,"end_line":261,"end_character":12},"updated":"2019-06-17 09:35:54.000000000","message":"Not understanding. What kind of \u0027order\u0027?","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"bdcc93caeeb44ce03dcb7fc758bc618b0f685e85","unresolved":false,"context_lines":[{"line_number":258,"context_line":"- ``MEMORY_BANDWIDTH_COS``"},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"The number of COS policies supported by a processor varies per SKU but is of"},{"line_number":261,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"},{"line_number":262,"context_line":"to exhaust the number of COS policies available so they must be tracked as"},{"line_number":263,"context_line":"consumable resources."},{"line_number":264,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_fc6eed94","line":261,"range":{"start_line":261,"start_character":3,"end_line":261,"end_character":12},"in_reply_to":"9fb8cfa7_21e8e27a","updated":"2019-06-19 13:38:04.000000000","message":"the phrase \"the order of\" refers to an approximant size that is within one order of magnitude fo the value.\n\nwhat im saying above is there is a high probability that between 8-16 or a moderate probability that there is between 4-7 or 17-32 and a lower probability that is \u003c4 or \u003e32.\n\nso my intent is to express simply that while this value will depend on hardware its much more likely to be close to 10  then 100 or 1000.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":299,"context_line":""},{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_bd29d737","line":302,"range":{"start_line":302,"start_character":32,"end_line":302,"end_character":38},"updated":"2019-06-24 18:33:01.000000000","message":"policies","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":299,"context_line":""},{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9d241362","line":302,"range":{"start_line":302,"start_character":42,"end_line":302,"end_character":50},"updated":"2019-06-24 18:33:01.000000000","message":"specified","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":299,"context_line":""},{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9547f54c","line":302,"range":{"start_line":302,"start_character":42,"end_line":302,"end_character":50},"in_reply_to":"9fb8cfa7_9d241362","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":299,"context_line":""},{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_55773da1","line":302,"range":{"start_line":302,"start_character":32,"end_line":302,"end_character":38},"in_reply_to":"9fb8cfa7_bd29d737","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_fa1995d6","line":303,"range":{"start_line":303,"start_character":68,"end_line":303,"end_character":74},"updated":"2019-06-24 18:33:01.000000000","message":"control","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_3a42cdac","line":303,"range":{"start_line":303,"start_character":55,"end_line":303,"end_character":60},"updated":"2019-06-24 18:33:01.000000000","message":"field","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_757cc182","line":303,"range":{"start_line":303,"start_character":55,"end_line":303,"end_character":60},"in_reply_to":"9fb8cfa7_3a42cdac","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":300,"context_line":".. note::"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_156d45cb","line":303,"range":{"start_line":303,"start_character":68,"end_line":303,"end_character":74},"in_reply_to":"9fb8cfa7_fa1995d6","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."},{"line_number":307,"context_line":"    The type filed can have 3 values. code, data and both. both corresponds"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9a0ad976","line":304,"range":{"start_line":304,"start_character":55,"end_line":304,"end_character":64},"updated":"2019-06-24 18:33:01.000000000","message":"attribute","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    The number of supported CoS policy is spcified by the maxAllocs field."},{"line_number":303,"context_line":"    The minium allcoation size is specified in the min filed of the contol"},{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."},{"line_number":307,"context_line":"    The type filed can have 3 values. code, data and both. both corresponds"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_3572c9ae","line":304,"range":{"start_line":304,"start_character":55,"end_line":304,"end_character":64},"in_reply_to":"9fb8cfa7_9a0ad976","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."},{"line_number":307,"context_line":"    The type filed can have 3 values. code, data and both. both corresponds"},{"line_number":308,"context_line":"    to a unifed cache that can be used for both code and data."},{"line_number":309,"context_line":"    libvirt reports the capasity of cache allocation in a multiple of bytes"},{"line_number":310,"context_line":"    not cacheways so nova will have to normalise form bytes to units of"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7a25a508","line":307,"range":{"start_line":307,"start_character":13,"end_line":307,"end_character":18},"updated":"2019-06-24 18:33:01.000000000","message":"field","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":304,"context_line":"    element and if not set is equal to the granularity attirbute."},{"line_number":305,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":306,"context_line":"    placement inventory."},{"line_number":307,"context_line":"    The type filed can have 3 values. code, data and both. both corresponds"},{"line_number":308,"context_line":"    to a unifed cache that can be used for both code and data."},{"line_number":309,"context_line":"    libvirt reports the capasity of cache allocation in a multiple of bytes"},{"line_number":310,"context_line":"    not cacheways so nova will have to normalise form bytes to units of"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_d56a4db1","line":307,"range":{"start_line":307,"start_character":13,"end_line":307,"end_character":18},"in_reply_to":"9fb8cfa7_7a25a508","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"923269cfa780c894dba4017a6e1081e3ae52d5a9","unresolved":false,"context_lines":[{"line_number":306,"context_line":"    placement inventory."},{"line_number":307,"context_line":"    The type filed can have 3 values. code, data and both. both corresponds"},{"line_number":308,"context_line":"    to a unifed cache that can be used for both code and data."},{"line_number":309,"context_line":"    libvirt reports the capasity of cache allocation in a multiple of bytes"},{"line_number":310,"context_line":"    not cacheways so nova will have to normalise form bytes to units of"},{"line_number":311,"context_line":"    cacheway accounting for the unit for granularity and size."},{"line_number":312,"context_line":"    if the contol element is not present the host does not support cache"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_56bcba6b","line":309,"range":{"start_line":309,"start_character":24,"end_line":309,"end_character":32},"updated":"2019-06-17 09:18:09.000000000","message":"s/capasity/capacity/","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":307,"context_line":"    The type filed can have 3 values. code, data and both. both corresponds"},{"line_number":308,"context_line":"    to a unifed cache that can be used for both code and data."},{"line_number":309,"context_line":"    libvirt reports the capasity of cache allocation in a multiple of bytes"},{"line_number":310,"context_line":"    not cacheways so nova will have to normalise form bytes to units of"},{"line_number":311,"context_line":"    cacheway accounting for the unit for granularity and size."},{"line_number":312,"context_line":"    if the contol element is not present the host does not support cache"},{"line_number":313,"context_line":"    allocation and no inventories or traits will be reported to placement."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_da7931fc","line":310,"range":{"start_line":310,"start_character":49,"end_line":310,"end_character":53},"updated":"2019-06-24 18:33:01.000000000","message":"from","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":307,"context_line":"    The type filed can have 3 values. code, data and both. both corresponds"},{"line_number":308,"context_line":"    to a unifed cache that can be used for both code and data."},{"line_number":309,"context_line":"    libvirt reports the capasity of cache allocation in a multiple of bytes"},{"line_number":310,"context_line":"    not cacheways so nova will have to normalise form bytes to units of"},{"line_number":311,"context_line":"    cacheway accounting for the unit for granularity and size."},{"line_number":312,"context_line":"    if the contol element is not present the host does not support cache"},{"line_number":313,"context_line":"    allocation and no inventories or traits will be reported to placement."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_559c1dbf","line":310,"range":{"start_line":310,"start_character":49,"end_line":310,"end_character":53},"in_reply_to":"9fb8cfa7_da7931fc","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":309,"context_line":"    libvirt reports the capasity of cache allocation in a multiple of bytes"},{"line_number":310,"context_line":"    not cacheways so nova will have to normalise form bytes to units of"},{"line_number":311,"context_line":"    cacheway accounting for the unit for granularity and size."},{"line_number":312,"context_line":"    if the contol element is not present the host does not support cache"},{"line_number":313,"context_line":"    allocation and no inventories or traits will be reported to placement."},{"line_number":314,"context_line":"    The granularity for memory bandwith will be mapped to step_size in the"},{"line_number":315,"context_line":"    placement inventory and the max_unit will be set to 100. The units of"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_ba847dd4","line":312,"range":{"start_line":312,"start_character":11,"end_line":312,"end_character":17},"updated":"2019-06-24 18:33:01.000000000","message":"control","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":309,"context_line":"    libvirt reports the capasity of cache allocation in a multiple of bytes"},{"line_number":310,"context_line":"    not cacheways so nova will have to normalise form bytes to units of"},{"line_number":311,"context_line":"    cacheway accounting for the unit for granularity and size."},{"line_number":312,"context_line":"    if the contol element is not present the host does not support cache"},{"line_number":313,"context_line":"    allocation and no inventories or traits will be reported to placement."},{"line_number":314,"context_line":"    The granularity for memory bandwith will be mapped to step_size in the"},{"line_number":315,"context_line":"    placement inventory and the max_unit will be set to 100. The units of"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_75a121f8","line":312,"range":{"start_line":312,"start_character":11,"end_line":312,"end_character":17},"in_reply_to":"9fb8cfa7_ba847dd4","updated":"2019-06-25 21:26:26.000000000","message":"Done","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":312,"context_line":"    if the contol element is not present the host does not support cache"},{"line_number":313,"context_line":"    allocation and no inventories or traits will be reported to placement."},{"line_number":314,"context_line":"    The granularity for memory bandwith will be mapped to step_size in the"},{"line_number":315,"context_line":"    placement inventory and the max_unit will be set to 100. The units of"},{"line_number":316,"context_line":"    memory bandwith are in percent of the memory contoler bandwith and"},{"line_number":317,"context_line":"    represents a max bandwidth limit not a minium bandwith allocation."},{"line_number":318,"context_line":""},{"line_number":319,"context_line":".. note::"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_3a496db6","line":317,"range":{"start_line":315,"start_character":32,"end_line":317,"end_character":36},"updated":"2019-06-24 18:33:01.000000000","message":"So what\u0027s the `total`? The `allocation_ratio`? I guess I\u0027m asking: can you \"overallocate\" this thing?\n\nAlso: I thought the unit for cache was \"ways\", but this is implying something more like \"percentage\".","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":314,"context_line":"    The granularity for memory bandwith will be mapped to step_size in the"},{"line_number":315,"context_line":"    placement inventory and the max_unit will be set to 100. The units of"},{"line_number":316,"context_line":"    memory bandwith are in percent of the memory contoler bandwith and"},{"line_number":317,"context_line":"    represents a max bandwidth limit not a minium bandwith allocation."},{"line_number":318,"context_line":""},{"line_number":319,"context_line":".. note::"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_1a734918","line":317,"range":{"start_line":317,"start_character":43,"end_line":317,"end_character":49},"updated":"2019-06-24 18:33:01.000000000","message":"minimum","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":312,"context_line":"    if the contol element is not present the host does not support cache"},{"line_number":313,"context_line":"    allocation and no inventories or traits will be reported to placement."},{"line_number":314,"context_line":"    The granularity for memory bandwith will be mapped to step_size in the"},{"line_number":315,"context_line":"    placement inventory and the max_unit will be set to 100. The units of"},{"line_number":316,"context_line":"    memory bandwith are in percent of the memory contoler bandwith and"},{"line_number":317,"context_line":"    represents a max bandwidth limit not a minium bandwith allocation."},{"line_number":318,"context_line":""},{"line_number":319,"context_line":".. note::"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_b5e0f927","line":317,"range":{"start_line":315,"start_character":32,"end_line":317,"end_character":36},"in_reply_to":"9fb8cfa7_3a496db6","updated":"2019-06-25 21:26:26.000000000","message":"i meant to delete this. that is form the initial draft before i validated that the hardware does not support minimum bandwidth allocation and only support a max bandwidth limit.\n\nthere will be no inventory for memory bandwidth i will remove this.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":316,"context_line":"    memory bandwith are in percent of the memory contoler bandwith and"},{"line_number":317,"context_line":"    represents a max bandwidth limit not a minium bandwith allocation."},{"line_number":318,"context_line":""},{"line_number":319,"context_line":".. note::"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":"   CAT requires contiguous allocation of cacheways. This limitation can lead"},{"line_number":322,"context_line":"   to fragmentation issues. As the libvirt api does not allow Nova to directly"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_baa0fd43","line":319,"range":{"start_line":319,"start_character":3,"end_line":319,"end_character":7},"updated":"2019-06-24 18:33:01.000000000","message":"nit, overuse of .. note:: makes this kind of unreadable. Does everything need to be a .. note::?","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":316,"context_line":"    memory bandwith are in percent of the memory contoler bandwith and"},{"line_number":317,"context_line":"    represents a max bandwidth limit not a minium bandwith allocation."},{"line_number":318,"context_line":""},{"line_number":319,"context_line":".. note::"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":"   CAT requires contiguous allocation of cacheways. This limitation can lead"},{"line_number":322,"context_line":"   to fragmentation issues. As the libvirt api does not allow Nova to directly"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_75c2815b","line":319,"range":{"start_line":319,"start_character":3,"end_line":319,"end_character":7},"in_reply_to":"9fb8cfa7_baa0fd43","updated":"2019-06-25 21:26:26.000000000","message":"i marked this as a note as it is background context that i am adding so that people not familiar with the hardware technology can follow the spec and to document some of my reasoning for my choices. the section in the notes should not be required to implement the feature and all of them could in theory be removed as they are elaborating on way the proposed change can be use.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"923269cfa780c894dba4017a6e1081e3ae52d5a9","unresolved":false,"context_lines":[{"line_number":322,"context_line":"   to fragmentation issues. As the libvirt api does not allow Nova to directly"},{"line_number":323,"context_line":"   manage the allocation of specific cacheways to instances the only way to"},{"line_number":324,"context_line":"   prevent fragmentation is to ensure that all allocations on the same host"},{"line_number":325,"context_line":"   are of the same size. One way to Achive this would be to restict the"},{"line_number":326,"context_line":"   max_unit and to equal the min_unit in the placment inventory. This would"},{"line_number":327,"context_line":"   be overly restitive and would not support all workloads. By default when"},{"line_number":328,"context_line":"   nova create the inventory for cache initially it will set the max_unit"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_166ba2f1","line":325,"range":{"start_line":325,"start_character":36,"end_line":325,"end_character":42},"updated":"2019-06-17 09:18:09.000000000","message":"achieve","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":322,"context_line":"   to fragmentation issues. As the libvirt api does not allow Nova to directly"},{"line_number":323,"context_line":"   manage the allocation of specific cacheways to instances the only way to"},{"line_number":324,"context_line":"   prevent fragmentation is to ensure that all allocations on the same host"},{"line_number":325,"context_line":"   are of the same size. One way to Achive this would be to restict the"},{"line_number":326,"context_line":"   max_unit and to equal the min_unit in the placment inventory. This would"},{"line_number":327,"context_line":"   be overly restitive and would not support all workloads. By default when"},{"line_number":328,"context_line":"   nova create the inventory for cache initially it will set the max_unit"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_9acdf931","line":325,"range":{"start_line":325,"start_character":60,"end_line":325,"end_character":67},"updated":"2019-06-24 18:33:01.000000000","message":"restrict","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":324,"context_line":"   prevent fragmentation is to ensure that all allocations on the same host"},{"line_number":325,"context_line":"   are of the same size. One way to Achive this would be to restict the"},{"line_number":326,"context_line":"   max_unit and to equal the min_unit in the placment inventory. This would"},{"line_number":327,"context_line":"   be overly restitive and would not support all workloads. By default when"},{"line_number":328,"context_line":"   nova create the inventory for cache initially it will set the max_unit"},{"line_number":329,"context_line":"   and step_size to the larger of grunularity and min field when converted"},{"line_number":330,"context_line":"   to cache ways. when the compute agent start up if the inventory already"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_daf0516a","line":327,"range":{"start_line":327,"start_character":13,"end_line":327,"end_character":22},"updated":"2019-06-24 18:33:01.000000000","message":"restrictive?","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":325,"context_line":"   are of the same size. One way to Achive this would be to restict the"},{"line_number":326,"context_line":"   max_unit and to equal the min_unit in the placment inventory. This would"},{"line_number":327,"context_line":"   be overly restitive and would not support all workloads. By default when"},{"line_number":328,"context_line":"   nova create the inventory for cache initially it will set the max_unit"},{"line_number":329,"context_line":"   and step_size to the larger of grunularity and min field when converted"},{"line_number":330,"context_line":"   to cache ways. when the compute agent start up if the inventory already"},{"line_number":331,"context_line":"   exists the libvirt virt driver will not modify the max_unit or step_size."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_1ae1099c","line":328,"range":{"start_line":328,"start_character":8,"end_line":328,"end_character":14},"updated":"2019-06-24 18:33:01.000000000","message":"creates","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":326,"context_line":"   max_unit and to equal the min_unit in the placment inventory. This would"},{"line_number":327,"context_line":"   be overly restitive and would not support all workloads. By default when"},{"line_number":328,"context_line":"   nova create the inventory for cache initially it will set the max_unit"},{"line_number":329,"context_line":"   and step_size to the larger of grunularity and min field when converted"},{"line_number":330,"context_line":"   to cache ways. when the compute agent start up if the inventory already"},{"line_number":331,"context_line":"   exists the libvirt virt driver will not modify the max_unit or step_size."},{"line_number":332,"context_line":"   This will allow operators to modify the stepsize and max_unit per host if"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_faf51558","line":329,"range":{"start_line":329,"start_character":34,"end_line":329,"end_character":45},"updated":"2019-06-24 18:33:01.000000000","message":"granularity","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":394,"context_line":"cache affinity which many degrade the performance of that filter. This spec"},{"line_number":395,"context_line":"also proposes the introduction of a prefilter to transform the flavor request"},{"line_number":396,"context_line":"into placement requests. This prefilter will be disabled by default. When"},{"line_number":397,"context_line":"enabled, it will add a forbidden trait to the unnumbered request group when"},{"line_number":398,"context_line":"cache and/or bandwidth limits are not requested and will add a required trait"},{"line_number":399,"context_line":"and additional resource requests when they are. It is intended for this to"},{"line_number":400,"context_line":"increase scheduling performance by restricting the set of possible host by"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7d391fd5","line":397,"range":{"start_line":397,"start_character":46,"end_line":397,"end_character":70},"updated":"2019-06-24 18:33:01.000000000","message":"Use root_required [1][2] instead.\n\n[1] https://docs.openstack.org/placement/latest/specs/train/approved/2005575-nested-magic-1.html#root-required\n[2] https://review.opendev.org/#/c/665492/","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":394,"context_line":"cache affinity which many degrade the performance of that filter. This spec"},{"line_number":395,"context_line":"also proposes the introduction of a prefilter to transform the flavor request"},{"line_number":396,"context_line":"into placement requests. This prefilter will be disabled by default. When"},{"line_number":397,"context_line":"enabled, it will add a forbidden trait to the unnumbered request group when"},{"line_number":398,"context_line":"cache and/or bandwidth limits are not requested and will add a required trait"},{"line_number":399,"context_line":"and additional resource requests when they are. It is intended for this to"},{"line_number":400,"context_line":"increase scheduling performance by restricting the set of possible host by"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_959b7571","line":397,"range":{"start_line":397,"start_character":46,"end_line":397,"end_character":70},"in_reply_to":"9fb8cfa7_7d391fd5","updated":"2019-06-25 21:26:26.000000000","message":"i guess its too late to say i don\u0027t like that paramater name.\n\nroot_requried implies that you cant negate due to the previous exsiting usage for required and forbiding traits.\n\nroot_traits would have been better IMO.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":399,"context_line":"and additional resource requests when they are. It is intended for this to"},{"line_number":400,"context_line":"increase scheduling performance by restricting the set of possible host by"},{"line_number":401,"context_line":"leveraging granular requests while also ensuring host are dynamically"},{"line_number":402,"context_line":"partitions such that no single host will be used for both cache allocated"},{"line_number":403,"context_line":"guests and guests without allocations."},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_3d2f2789","line":402,"range":{"start_line":402,"start_character":0,"end_line":402,"end_character":10},"updated":"2019-06-24 18:33:01.000000000","message":"partitioned","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":400,"context_line":"increase scheduling performance by restricting the set of possible host by"},{"line_number":401,"context_line":"leveraging granular requests while also ensuring host are dynamically"},{"line_number":402,"context_line":"partitions such that no single host will be used for both cache allocated"},{"line_number":403,"context_line":"guests and guests without allocations."},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"Other deployer impact"},{"line_number":406,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_fd52ef12","line":403,"updated":"2019-06-24 18:33:01.000000000","message":"again this controverts L36 (or I\u0027m missing the boat)","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":400,"context_line":"increase scheduling performance by restricting the set of possible host by"},{"line_number":401,"context_line":"leveraging granular requests while also ensuring host are dynamically"},{"line_number":402,"context_line":"partitions such that no single host will be used for both cache allocated"},{"line_number":403,"context_line":"guests and guests without allocations."},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"Other deployer impact"},{"line_number":406,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_f5019139","line":403,"in_reply_to":"9fb8cfa7_fd52ef12","updated":"2019-06-25 21:26:26.000000000","message":"i had to change the design when i determined that we could not pass a mask to libvirt.\n\nlibvirt implicitly looks at the cache allocation mask that the libvirt daemon is running under and only spawn allocates vms to the  remainder of the cache not used libvirt and its process group.\n\nTechnically we could rely on this undocumented behaviour and libvirt would allow us to correctly have instance that request cache allocation with those that don\u0027t on the same host.\n\nHowever i\u0027m not sure i am comfortable with that,\non later reflection i was leaning towards not allowing mixing cache confined instance with non cache confined instances on the same host.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":412,"context_line":"  :Type: bool"},{"line_number":413,"context_line":"  :Default: False"},{"line_number":414,"context_line":"  :Description: When set to True enable reporting of cache allocation compute"},{"line_number":415,"context_line":"    capability trait and cache resouce classes. This config option controls"},{"line_number":416,"context_line":"    generation of cache allocation element in the domain XML. This value"},{"line_number":417,"context_line":"    should be set to false on hosts that will not be used for cache confined"},{"line_number":418,"context_line":"    guest or when cache allocation is enforced by an external agent."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_3d440747","line":415,"range":{"start_line":415,"start_character":31,"end_line":415,"end_character":38},"updated":"2019-06-24 18:33:01.000000000","message":"resource","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":436,"context_line":"  :Type: bool"},{"line_number":437,"context_line":"  :Default: False"},{"line_number":438,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"},{"line_number":439,"context_line":"    memory bandwidth extra specs into placement requests."},{"line_number":440,"context_line":""},{"line_number":441,"context_line":"Developer impact"},{"line_number":442,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_1db58332","line":439,"updated":"2019-06-24 18:33:01.000000000","message":"Otherwise the new extra specs will be ignored? cause an error?","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":436,"context_line":"  :Type: bool"},{"line_number":437,"context_line":"  :Default: False"},{"line_number":438,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"},{"line_number":439,"context_line":"    memory bandwidth extra specs into placement requests."},{"line_number":440,"context_line":""},{"line_number":441,"context_line":"Developer impact"},{"line_number":442,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7537e100","line":439,"in_reply_to":"9fb8cfa7_1db58332","updated":"2019-06-25 21:26:26.000000000","message":"The extra specs would be ignored.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":437,"context_line":"  :Default: False"},{"line_number":438,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"},{"line_number":439,"context_line":"    memory bandwidth extra specs into placement requests."},{"line_number":440,"context_line":""},{"line_number":441,"context_line":"Developer impact"},{"line_number":442,"context_line":"----------------"},{"line_number":443,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_7d959f86","line":440,"updated":"2019-06-24 18:33:01.000000000","message":"As noted elsewhere, we should address whether the raw placement-ese syntax will be allowed, or whether the \"native\" syntax proposed above is the only supported mechanism.\n\nMy preference is for the latter: we should implement a hard check to forbid the former.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":437,"context_line":"  :Default: False"},{"line_number":438,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"},{"line_number":439,"context_line":"    memory bandwidth extra specs into placement requests."},{"line_number":440,"context_line":""},{"line_number":441,"context_line":"Developer impact"},{"line_number":442,"context_line":"----------------"},{"line_number":443,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_b5a55949","line":440,"in_reply_to":"9fb8cfa7_7d959f86","updated":"2019-06-25 21:26:26.000000000","message":"personally i would prefer to forbid using resource: in general.\n\ni had intended to only allow the new extra_specs. i will clarify this.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"ff6ddb283df7ea7cda4382edb24517250faa3fdc","unresolved":false,"context_lines":[{"line_number":457,"context_line":"Similarly, as this spec introduces new NUMA-affined resources, the"},{"line_number":458,"context_line":"implementation will have to collaborate with the implementation of PCUs in"},{"line_number":459,"context_line":"placement, pMEM support and NUMA-aware vGPUS when extending the NUMA topology"},{"line_number":460,"context_line":"filter."},{"line_number":461,"context_line":""},{"line_number":462,"context_line":"Upgrade impact"},{"line_number":463,"context_line":"--------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_fdee6f20","line":460,"updated":"2019-06-24 18:33:01.000000000","message":"links to the various specs mentioned in this section","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5bbe91098a754f63913c9c2eccfeddfb8c4aad95","unresolved":false,"context_lines":[{"line_number":457,"context_line":"Similarly, as this spec introduces new NUMA-affined resources, the"},{"line_number":458,"context_line":"implementation will have to collaborate with the implementation of PCUs in"},{"line_number":459,"context_line":"placement, pMEM support and NUMA-aware vGPUS when extending the NUMA topology"},{"line_number":460,"context_line":"filter."},{"line_number":461,"context_line":""},{"line_number":462,"context_line":"Upgrade impact"},{"line_number":463,"context_line":"--------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_35164957","line":460,"in_reply_to":"9fb8cfa7_fdee6f20","updated":"2019-06-25 21:26:26.000000000","message":"ill try and update this section in the next revsion but ill push what i currently have","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":30209,"name":"Huaqiang","email":"huaqiang.wang@intel.com","username":"Huaqiang.Wang"},"change_message_id":"923269cfa780c894dba4017a6e1081e3ae52d5a9","unresolved":false,"context_lines":[{"line_number":546,"context_line":".. note::"},{"line_number":547,"context_line":""},{"line_number":548,"context_line":"    ``max_allocations`` refers to the number of COS profiles, ``granularity``"},{"line_number":549,"context_line":"    refers to the cache way size, the minimum number of cache ways is ``2 *"},{"line_number":550,"context_line":"    granularity``, and cache way associativity means the size/granularity"},{"line_number":551,"context_line":"    which is equal to the number of total physical cores on a CPU package,"},{"line_number":552,"context_line":"    not the enabled cores on a CPU package."},{"line_number":553,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_d622eaba","line":550,"range":{"start_line":549,"start_character":34,"end_line":550,"end_character":18},"updated":"2019-06-17 09:18:09.000000000","message":"I think you have forgotten to change this.\n\nYou have already explained these two phrases in the notes of cache capability XML section, maybe you could remove this paragraph.","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6e0f035b8a625734d3e3d9c5555dc2ddf20d3cd7","unresolved":false,"context_lines":[{"line_number":546,"context_line":".. note::"},{"line_number":547,"context_line":""},{"line_number":548,"context_line":"    ``max_allocations`` refers to the number of COS profiles, ``granularity``"},{"line_number":549,"context_line":"    refers to the cache way size, the minimum number of cache ways is ``2 *"},{"line_number":550,"context_line":"    granularity``, and cache way associativity means the size/granularity"},{"line_number":551,"context_line":"    which is equal to the number of total physical cores on a CPU package,"},{"line_number":552,"context_line":"    not the enabled cores on a CPU package."},{"line_number":553,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9fb8cfa7_a19052ef","line":550,"range":{"start_line":549,"start_character":34,"end_line":550,"end_character":18},"in_reply_to":"9fb8cfa7_d622eaba","updated":"2019-06-17 09:33:50.000000000","message":"ya your right i forgot to remove this\nill drop this note","commit_id":"4cee220ab3beafa0147ae5f6dc422fc09ba54c74"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3df58643ffc3f4aa6fc8511000cd3e16802a5dbd","unresolved":false,"context_lines":[{"line_number":65,"context_line":"allocations of CPU cacheways and enforcement of maximum memory bandwidth"},{"line_number":66,"context_line":"limits. ``Cacheways`` refer to the associativity of memory addresses to cpu"},{"line_number":67,"context_line":"cache entries typically implemented in hardware via hash algorithms which"},{"line_number":68,"context_line":"device a logic set of valid cache entreis for each memory location. Cache"},{"line_number":69,"context_line":"allocation Technology allows parameterising the hash with a mask to restict"},{"line_number":70,"context_line":"the entries that comprise the set of valid cache entries for a given address."},{"line_number":71,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"9fb8cfa7_70635d1b","line":68,"range":{"start_line":68,"start_character":0,"end_line":68,"end_character":41},"updated":"2019-07-02 16:07:29.000000000","message":"devise a logical set of valid cache entries?","commit_id":"bdaddc96b8966b1c8ffb01c5be70415f5c4233ed"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3ad9d8d1653d31211a0563d192ed3d946b660767","unresolved":false,"context_lines":[{"line_number":65,"context_line":"allocations of CPU cacheways and enforcement of maximum memory bandwidth"},{"line_number":66,"context_line":"limits. ``Cacheways`` refer to the associativity of memory addresses to cpu"},{"line_number":67,"context_line":"cache entries typically implemented in hardware via hash algorithms which"},{"line_number":68,"context_line":"device a logic set of valid cache entreis for each memory location. Cache"},{"line_number":69,"context_line":"allocation Technology allows parameterising the hash with a mask to restict"},{"line_number":70,"context_line":"the entries that comprise the set of valid cache entries for a given address."},{"line_number":71,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"9fb8cfa7_9bf01b94","line":68,"range":{"start_line":68,"start_character":0,"end_line":68,"end_character":41},"in_reply_to":"9fb8cfa7_70635d1b","updated":"2019-07-02 17:24:18.000000000","message":"fixed in later version","commit_id":"bdaddc96b8966b1c8ffb01c5be70415f5c4233ed"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3df58643ffc3f4aa6fc8511000cd3e16802a5dbd","unresolved":false,"context_lines":[{"line_number":66,"context_line":"limits. ``Cacheways`` refer to the associativity of memory addresses to cpu"},{"line_number":67,"context_line":"cache entries typically implemented in hardware via hash algorithms which"},{"line_number":68,"context_line":"device a logic set of valid cache entreis for each memory location. Cache"},{"line_number":69,"context_line":"allocation Technology allows parameterising the hash with a mask to restict"},{"line_number":70,"context_line":"the entries that comprise the set of valid cache entries for a given address."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Delegation of discovery and enforcement to an external agent (RMD, CYBORG, cron"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9fb8cfa7_8b2d8496","line":69,"range":{"start_line":69,"start_character":68,"end_line":69,"end_character":75},"updated":"2019-07-02 16:07:29.000000000","message":"restrict","commit_id":"bdaddc96b8966b1c8ffb01c5be70415f5c4233ed"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"3ad9d8d1653d31211a0563d192ed3d946b660767","unresolved":false,"context_lines":[{"line_number":66,"context_line":"limits. ``Cacheways`` refer to the associativity of memory addresses to cpu"},{"line_number":67,"context_line":"cache entries typically implemented in hardware via hash algorithms which"},{"line_number":68,"context_line":"device a logic set of valid cache entreis for each memory location. Cache"},{"line_number":69,"context_line":"allocation Technology allows parameterising the hash with a mask to restict"},{"line_number":70,"context_line":"the entries that comprise the set of valid cache entries for a given address."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Delegation of discovery and enforcement to an external agent (RMD, CYBORG, cron"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9fb8cfa7_fbf3579e","line":69,"range":{"start_line":69,"start_character":68,"end_line":69,"end_character":75},"in_reply_to":"9fb8cfa7_8b2d8496","updated":"2019-07-02 17:24:18.000000000","message":"Done","commit_id":"bdaddc96b8966b1c8ffb01c5be70415f5c4233ed"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"16f71f5158e461e6fc250798d603b8639221d751","unresolved":false,"context_lines":[{"line_number":17,"context_line":"the guest can be undone due to noisy neighbor effects such as cache thrashing"},{"line_number":18,"context_line":"or memory bandwidth exhaustion. This spec seeks to address this via static"},{"line_number":19,"context_line":"allocation of cache and memory bandwidth limits via the flavor, leveraging"},{"line_number":20,"context_line":"placement for resource accounting and the NUMA topology filter for affinity."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"https://blueprints.launchpad.net/nova/+spec/libvirt-pqos"},{"line_number":23,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_7b03ec0d","line":20,"range":{"start_line":20,"start_character":34,"end_line":20,"end_character":75},"updated":"2019-07-02 13:39:45.000000000","message":"Is it worth stating clearly that ultimately the issue of noisy neighbours is addressed here by ensuring allocated and non-allocated instances don\u0027t end up on the same host?","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"16f71f5158e461e6fc250798d603b8639221d751","unresolved":false,"context_lines":[{"line_number":66,"context_line":"allocations of CPU cacheways and enforcement of maximum memory bandwidth"},{"line_number":67,"context_line":"limits. ``Cacheways`` refer to the associativity of memory addresses to cpu"},{"line_number":68,"context_line":"cache entries typically implemented in hardware via hash algorithms which"},{"line_number":69,"context_line":"device a logic set of valid cache entreis for each memory location. Cache"},{"line_number":70,"context_line":"allocation Technology allows parameterising the hash with a mask to restict"},{"line_number":71,"context_line":"the entries that comprise the set of valid cache entries for a given address."},{"line_number":72,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_a01e15b8","line":69,"range":{"start_line":69,"start_character":0,"end_line":69,"end_character":6},"updated":"2019-07-02 13:39:45.000000000","message":"provide?","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7ebf07905850536bba4b0a2510dc99ca5aa0051f","unresolved":false,"context_lines":[{"line_number":66,"context_line":"allocations of CPU cacheways and enforcement of maximum memory bandwidth"},{"line_number":67,"context_line":"limits. ``Cacheways`` refer to the associativity of memory addresses to cpu"},{"line_number":68,"context_line":"cache entries typically implemented in hardware via hash algorithms which"},{"line_number":69,"context_line":"device a logic set of valid cache entreis for each memory location. Cache"},{"line_number":70,"context_line":"allocation Technology allows parameterising the hash with a mask to restict"},{"line_number":71,"context_line":"the entries that comprise the set of valid cache entries for a given address."},{"line_number":72,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_0aaffb7d","line":69,"range":{"start_line":69,"start_character":0,"end_line":69,"end_character":6},"in_reply_to":"9fb8cfa7_a01e15b8","updated":"2019-07-02 15:25:28.000000000","message":"yes","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":127,"context_line":"These new extra spec will also support the dot notation supported by the other"},{"line_number":128,"context_line":"``hw:numa_*`` extra specs to allow specifying cache and memory bandwidth per"},{"line_number":129,"context_line":"virtual NUMA node. For example, consider a guest with two virtual NUMA nodes,"},{"line_number":130,"context_line":"two cache ways per NUMA node and a limit of 10% of the host NUMA node"},{"line_number":131,"context_line":"memory bandwidth ::"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"    hw:numa_nodes\u003d2"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_6a90fc43","line":130,"range":{"start_line":130,"start_character":0,"end_line":130,"end_character":28},"updated":"2019-07-02 11:22:58.000000000","message":"so an equal split between the nodes.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":127,"context_line":"These new extra spec will also support the dot notation supported by the other"},{"line_number":128,"context_line":"``hw:numa_*`` extra specs to allow specifying cache and memory bandwidth per"},{"line_number":129,"context_line":"virtual NUMA node. For example, consider a guest with two virtual NUMA nodes,"},{"line_number":130,"context_line":"two cache ways per NUMA node and a limit of 10% of the host NUMA node"},{"line_number":131,"context_line":"memory bandwidth ::"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"    hw:numa_nodes\u003d2"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_e022cd51","line":130,"range":{"start_line":130,"start_character":0,"end_line":130,"end_character":28},"in_reply_to":"9fb8cfa7_6a90fc43","updated":"2019-07-02 13:03:55.000000000","message":"yes the same way as cpus or ram.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":127,"context_line":"These new extra spec will also support the dot notation supported by the other"},{"line_number":128,"context_line":"``hw:numa_*`` extra specs to allow specifying cache and memory bandwidth per"},{"line_number":129,"context_line":"virtual NUMA node. For example, consider a guest with two virtual NUMA nodes,"},{"line_number":130,"context_line":"two cache ways per NUMA node and a limit of 10% of the host NUMA node"},{"line_number":131,"context_line":"memory bandwidth ::"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":134,"context_line":"    hw:numa_cache\u003d4"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_2a96843f","line":131,"range":{"start_line":130,"start_character":35,"end_line":131,"end_character":16},"updated":"2019-07-02 11:22:58.000000000","message":"10% of the memory bandwidth of each NUMA nodes?","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":127,"context_line":"These new extra spec will also support the dot notation supported by the other"},{"line_number":128,"context_line":"``hw:numa_*`` extra specs to allow specifying cache and memory bandwidth per"},{"line_number":129,"context_line":"virtual NUMA node. For example, consider a guest with two virtual NUMA nodes,"},{"line_number":130,"context_line":"two cache ways per NUMA node and a limit of 10% of the host NUMA node"},{"line_number":131,"context_line":"memory bandwidth ::"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":134,"context_line":"    hw:numa_cache\u003d4"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_8032f1a5","line":131,"range":{"start_line":130,"start_character":35,"end_line":131,"end_character":16},"in_reply_to":"9fb8cfa7_2a96843f","updated":"2019-07-02 13:03:55.000000000","message":"yes this has chagne back and forth\n\nerric pointed out that deviding a percentage by the number of numa nodes felt weird so in this version i removed that behavior so its X% per numa node rather then X/N % where N is the number or numa nodes.\n\nill clarify in the next version","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":165,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":166,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":167,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":168,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":169,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":".. note::"},{"line_number":172,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_caeb28a1","line":169,"range":{"start_line":168,"start_character":5,"end_line":169,"end_character":50},"updated":"2019-07-02 11:22:58.000000000","message":"But then what is the meaning of the following?\n    hw:numa_bandwidth.0\u003d10\n    hw:numa_bandwidth.1\u003d30\n\nIs it 10% + 30% of the whole host bandwidth? And no enforcement that only 10% is used by virtual numa 0?","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":166,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":167,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":168,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":169,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":".. note::"},{"line_number":172,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_0ac86052","line":169,"range":{"start_line":169,"start_character":15,"end_line":169,"end_character":22},"updated":"2019-07-02 11:22:58.000000000","message":"divided","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":165,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":166,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":167,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":168,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":169,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":".. note::"},{"line_number":172,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_c5bd5bb1","line":169,"range":{"start_line":168,"start_character":5,"end_line":169,"end_character":50},"in_reply_to":"9fb8cfa7_caeb28a1","updated":"2019-07-02 13:03:55.000000000","message":"it means 10% limit on memory bandwidth of the host numa node corresponidn to virutal numa node0 + 30% of host numa node bandwidth for the host numa node corresponidnt to virtual numa node 1\n\n\nwe map each virtual numa node to a sperate host numa node\nso when you say\n\nhw:numa_bandwidth.0\u003d10\nhw:numa_bandwidth.1\u003d30\n\nwe are saying that the acess to memory attached to virtual numa node 0 will be limited to using at most 10% of the host numa node the get mapped too. in parallel to that we also enforfce a max memory bandwith limit on virtual numa node 1 where   acess to memory on virtual numa node 1 is limited to 30% of the host numa nodes bandwidth.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":172,"context_line":""},{"line_number":173,"context_line":"    As these extra specs affect the quantitative resources consumed by the"},{"line_number":174,"context_line":"    instance, nova will not support equivalent properties for cache in the"},{"line_number":175,"context_line":"    image metadata. As memory limits are not quantitative and do not consume"},{"line_number":176,"context_line":"    resources they can optionally be supported in the image but that is left"},{"line_number":177,"context_line":"    to the implementation to decide."},{"line_number":178,"context_line":""},{"line_number":179,"context_line":".. note::"},{"line_number":180,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_8a2f70f4","line":177,"range":{"start_line":175,"start_character":20,"end_line":177,"end_character":36},"updated":"2019-07-02 11:22:58.000000000","message":"Can we simply say out of scope of this spec instead?","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":172,"context_line":""},{"line_number":173,"context_line":"    As these extra specs affect the quantitative resources consumed by the"},{"line_number":174,"context_line":"    instance, nova will not support equivalent properties for cache in the"},{"line_number":175,"context_line":"    image metadata. As memory limits are not quantitative and do not consume"},{"line_number":176,"context_line":"    resources they can optionally be supported in the image but that is left"},{"line_number":177,"context_line":"    to the implementation to decide."},{"line_number":178,"context_line":""},{"line_number":179,"context_line":".. note::"},{"line_number":180,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_053b7337","line":177,"range":{"start_line":175,"start_character":20,"end_line":177,"end_character":36},"in_reply_to":"9fb8cfa7_8a2f70f4","updated":"2019-07-02 13:03:55.000000000","message":"sure im fine with doing this in the flavor only. ill update","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":231,"context_line":"    region."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":235,"context_line":"with an inventory of CoS policies."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":238,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_8a18d0d2","line":235,"range":{"start_line":234,"start_character":40,"end_line":235,"end_character":34},"updated":"2019-07-02 11:22:58.000000000","message":"So just to be sure. Is this memory controller RP in 1:1 relationship to the NUMA RP but it is a different RP to avoid dependency to the NUMA in placement spec?","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":231,"context_line":"    region."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":235,"context_line":"with an inventory of CoS policies."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":238,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_05ae132d","line":235,"range":{"start_line":234,"start_character":40,"end_line":235,"end_character":34},"in_reply_to":"9fb8cfa7_8a18d0d2","updated":"2019-07-02 13:03:55.000000000","message":"yes a Numa node is really just a slang term for a memroy controler + the resouces that have phyical memory that is managed by that memory contoler.\n\nit is a sperate RP to have no dependency on the NUMA in placement spec but the intent that we will just reshape the invetory/allocations onto the numa node RP whenever that gets created.\n\nby tracking it as separate RPs from the start we wont have issue with incorrect accounting from a numa affinity perspective.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"f663d0887473fd27dff94f7bd620a6a06160d530","unresolved":false,"context_lines":[{"line_number":234,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":235,"context_line":"with an inventory of CoS policies."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":238,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"},{"line_number":239,"context_line":"filter to ensure the NUMA affinity of the cache memory bandwidth, guest memory"},{"line_number":240,"context_line":"and guest CPUs."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_629fca01","line":237,"updated":"2019-07-02 06:34:13.000000000","message":"I guess you also need to extend the InstanceNumaCell object to store the request.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":234,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":235,"context_line":"with an inventory of CoS policies."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":238,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"},{"line_number":239,"context_line":"filter to ensure the NUMA affinity of the cache memory bandwidth, guest memory"},{"line_number":240,"context_line":"and guest CPUs."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_050a9324","line":237,"in_reply_to":"9fb8cfa7_629fca01","updated":"2019-07-02 13:03:55.000000000","message":"im not sure if i do.\n\nwe dont need to actully assing specifc cacheway to the guest so the isntace numa toployg likely does not need to be modifyed. the request for the resouces will be in the request spec and the the rest will be stored in the placmenet allocation.\n\nthere may be a reason to store it in the instnace numa toplogy but i dont know what that is thinking about it quickly.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"f663d0887473fd27dff94f7bd620a6a06160d530","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"- ``L3_DATA_CACHE_WAY``"},{"line_number":254,"context_line":"- ``L3_CODE_CACHE_WAY``"},{"line_number":255,"context_line":"- ``L3_UNIFIED_CACHE_WAY``"},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":258,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_0205ae45","line":255,"updated":"2019-07-02 06:34:13.000000000","message":"Sean, I\u0027m still stuck on the fragmental issue. Do we want to the virt driver raise exception when facing fragmental issue, then asking the scheduler to do a rescheduling? and expecting the libvirt resolve that problem in the future.\n\nOr we should use fixed size in the nova. This will lose some flexible.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"- ``L3_DATA_CACHE_WAY``"},{"line_number":254,"context_line":"- ``L3_CODE_CACHE_WAY``"},{"line_number":255,"context_line":"- ``L3_UNIFIED_CACHE_WAY``"},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":258,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_a019357a","line":255,"in_reply_to":"9fb8cfa7_0205ae45","updated":"2019-07-02 13:03:55.000000000","message":"the intent is to use a fixed size per host and manage it by chanign the step size in placement.\n\nso fragmentation will be avoided the same way as it was in the vpmem spec.\n\nin this case im not adding a config option to set this and am instead saying use the placement api directly to manage this but we could have a config for this if perople prefer.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"ed462c3bf201a4065ddb4693924b18bd01fdc85b","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"- ``L3_DATA_CACHE_WAY``"},{"line_number":254,"context_line":"- ``L3_CODE_CACHE_WAY``"},{"line_number":255,"context_line":"- ``L3_UNIFIED_CACHE_WAY``"},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":258,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_bb42bfaa","line":255,"in_reply_to":"9fb8cfa7_101f2a5c","updated":"2019-07-02 17:26:15.000000000","message":"yes we could use providers.yaml if its implemented this cycle. im not opposed to that but i don\u0027t want to make it a requirement.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3df58643ffc3f4aa6fc8511000cd3e16802a5dbd","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"- ``L3_DATA_CACHE_WAY``"},{"line_number":254,"context_line":"- ``L3_CODE_CACHE_WAY``"},{"line_number":255,"context_line":"- ``L3_UNIFIED_CACHE_WAY``"},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":258,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_101f2a5c","line":255,"in_reply_to":"9fb8cfa7_4a8c132e","updated":"2019-07-02 16:07:29.000000000","message":"Or providers.yaml...","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7ebf07905850536bba4b0a2510dc99ca5aa0051f","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"- ``L3_DATA_CACHE_WAY``"},{"line_number":254,"context_line":"- ``L3_CODE_CACHE_WAY``"},{"line_number":255,"context_line":"- ``L3_UNIFIED_CACHE_WAY``"},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":258,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_4a8c132e","line":255,"in_reply_to":"9fb8cfa7_a019357a","updated":"2019-07-02 15:25:28.000000000","message":"we discussed this some more on irc.\n\nto avoid fragmentation we need the step_size min_allocation and max allocation to all be equal to each other we also said a new config option for this rather then relying only on peole using the placement api is better so ill add that.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":312,"context_line":".. note::"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"    The number of supported CoS policies is specified by the maxAllocs field."},{"line_number":315,"context_line":"    The minium allcoation size is specified in the min field of the control"},{"line_number":316,"context_line":"    element and if not set is equal to the granularity attribute."},{"line_number":317,"context_line":"    The minium allocation size will be reported as the min_unit in the"},{"line_number":318,"context_line":"    placement inventory."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_aa4cb4c7","line":315,"range":{"start_line":315,"start_character":15,"end_line":315,"end_character":25},"updated":"2019-07-02 11:22:58.000000000","message":"nit:allocation","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e152c52e78ee5d4d4485ebe4b6c35aca83563fb5","unresolved":false,"context_lines":[{"line_number":343,"context_line":""},{"line_number":344,"context_line":".. todo::"},{"line_number":345,"context_line":""},{"line_number":346,"context_line":"    Add ascii diagram of resouce provider tree for a sample typical host."},{"line_number":347,"context_line":""},{"line_number":348,"context_line":""},{"line_number":349,"context_line":"REST API impact"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_6af1dc6a","line":346,"updated":"2019-07-02 11:22:58.000000000","message":"+1","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"f663d0887473fd27dff94f7bd620a6a06160d530","unresolved":false,"context_lines":[{"line_number":441,"context_line":"  :Description: Comma-separated list of reserved cacheways per NUMA node."},{"line_number":442,"context_line":"    For example, ``reserved_cacheways\u003d2,2``"},{"line_number":443,"context_line":""},{"line_number":444,"context_line":"``[scheduler] pqos_prefilter``"},{"line_number":445,"context_line":"  :Type: bool"},{"line_number":446,"context_line":"  :Default: False"},{"line_number":447,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"},{"line_number":448,"context_line":"    memory bandwidth extra specs into placement requests."},{"line_number":449,"context_line":""},{"line_number":450,"context_line":"Developer impact"},{"line_number":451,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_8263de41","line":448,"range":{"start_line":444,"start_character":0,"end_line":448,"end_character":57},"updated":"2019-07-02 06:34:13.000000000","message":"I\u0027m not sure we need this option. We can enable it all the times. Why people want to disable it, if the user doesn\u0027t request cache, it won\u0027t do anything.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7aeeed6e8309c21ee770d8a184e7a68c741062c1","unresolved":false,"context_lines":[{"line_number":441,"context_line":"  :Description: Comma-separated list of reserved cacheways per NUMA node."},{"line_number":442,"context_line":"    For example, ``reserved_cacheways\u003d2,2``"},{"line_number":443,"context_line":""},{"line_number":444,"context_line":"``[scheduler] pqos_prefilter``"},{"line_number":445,"context_line":"  :Type: bool"},{"line_number":446,"context_line":"  :Default: False"},{"line_number":447,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"},{"line_number":448,"context_line":"    memory bandwidth extra specs into placement requests."},{"line_number":449,"context_line":""},{"line_number":450,"context_line":"Developer impact"},{"line_number":451,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_c04c6940","line":448,"range":{"start_line":444,"start_character":0,"end_line":448,"end_character":57},"in_reply_to":"9fb8cfa7_8263de41","updated":"2019-07-02 13:03:55.000000000","message":"if that is what people prefer then i can remvoe.\nall the other prefilters have config options i think.\n\ni originally did not have the per host\ncache_allocation_support, memory_bandwidth_limiting config options in an earlier draft of this before i pushed it up to gerrit. in that version this was needed for upgrades but in its current form this is not needed.\n\n\nthere is still one open question on if we should support mixing cache affined/memory limited guest with those without cache/memory alloction on the same host. if we decide not to support mixing on the same host in train, which i think i would prefer at least for the initial release, if people dont request cache or memory bandwidth the prefilter would add a forbiden trait to the request.\n\nif we allow mixing on the same host and rely on the undocuemnted behavior in libvirt that actully will do the right thing then the prefilter would do nothing if you dont request cache allocation or memory limits.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"16f71f5158e461e6fc250798d603b8639221d751","unresolved":false,"context_lines":[{"line_number":498,"context_line":"  sean-k-mooney?"},{"line_number":499,"context_line":""},{"line_number":500,"context_line":"Other contributors:"},{"line_number":501,"context_line":"  sean-k-mooney?"},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"Work Items"},{"line_number":504,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_5b294887","line":501,"range":{"start_line":501,"start_character":2,"end_line":501,"end_character":16},"updated":"2019-07-02 13:39:45.000000000","message":"As discussed downstream I\u0027m happy to help with this for what remains of Train.","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7ebf07905850536bba4b0a2510dc99ca5aa0051f","unresolved":false,"context_lines":[{"line_number":498,"context_line":"  sean-k-mooney?"},{"line_number":499,"context_line":""},{"line_number":500,"context_line":"Other contributors:"},{"line_number":501,"context_line":"  sean-k-mooney?"},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"Work Items"},{"line_number":504,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9fb8cfa7_8a9beb71","line":501,"range":{"start_line":501,"start_character":2,"end_line":501,"end_character":16},"in_reply_to":"9fb8cfa7_5b294887","updated":"2019-07-02 15:25:28.000000000","message":"yep ill add you in the next version","commit_id":"e292a1c95527f0dc5d5a6b2990cc14de73277605"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":83,"context_line":"Four new config options will be added to the libvirt virt driver, two to"},{"line_number":84,"context_line":"enable support for memory limits and cache allocation, and one to declare the"},{"line_number":85,"context_line":"reserved number of cacheways for the host per cache region/NUMA node,"},{"line_number":86,"context_line":"and one to contol the allocation size in placement."},{"line_number":87,"context_line":""},{"line_number":88,"context_line":".. code::"},{"line_number":89,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_1022c502","line":86,"range":{"start_line":86,"start_character":11,"end_line":86,"end_character":17},"updated":"2019-07-02 22:55:02.000000000","message":"control","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":90,"context_line":"  [libvirt]"},{"line_number":91,"context_line":"  cache_allocation_support\u003dtrue  # true|false default(false)"},{"line_number":92,"context_line":"  memory_bandwidth_limiting\u003dtrue  # true|false default(false)"},{"line_number":93,"context_line":"  reserved_cacheways\u003d2,0  #reserved cacheways per NUMA node"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"The detail of the config parameters are specified in the \"Other deployer"},{"line_number":96,"context_line":"impact\" section of this spec."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_308029c0","line":93,"updated":"2019-07-02 22:55:02.000000000","message":"nts: where\u0027s the fourth one (allocation size)?","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":90,"context_line":"  [libvirt]"},{"line_number":91,"context_line":"  cache_allocation_support\u003dtrue  # true|false default(false)"},{"line_number":92,"context_line":"  memory_bandwidth_limiting\u003dtrue  # true|false default(false)"},{"line_number":93,"context_line":"  reserved_cacheways\u003d2,0  #reserved cacheways per NUMA node"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"The detail of the config parameters are specified in the \"Other deployer"},{"line_number":96,"context_line":"impact\" section of this spec."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_7328ebed","line":93,"in_reply_to":"9fb8cfa7_308029c0","updated":"2019-07-03 00:57:06.000000000","message":"on its in it in the Other deployer impact section below but ya i should provide an example here","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":97,"context_line":""},{"line_number":98,"context_line":"Two new compute capability traits will be added to indicate if a host supports"},{"line_number":99,"context_line":"cache allocation or memory bandwidth limits based on the values in the config"},{"line_number":100,"context_line":"options. These new traits can be used in combination with a new request"},{"line_number":101,"context_line":"prefilter to ensure that we do not mix cache allocated instances and non cache"},{"line_number":102,"context_line":"allocated instances on the same host. Similarly the same will be done for"},{"line_number":103,"context_line":"memory limits. This behaviour will be configurable via the scheduler section"},{"line_number":104,"context_line":"of the nova.conf"},{"line_number":105,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_b073b916","line":102,"range":{"start_line":100,"start_character":9,"end_line":102,"end_character":37},"updated":"2019-07-02 22:55:02.000000000","message":"This is not needed. Use forbidden aggregates if you want to isolate.\n\n[Later] or make isolation optional in the request filter for this -- see below.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":30001,"name":"Ivens Zambrano","email":"ivens.zambrano@intel.com","username":"IvensZambrano"},"change_message_id":"78a6cd619d33b60ba95fee447850c667c05be1c6","unresolved":false,"context_lines":[{"line_number":120,"context_line":"    It is considered out of scope of this spec to define how the resource"},{"line_number":121,"context_line":"    providers are created when cache and memory bandwidth are manged"},{"line_number":122,"context_line":"    externally. They could be created manually using ``osc-placement``,"},{"line_number":123,"context_line":"    statically via a YAML file + additional change to the compute agent"},{"line_number":124,"context_line":"    or dynamically via the external agent directly calling the placement API."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Two new flavor extra specs will be introduced to enable requesting bandwidth"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_20243dc6","line":123,"range":{"start_line":123,"start_character":4,"end_line":123,"end_character":71},"updated":"2019-07-05 19:39:06.000000000","message":"Reference to spec https://review.opendev.org/#/c/612497/ may be needed here or with a pointer to the references section","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"41084019e45193cc55326b251e77d501a700ef2b","unresolved":false,"context_lines":[{"line_number":120,"context_line":"    It is considered out of scope of this spec to define how the resource"},{"line_number":121,"context_line":"    providers are created when cache and memory bandwidth are manged"},{"line_number":122,"context_line":"    externally. They could be created manually using ``osc-placement``,"},{"line_number":123,"context_line":"    statically via a YAML file + additional change to the compute agent"},{"line_number":124,"context_line":"    or dynamically via the external agent directly calling the placement API."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Two new flavor extra specs will be introduced to enable requesting bandwidth"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_1fb291e7","line":123,"range":{"start_line":123,"start_character":4,"end_line":123,"end_character":71},"in_reply_to":"7faddb67_20243dc6","updated":"2019-07-09 18:57:01.000000000","message":"well the intent was not to have a depency on that but i can add it as a referece.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"f1680bd367cf31e1266a3a401cd512f52856d96f","unresolved":false,"context_lines":[{"line_number":133,"context_line":""},{"line_number":134,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":135,"context_line":"    hw:numa_cache\u003d4"},{"line_number":136,"context_line":"    hw:numa_bandwidth\u003d10"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"In this example the cache is divided evenly across each numa node so we"},{"line_number":139,"context_line":"request 4 cache ways total to result in 2 cache ways per numa node."}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_31033945","line":136,"updated":"2019-07-05 06:18:22.000000000","message":"nit, hw:cpu_policy\u003ddedicated","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"f1680bd367cf31e1266a3a401cd512f52856d96f","unresolved":false,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"    flavor.cpu\u003d8"},{"line_number":147,"context_line":"    flavor.ram\u003d4096"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"    hw:numa_nodes\u003d2"},{"line_number":150,"context_line":"    hw:numa_cpus.0\u003d2"},{"line_number":151,"context_line":"    hw:numa_cpus.1\u003d6"}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_51003535","line":148,"updated":"2019-07-05 06:18:22.000000000","message":"nit, hw:cpu_policy\u003ddedicated","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":173,"context_line":"    and ``hw:numa_data_cache`` will be used instead of ``hw:numa_cache``. While"},{"line_number":174,"context_line":"    code and data partitioning (CDP) can be enabled at runtime, The state of"},{"line_number":175,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":176,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":177,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":178,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":179,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":180,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_d0f74d59","line":177,"range":{"start_line":176,"start_character":4,"end_line":177,"end_character":78},"updated":"2019-07-02 22:55:02.000000000","message":"This still makes me uncomfortable. Guests that don\u0027t care about CDP will only be able to land on hosts that don\u0027t expose it. Is that acceptable?\n\nDo we have a real use case for CDP? Could we maybe support only the unified version for the first pass?","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"41084019e45193cc55326b251e77d501a700ef2b","unresolved":false,"context_lines":[{"line_number":173,"context_line":"    and ``hw:numa_data_cache`` will be used instead of ``hw:numa_cache``. While"},{"line_number":174,"context_line":"    code and data partitioning (CDP) can be enabled at runtime, The state of"},{"line_number":175,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":176,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":177,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":178,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":179,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":180,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_9f48c1cc","line":177,"range":{"start_line":176,"start_character":4,"end_line":177,"end_character":78},"in_reply_to":"7faddb67_ca822750","updated":"2019-07-09 18:57:01.000000000","message":"if you enable CDP you cant request cache allocation of unifed cache but you can run worklaods that dont request any cache allocation at all yes.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":30001,"name":"Ivens Zambrano","email":"ivens.zambrano@intel.com","username":"IvensZambrano"},"change_message_id":"78a6cd619d33b60ba95fee447850c667c05be1c6","unresolved":false,"context_lines":[{"line_number":173,"context_line":"    and ``hw:numa_data_cache`` will be used instead of ``hw:numa_cache``. While"},{"line_number":174,"context_line":"    code and data partitioning (CDP) can be enabled at runtime, The state of"},{"line_number":175,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":176,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":177,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":178,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":179,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":180,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_ca822750","line":177,"range":{"start_line":176,"start_character":4,"end_line":177,"end_character":78},"in_reply_to":"9fb8cfa7_b34c23c9","updated":"2019-07-05 19:39:06.000000000","message":"In tat case you may not need to prevent non CDP workloads to land on that node, they won\u0027t use, it will be a waste, but the workload will be able to work as if it was on a non CDP node. right?","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":173,"context_line":"    and ``hw:numa_data_cache`` will be used instead of ``hw:numa_cache``. While"},{"line_number":174,"context_line":"    code and data partitioning (CDP) can be enabled at runtime, The state of"},{"line_number":175,"context_line":"    CDP is not expected to change if the host has any allocation in placment."},{"line_number":176,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":177,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":178,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":179,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":180,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_b34c23c9","line":177,"range":{"start_line":176,"start_character":4,"end_line":177,"end_character":78},"in_reply_to":"9fb8cfa7_d0f74d59","updated":"2019-07-03 00:57:06.000000000","message":"well there are two ways to control it\nfirst you have to enable cdp in the bios or using the runtime sysfs parameter. unless you opt into enabling it on the host it will present a unified cache.\n\nsecond you will have to also enable cat reporting in the nova.conf on the compute node. \n\nso you have to opt in twice for us to block you booting instance that dont care about cdp on a host that supports it.\n\nto your second question, the donwstream freature reqquest does not mention CDP at all. its just requestion support of CAT/RDT for a vRAN usecase so for V1 i.e. in train \ni think basic CAT support wihtout CDP would be ok if you would prefer to defer CDP.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":176,"context_line":"    Hosts with cache allocation enabled will either report inventories"},{"line_number":177,"context_line":"    of ``L3_CACHE`` or inventories of ``L3_DATA_CACHE`` and ``L3_CODE_CACHE``."},{"line_number":178,"context_line":"    ``hw:numa_bandwidth`` is spcified in percent of host numa node bandwidth"},{"line_number":179,"context_line":"    and is not devided by the numer of numa nodes."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":".. note::"},{"line_number":182,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_90c9d526","line":179,"range":{"start_line":179,"start_character":30,"end_line":179,"end_character":35},"updated":"2019-07-02 22:55:02.000000000","message":"number","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":216,"context_line":"Data model impact"},{"line_number":217,"context_line":"-----------------"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"The libvirt driver will be extended to create one nested RP per l3 cache"},{"line_number":220,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":221,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":222,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":223,"context_line":"Service (CoS) policies which will be allocated to instances."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_306a69ed","line":220,"range":{"start_line":219,"start_character":46,"end_line":220,"end_character":6},"updated":"2019-07-02 22:55:02.000000000","message":"Still no. This does no good until CPUs are modeled under NUMA RPs.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"41084019e45193cc55326b251e77d501a700ef2b","unresolved":false,"context_lines":[{"line_number":216,"context_line":"Data model impact"},{"line_number":217,"context_line":"-----------------"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"The libvirt driver will be extended to create one nested RP per l3 cache"},{"line_number":220,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":221,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":222,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":223,"context_line":"Service (CoS) policies which will be allocated to instances."}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_ffc69549","line":220,"range":{"start_line":219,"start_character":46,"end_line":220,"end_character":6},"in_reply_to":"7faddb67_b42dd747","updated":"2019-07-09 18:57:01.000000000","message":"right we will need to do cache aware numa pinning which i would like to do this cycle if we proceed with this spec and make it depen ont the placment nested magic spec.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":216,"context_line":"Data model impact"},{"line_number":217,"context_line":"-----------------"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"The libvirt driver will be extended to create one nested RP per l3 cache"},{"line_number":220,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":221,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":222,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":223,"context_line":"Service (CoS) policies which will be allocated to instances."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_73ed4b8d","line":220,"range":{"start_line":219,"start_character":46,"end_line":220,"end_character":6},"in_reply_to":"9fb8cfa7_306a69ed","updated":"2019-07-03 00:57:06.000000000","message":"cache regions may not align with numa node in all cases.\nthe do today but i was told but intels silicon team to not depend on that. i also think on the amd with ther ccx sub modules and a sperate io die we will have a non liniar mapping of cache regions to numa nodes so even with cpus modeled per numa node we may  or may not want the cache inventires to be on a numa node and might still want to keep them as seperate nested inventories.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"f1680bd367cf31e1266a3a401cd512f52856d96f","unresolved":false,"context_lines":[{"line_number":216,"context_line":"Data model impact"},{"line_number":217,"context_line":"-----------------"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"The libvirt driver will be extended to create one nested RP per l3 cache"},{"line_number":220,"context_line":"region. PQoS and CAT in particular works by restricting the cacheways a"},{"line_number":221,"context_line":"process can write to but allows reads from all cacheways on the processor."},{"line_number":222,"context_line":"The libvirt driver will represent inventories of cacheways and Class of"},{"line_number":223,"context_line":"Service (CoS) policies which will be allocated to instances."}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_b42dd747","line":220,"range":{"start_line":219,"start_character":46,"end_line":220,"end_character":6},"in_reply_to":"9fb8cfa7_73ed4b8d","updated":"2019-07-05 06:18:22.000000000","message":"actually, if we have mulitple caches region for one numa node in the future, then if nova doesn\u0027t aware the cpu pinning based on the cache region, it is meanless also.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":235,"context_line":"    When CoD is disabled the cpu cache and physical memory connected to a cpu"},{"line_number":236,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":237,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":238,"context_line":"    partitioned into 1 numa node and 1 cache region per memory controller. As"},{"line_number":239,"context_line":"    the 1:1 mapping between cache region and numa node may change in the"},{"line_number":240,"context_line":"    future with new hardware, cache inventories will be tracked per cache"},{"line_number":241,"context_line":"    region."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_3043496f","line":241,"range":{"start_line":238,"start_character":75,"end_line":241,"end_character":11},"updated":"2019-07-02 22:55:02.000000000","message":"Yup, great idea - for later, when there\u0027s a reason to track them separately. Otherwise you\u0027re just adding complexity for no benefit.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":235,"context_line":"    When CoD is disabled the cpu cache and physical memory connected to a cpu"},{"line_number":236,"context_line":"    socket is reported as a single numa node and the L3 cache is reported as"},{"line_number":237,"context_line":"    single cache region per socket. When CoD is enabled memory and cache are"},{"line_number":238,"context_line":"    partitioned into 1 numa node and 1 cache region per memory controller. As"},{"line_number":239,"context_line":"    the 1:1 mapping between cache region and numa node may change in the"},{"line_number":240,"context_line":"    future with new hardware, cache inventories will be tracked per cache"},{"line_number":241,"context_line":"    region."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_e60f3b2d","line":241,"range":{"start_line":238,"start_character":75,"end_line":241,"end_character":11},"in_reply_to":"9fb8cfa7_3043496f","updated":"2019-07-03 00:57:06.000000000","message":"well i pretty sure we need this for AMD\u0027s ZEN 2 based  Rome server chips that were detailed at compute tex last year are are expect later this year. i know the destop version are launching on july 7th im not sure if the launch date other then Q2 2019 has been annouched for the rome chips.\n\nif you look at the process that was shown at compute text last month https://images.anandtech.com/doci/14525/amd_rome-678_678x452.png\n\nyou will see that the is a singel large die in the midel and 8 smaller dies.\n\nthe central die is the io die which has the memory controlers and pci controlers and other cpu supproted perfiferals.\n\nhttps://www.semiaccurate.com/assets/uploads/2018/11/AMD_Rome_layout.jpg\n\nthe 8 smaller dies are the CCXs or  core complexs\n\nhttps://image-private.slidesharecdn.com/nexthorizongamingtechdaygeneralsessionmarkpapermaster-190619185900/95/slide-12-1024.jpg?hdnea\u003dacl\u003d/nexthorizongamingtechdaygeneralsessionmarkpapermaster-190619185900/95/slide-12-1024.jpg*~exp\u003d1562111281~hmac\u003d019dfbc7e93304e4760e12677f6569f34596d6d5c41b530f8c48f11f335449bf\u0026cb\u003d1560971064\n\neach ccx has a 4  cpu cores each of which has a dedicated l1 and l2 cache and  there is an l3 cache that is shared. \n\nhttps://www.techpowerup.com/img/9hHbdiSLzqzOCNgx.jpg\n\nhttps://images.anandtech.com/doci/14525/Mike_Clark-Next_Horizon_Gaming-CPU_Architecture_06092019-page-010.jpg\n\nto use cache form a different ccx i belive the core has to communicate via the io die as i dont think ccx\u0027s can communicate directly so there will be higher latency.\n\nwhile intel has similar featrures  with its mesh architecture intel has 1 memory controler per l3 cahce region but i dont think that will be the case for amd and i dont think we shoudl assume intel wont change that in the futrue if they determin there is a better way to do it.\n\non the previous generation of amd eypc cpus\n\nhttps://www.servethehome.com/wp-content/uploads/2017/08/AMD-EPYC-Infinity-Fabric-Topology-Mapping.jpg\n\nthe io  and memory contolers are distibuted to each of the 4 dies so each die was a numa node and l3 cache region with an infintiy fabric interconnect between the dies.\nif you squint a little it s similar to what intel used to do before it move to the purley architecture.\n\nso im trying to make sure we can supprot zen2 when it launches.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":241,"context_line":"    region."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":245,"context_line":"with an inventory of CoS policies."},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":248,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_130def34","line":245,"range":{"start_line":244,"start_character":36,"end_line":245,"end_character":33},"updated":"2019-07-02 22:55:02.000000000","message":"Okay, *really* no.\n\nFor a minute as I read below I was thinking there might actually be merit in using nested providers if the COS policy needs to be consumed from the same provider (or subtree) as the cache. But having it be in a separate RP that\u0027s not tied to the corresponding cache in any way? That makes no sense.\n\nThe only way I could possibly be on board with using nested providers right now (rather than waiting until the VCPU/PCPU/MEMORY_MB is also split) is if you had one RP per NUMA node that housed all the cache and COS inventories for that whole NUMA node. That would let you ensure *part of* the affinity model directly by putting the cache and corresponding COS in the same request group.\n\nA future reshape would split the cache regions into children *of the NUMA node RP*, but only once the placement-side affinity magic is supported.\n\nBut again, I feel pretty strongly that we shouldn\u0027t be trying to use nested providers at all at this stage.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":241,"context_line":"    region."},{"line_number":242,"context_line":""},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"The libvirt driver will also create one RP per memory controller (numa node)"},{"line_number":245,"context_line":"with an inventory of CoS policies."},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"The ``HostCell`` object will be extended to maintain a list of placement"},{"line_number":248,"context_line":"resource providers UUIDs per host cell. This will be used by the NUMA topology"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_4643a714","line":245,"range":{"start_line":244,"start_character":36,"end_line":245,"end_character":33},"in_reply_to":"9fb8cfa7_130def34","updated":"2019-07-03 00:57:06.000000000","message":"well the cache and the cache CoS inventories would be in the same RP the memory would be in a seperate RP.\n\nthe memory RP really is a numa node but the cache region RP are not. see my answer above related to the upcoming z2 base amd eypc cpus.\n\ni shoudl proably say that cache allocation should always be form the local cache region and define that as cache affinity\n\nand use the term \"numa affinity\" only for affinity to the memory controller.\n\nif you think we will land the placement support for numa soon enough to use it for this feature in train im happy to say we will use that for the memory bandiwt rp.\n\ni think we will end up wanted to model cpu inventores and cpu cache inventories in the same RP which will be child RP of the numa node RP that contains hugepage memory and the 4k memroy e.g. MEMORY_MB that is not numa ffined will like stay on the root compute node RP.\n\nthat is jsut a guess but i think we will need a 3 level tree to model this correctly in placmenet assuming we dont jsut do the nuam and cache affintiy checks in nova.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":260,"context_line":""},{"line_number":261,"context_line":"This spec proposes introducing three new standard resource classes:"},{"line_number":262,"context_line":""},{"line_number":263,"context_line":"- ``L3_DATA_CACHE_WAY``"},{"line_number":264,"context_line":"- ``L3_CODE_CACHE_WAY``"},{"line_number":265,"context_line":"- ``L3_UNIFIED_CACHE_WAY``"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":268,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_105ac54f","line":265,"range":{"start_line":263,"start_character":0,"end_line":265,"end_character":26},"updated":"2019-07-02 22:55:02.000000000","message":"These don\u0027t match L177","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":260,"context_line":""},{"line_number":261,"context_line":"This spec proposes introducing three new standard resource classes:"},{"line_number":262,"context_line":""},{"line_number":263,"context_line":"- ``L3_DATA_CACHE_WAY``"},{"line_number":264,"context_line":"- ``L3_CODE_CACHE_WAY``"},{"line_number":265,"context_line":"- ``L3_UNIFIED_CACHE_WAY``"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"The units of the CACHE resource classes will be \"cache ways\". Each processor"},{"line_number":268,"context_line":"cache is divided into a number of cache ways each of which is approximately"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_665d8beb","line":265,"range":{"start_line":263,"start_character":0,"end_line":265,"end_character":26},"in_reply_to":"9fb8cfa7_105ac54f","updated":"2019-07-03 00:57:06.000000000","message":"your right i update this to *cache_way as you asked but i missed updating 177 ill fix that in the next version.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":270,"context_line":"number of CoS polices is reduced by half, as this is hardware dependent the"},{"line_number":271,"context_line":"values will be transposed directly from the data reported by libvirt."},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"In addition to the resource classes, above two additional resources class will"},{"line_number":274,"context_line":"be added to model the number of \"Class of Service (COS)\" or"},{"line_number":275,"context_line":"\"Quality of Service (QoS)\" policies supported by the CPU."},{"line_number":276,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_702ea1ac","line":273,"range":{"start_line":273,"start_character":35,"end_line":273,"end_character":42},"updated":"2019-07-02 22:55:02.000000000","message":"above,","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":270,"context_line":"number of CoS polices is reduced by half, as this is hardware dependent the"},{"line_number":271,"context_line":"values will be transposed directly from the data reported by libvirt."},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"In addition to the resource classes, above two additional resources class will"},{"line_number":274,"context_line":"be added to model the number of \"Class of Service (COS)\" or"},{"line_number":275,"context_line":"\"Quality of Service (QoS)\" policies supported by the CPU."},{"line_number":276,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_303829ed","line":273,"range":{"start_line":273,"start_character":68,"end_line":273,"end_character":73},"updated":"2019-07-02 22:55:02.000000000","message":"classes","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":281,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"},{"line_number":282,"context_line":"to exhaust the number of COS policies available so they must be tracked as"},{"line_number":283,"context_line":"consumable resources. The ``CACHE_COS`` and ``MEMORY_BANDWIDTH_COS`` resource"},{"line_number":284,"context_line":"classes are unitless as they represent the cardinality of CoS polices"},{"line_number":285,"context_line":"supported by the CPU package and is therefore a dimensionless quantity."},{"line_number":286,"context_line":""},{"line_number":287,"context_line":".. code-block:: XML"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_d028ad98","line":284,"range":{"start_line":284,"start_character":12,"end_line":284,"end_character":20},"updated":"2019-07-02 22:55:02.000000000","message":"Not really, they track \"number of policies\". As such, IMO we should suffix with _POLICY\n\n CACHE_COS_POLICY\n MEMORY_BANDWIDTH_COS_POLICY","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":281,"context_line":"the order of 8-16 COS polices, e.g. low double figures. As such it is possible"},{"line_number":282,"context_line":"to exhaust the number of COS policies available so they must be tracked as"},{"line_number":283,"context_line":"consumable resources. The ``CACHE_COS`` and ``MEMORY_BANDWIDTH_COS`` resource"},{"line_number":284,"context_line":"classes are unitless as they represent the cardinality of CoS polices"},{"line_number":285,"context_line":"supported by the CPU package and is therefore a dimensionless quantity."},{"line_number":286,"context_line":""},{"line_number":287,"context_line":".. code-block:: XML"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_662b4b33","line":284,"range":{"start_line":284,"start_character":12,"end_line":284,"end_character":20},"in_reply_to":"9fb8cfa7_d028ad98","updated":"2019-07-03 00:57:06.000000000","message":"ok i can add a policy sufix","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":30001,"name":"Ivens Zambrano","email":"ivens.zambrano@intel.com","username":"IvensZambrano"},"change_message_id":"78a6cd619d33b60ba95fee447850c667c05be1c6","unresolved":false,"context_lines":[{"line_number":284,"context_line":"classes are unitless as they represent the cardinality of CoS polices"},{"line_number":285,"context_line":"supported by the CPU package and is therefore a dimensionless quantity."},{"line_number":286,"context_line":""},{"line_number":287,"context_line":".. code-block:: XML"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"    \u003c!-- Cache elements with CDP disabled --\u003e"},{"line_number":290,"context_line":"    \u003ccache\u003e"},{"line_number":291,"context_line":"        \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002733\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270\u0027\u003e"},{"line_number":292,"context_line":"            \u003ccontrol granularity\u003d\u00273\u0027 unit\u003d\u0027MiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u002716\u0027/\u003e"},{"line_number":293,"context_line":"        \u003c/bank\u003e"},{"line_number":294,"context_line":"    \u003c/cache\u003e"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":".. code-block:: XML"},{"line_number":297,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_33c9f677","line":294,"range":{"start_line":287,"start_character":0,"end_line":294,"end_character":12},"updated":"2019-07-05 19:39:06.000000000","message":"where are these policies stored?? is part of the nova node artifacts?","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"41084019e45193cc55326b251e77d501a700ef2b","unresolved":false,"context_lines":[{"line_number":284,"context_line":"classes are unitless as they represent the cardinality of CoS polices"},{"line_number":285,"context_line":"supported by the CPU package and is therefore a dimensionless quantity."},{"line_number":286,"context_line":""},{"line_number":287,"context_line":".. code-block:: XML"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"    \u003c!-- Cache elements with CDP disabled --\u003e"},{"line_number":290,"context_line":"    \u003ccache\u003e"},{"line_number":291,"context_line":"        \u003cbank id\u003d\u00270\u0027 level\u003d\u00273\u0027 type\u003d\u0027both\u0027 size\u003d\u002733\u0027 unit\u003d\u0027MiB\u0027 cpus\u003d\u00270\u0027\u003e"},{"line_number":292,"context_line":"            \u003ccontrol granularity\u003d\u00273\u0027 unit\u003d\u0027MiB\u0027 type\u003d\u0027both\u0027 maxAllocs\u003d\u002716\u0027/\u003e"},{"line_number":293,"context_line":"        \u003c/bank\u003e"},{"line_number":294,"context_line":"    \u003c/cache\u003e"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":".. code-block:: XML"},{"line_number":297,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_9fda0115","line":294,"range":{"start_line":287,"start_character":0,"end_line":294,"end_character":12},"in_reply_to":"7faddb67_33c9f677","updated":"2019-07-09 18:57:01.000000000","message":"this is coming for libvirt and it is derived direclty form the info reported in sysfs which comes form the bios/hardwared confiuration.\n\nso this is all configured out of band of nova/libvirt at the os/bios/uefi level we are just using the data that is reported.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":351,"context_line":"   This will allow operators to modify the stepsize and max_unit per host if"},{"line_number":352,"context_line":"   desired to allow larger allocations. A conf option will also be intoduced"},{"line_number":353,"context_line":"   to specify the granularity which will be used to set min_unit, max_unit"},{"line_number":354,"context_line":"   and step_size."},{"line_number":355,"context_line":""},{"line_number":356,"context_line":""},{"line_number":357,"context_line":"A compute node with 2 cache regions and 1 memory controller ::"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_53592733","line":354,"updated":"2019-07-02 22:55:02.000000000","message":"nit: I found this (since L287) confusing to read as it seems to jump back and forth between the cache and memory bandwidth. It might read better if you introduced the XML for cache, then described the cache aspects; then introduced the XML for memory bandwidth and described that.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":351,"context_line":"   This will allow operators to modify the stepsize and max_unit per host if"},{"line_number":352,"context_line":"   desired to allow larger allocations. A conf option will also be intoduced"},{"line_number":353,"context_line":"   to specify the granularity which will be used to set min_unit, max_unit"},{"line_number":354,"context_line":"   and step_size."},{"line_number":355,"context_line":""},{"line_number":356,"context_line":""},{"line_number":357,"context_line":"A compute node with 2 cache regions and 1 memory controller ::"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_2625535d","line":354,"in_reply_to":"9fb8cfa7_53592733","updated":"2019-07-03 00:57:06.000000000","message":"am ill address the other comments and then ill think about re re organising this to describe them serialy as you suggest.\n\nill need to take a step back and think about how to do that so it will take more time then the other changes.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":389,"context_line":"cacheways when CDP is enabled. In both cache the cache region will have"},{"line_number":390,"context_line":"an inventory of CoS polices."},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"The memory controller RP will only contian an inventory of memory bandwidth"},{"line_number":393,"context_line":"CoS policies. there will be 1 RP per numa node. In future versions of"},{"line_number":394,"context_line":"openstack the MEMORY_BANDWIDTH_COS inventory and allocations will be"},{"line_number":395,"context_line":"reshaped onto a NUMA node RP."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_133bafc8","line":392,"range":{"start_line":392,"start_character":35,"end_line":392,"end_character":42},"updated":"2019-07-02 22:55:02.000000000","message":"contain","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":395,"context_line":"reshaped onto a NUMA node RP."},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"The HW_CPU_L3_CACHE_ALLOCATION and HW_MEMORY_BANDWIDTH_LIMTS traits"},{"line_number":398,"context_line":"will be reported on the root RP not the nested RPs."},{"line_number":399,"context_line":""},{"line_number":400,"context_line":"REST API impact"},{"line_number":401,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_53ce07b5","line":398,"updated":"2019-07-02 22:55:02.000000000","message":"These should be named like compute capabilities (COMPUTE_*). But see below.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":395,"context_line":"reshaped onto a NUMA node RP."},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"The HW_CPU_L3_CACHE_ALLOCATION and HW_MEMORY_BANDWIDTH_LIMTS traits"},{"line_number":398,"context_line":"will be reported on the root RP not the nested RPs."},{"line_number":399,"context_line":""},{"line_number":400,"context_line":"REST API impact"},{"line_number":401,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_061ccf11","line":398,"in_reply_to":"9fb8cfa7_53ce07b5","updated":"2019-07-03 00:57:06.000000000","message":"im not sure about that as COMPUTE_* is used for virtualisation capablites not hardware capablites but i haven\u0027t read below yet so im not 100% against changing them to the compute namespace.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":418,"context_line":"``hw:numa_bandwidth``"},{"line_number":419,"context_line":"    Specifies the maximum memory bandwidth an instance can consume per NUMA"},{"line_number":420,"context_line":"    node. This is set as a percentage of the total host NUMA node bandwidth."},{"line_number":421,"context_line":"    The granularity of this depend on the host but is typically increments of"},{"line_number":422,"context_line":"    10% with a minimum limit of 10%."},{"line_number":423,"context_line":""},{"line_number":424,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_537287af","line":421,"range":{"start_line":421,"start_character":28,"end_line":421,"end_character":34},"updated":"2019-07-02 22:55:02.000000000","message":"depends","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":429,"context_line":"    all guest NUMA nodes. It is an error to use the standard form if the"},{"line_number":430,"context_line":"    minimum allocation of two cacheways per CoS is not possible or if the"},{"line_number":431,"context_line":"    total allocation is not an integer multiple of the number of NUMA nodes"},{"line_number":432,"context_line":"    specified by ``hw:numa_nodes``. Specifing resource requests directly"},{"line_number":433,"context_line":"    using the resource classes in the flavor is not supported."},{"line_number":434,"context_line":""},{"line_number":435,"context_line":"Security impact"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_b360a3d1","line":432,"range":{"start_line":432,"start_character":36,"end_line":432,"end_character":45},"updated":"2019-07-02 22:55:02.000000000","message":"Specifying","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":430,"context_line":"    minimum allocation of two cacheways per CoS is not possible or if the"},{"line_number":431,"context_line":"    total allocation is not an integer multiple of the number of NUMA nodes"},{"line_number":432,"context_line":"    specified by ``hw:numa_nodes``. Specifing resource requests directly"},{"line_number":433,"context_line":"    using the resource classes in the flavor is not supported."},{"line_number":434,"context_line":""},{"line_number":435,"context_line":"Security impact"},{"line_number":436,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_d35d1716","line":433,"range":{"start_line":433,"start_character":4,"end_line":433,"end_character":62},"updated":"2019-07-02 22:55:02.000000000","message":"++\n\nbut\n\nthat means you need to check for that explicitly and generate an error, which we don\u0027t have a precedent for","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":454,"context_line":"cache affinity which many degrade the performance of that filter. This spec"},{"line_number":455,"context_line":"also proposes the introduction of a prefilter to transform the flavor request"},{"line_number":456,"context_line":"into placement requests. This prefilter will be disabled by default. When"},{"line_number":457,"context_line":"enabled, it will add a forbidden trait via the root_required parameter when"},{"line_number":458,"context_line":"cache and/or bandwidth limits are not requested and will add a required trait"},{"line_number":459,"context_line":"and additional resource requests when they are. It is intended for this to"},{"line_number":460,"context_line":"increase scheduling performance by restricting the set of possible host by"},{"line_number":461,"context_line":"leveraging granular requests while also ensuring host are dynamically"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_93eb1f11","line":458,"range":{"start_line":457,"start_character":17,"end_line":458,"end_character":47},"updated":"2019-07-02 22:55:02.000000000","message":"As above, not on board with this. Means I can\u0027t land my normal NUMA-y VM on a host that exposes these resources. It should be up to the operator whether they want to isolate to that degree.\n\nI could be convinced that this could be controlled by (yet) another conf option. I grasp that it would be convenient to do it in the same request filter, since it\u0027s going to be written anyway.\n\nAlso, you imply that each trait will be controlled independently. So if I ask for cache but not membw, I will *only* land on hosts with ``cache_allocation_support\u003dTrue`` \u0026\u0026 ``memory_bandwidth_limiting\u003dFalse``? If I didn\u0027t think we were isolating too granularly already, this would definitely push me over the edge.\n\nHow about this:\n- A single trait (e.g. COMPUTE_NUMA_PQOS) which gets flipped on if *either* ``cache_allocation_support`` or ``memory_bandwidth_limiting`` is true.\n- A scheduler conf option ``pqos_isolate_hosts\u003d$bool``.\n- Iff ``pqos_isolate_hosts\u003dTrue``, the request filter can add root_required\u003d!COMPUTE_NUMA_PQOS whenever *none* of the five resource classes is requested.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":455,"context_line":"also proposes the introduction of a prefilter to transform the flavor request"},{"line_number":456,"context_line":"into placement requests. This prefilter will be disabled by default. When"},{"line_number":457,"context_line":"enabled, it will add a forbidden trait via the root_required parameter when"},{"line_number":458,"context_line":"cache and/or bandwidth limits are not requested and will add a required trait"},{"line_number":459,"context_line":"and additional resource requests when they are. It is intended for this to"},{"line_number":460,"context_line":"increase scheduling performance by restricting the set of possible host by"},{"line_number":461,"context_line":"leveraging granular requests while also ensuring host are dynamically"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_b36ae37e","line":458,"range":{"start_line":458,"start_character":57,"end_line":458,"end_character":77},"updated":"2019-07-02 22:55:02.000000000","message":"This isn\u0027t necessary; the resource requests are sufficient to ensure we land on a viable host.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":454,"context_line":"cache affinity which many degrade the performance of that filter. This spec"},{"line_number":455,"context_line":"also proposes the introduction of a prefilter to transform the flavor request"},{"line_number":456,"context_line":"into placement requests. This prefilter will be disabled by default. When"},{"line_number":457,"context_line":"enabled, it will add a forbidden trait via the root_required parameter when"},{"line_number":458,"context_line":"cache and/or bandwidth limits are not requested and will add a required trait"},{"line_number":459,"context_line":"and additional resource requests when they are. It is intended for this to"},{"line_number":460,"context_line":"increase scheduling performance by restricting the set of possible host by"},{"line_number":461,"context_line":"leveraging granular requests while also ensuring host are dynamically"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_e6e3bb05","line":458,"range":{"start_line":457,"start_character":17,"end_line":458,"end_character":47},"in_reply_to":"9fb8cfa7_93eb1f11","updated":"2019-07-03 00:57:06.000000000","message":"well part of the reason for not leaving it up to the operator is how many operators shot them selves in the foot and still do when we left it up to them to isolate host properly fo cpus pinnign and hugepages.\n\nthat said i did brefly discuss should the isolation be configuratble with lee earlier today so i can see having a config option for that.\n\nalex was suggesting removing the option for enabling or disabling the prefilter and always enabling it which prometed the question instead of haveing a config option to enable or disable the prefilter perhaps the config option should enable or disable isoaltion.\n\nso we would replace the one on line 505","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":455,"context_line":"also proposes the introduction of a prefilter to transform the flavor request"},{"line_number":456,"context_line":"into placement requests. This prefilter will be disabled by default. When"},{"line_number":457,"context_line":"enabled, it will add a forbidden trait via the root_required parameter when"},{"line_number":458,"context_line":"cache and/or bandwidth limits are not requested and will add a required trait"},{"line_number":459,"context_line":"and additional resource requests when they are. It is intended for this to"},{"line_number":460,"context_line":"increase scheduling performance by restricting the set of possible host by"},{"line_number":461,"context_line":"leveraging granular requests while also ensuring host are dynamically"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_46b887e4","line":458,"range":{"start_line":458,"start_character":57,"end_line":458,"end_character":77},"in_reply_to":"9fb8cfa7_b36ae37e","updated":"2019-07-03 00:57:06.000000000","message":"yes the trait was for the anti affintiy.\n\nto prevent instance that dont request the resouce form landing on it if we choose to isolate them.\n\nwe can use aggreates for that but i think that is much more error prone and is more of a burden on operators.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":467,"context_line":""},{"line_number":468,"context_line":"To enable this feature, four config options will be introduced, three of which"},{"line_number":469,"context_line":"will be set on the compute nodes and one that will be set on the scheduler:"},{"line_number":470,"context_line":""},{"line_number":471,"context_line":"``[libvirt] cache_allocation_support``"},{"line_number":472,"context_line":"  :Type: bool"},{"line_number":473,"context_line":"  :Default: False"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_330cb3eb","line":470,"updated":"2019-07-02 22:55:02.000000000","message":"IMO these should be named in some way that helps me understand they\u0027re related, like prefixing with ``pqos_``.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":467,"context_line":""},{"line_number":468,"context_line":"To enable this feature, four config options will be introduced, three of which"},{"line_number":469,"context_line":"will be set on the compute nodes and one that will be set on the scheduler:"},{"line_number":470,"context_line":""},{"line_number":471,"context_line":"``[libvirt] cache_allocation_support``"},{"line_number":472,"context_line":"  :Type: bool"},{"line_number":473,"context_line":"  :Default: False"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_932d5fff","line":470,"in_reply_to":"9fb8cfa7_330cb3eb","updated":"2019-07-03 00:57:06.000000000","message":"ya that is a good idea.\nill add that in the next version.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"2ba0460b5d691a38583c80cbbb2bcd27a73d2d6b","unresolved":false,"context_lines":[{"line_number":502,"context_line":"    if unset the larger of minium allocation size and granularity will"},{"line_number":503,"context_line":"    be used."},{"line_number":504,"context_line":""},{"line_number":505,"context_line":"``[scheduler] pqos_prefilter``"},{"line_number":506,"context_line":"  :Type: bool"},{"line_number":507,"context_line":"  :Default: False"},{"line_number":508,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_13f3afe9","line":505,"range":{"start_line":505,"start_character":19,"end_line":505,"end_character":28},"updated":"2019-07-02 22:55:02.000000000","message":"\u003cdan_hat\u003eprefiltering\u003c/dan_hat\u003e","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"5a12be37ab369d57fdc79aefc7a633a94de32108","unresolved":false,"context_lines":[{"line_number":502,"context_line":"    if unset the larger of minium allocation size and granularity will"},{"line_number":503,"context_line":"    be used."},{"line_number":504,"context_line":""},{"line_number":505,"context_line":"``[scheduler] pqos_prefilter``"},{"line_number":506,"context_line":"  :Type: bool"},{"line_number":507,"context_line":"  :Default: False"},{"line_number":508,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9fb8cfa7_06ca8f6d","line":505,"range":{"start_line":505,"start_character":19,"end_line":505,"end_character":28},"in_reply_to":"9fb8cfa7_13f3afe9","updated":"2019-07-03 00:57:06.000000000","message":"sure ill change this.\n\nbut maybe this shoudl be pqos_host_isolation instead?\n\ni would prefer to default to True in that case but since you still have to opt into report the pqos resources by seeting the compute node config options but im ok to defualting to false and just documenting the pros and cons of either approach. this is somthing we would likely change in a deployment tool as we validate this feature in production but initially i was trying to be somewhat conservative with what we allow by default with the view to relax that over time.","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"f1680bd367cf31e1266a3a401cd512f52856d96f","unresolved":false,"context_lines":[{"line_number":504,"context_line":""},{"line_number":505,"context_line":"``[scheduler] pqos_prefilter``"},{"line_number":506,"context_line":"  :Type: bool"},{"line_number":507,"context_line":"  :Default: False"},{"line_number":508,"context_line":"  :Description: When set to ``True``, enable transformation of cache and"},{"line_number":509,"context_line":"    memory bandwidth extra specs into placement requests."},{"line_number":510,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7faddb67_f41a6f15","line":507,"updated":"2019-07-05 06:18:22.000000000","message":"Just want to see what other people\u0027s opinion on the extra specs discovering problem...","commit_id":"7cc67b359cb05666fc2bd91320ba29f3e7f3a408"}]}