)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":10,"context_line":"virtual CPUs to dedicated host CPUs, providing guarantees for"},{"line_number":11,"context_line":"CPU time and improved worst case latency for CPU scheduling."},{"line_number":12,"context_line":"The real time feature builds on that work to provide stronger"},{"line_number":13,"context_line":"guarantees for worst case schedular latency for vCPUs."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Blueprint: libvirt-real-time"},{"line_number":16,"context_line":"Change-Id: I79b3bfc555ba7ca1651685434b7136deed49515c"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"da20952f_1a55101c","line":13,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"}],"specs/backlog/approved/libvirt-real-time.rst":[{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":13,"context_line":"The CPU pinning feature added to the ability to assign guest virtual CPUs"},{"line_number":14,"context_line":"to dedicated host CPUs, providing guarantees for CPU time and improved worst"},{"line_number":15,"context_line":"case latency for CPU scheduling. The real time feature builds on that work"},{"line_number":16,"context_line":"to provide stronger guarantees for worst case schedular latency for vCPUs."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_1ac5105a","line":16,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":35,"context_line":"process can be swapped out too."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"There will be a number of more general host OS setup tasks that can be done"},{"line_number":38,"context_line":"too. For example, host interrpt affinity can be set to move avoid interrupt"},{"line_number":39,"context_line":"handlers running on pCPUs assigned to guests. Many of these setup tasks will"},{"line_number":40,"context_line":"not be the responsiblity of Nova, but rather documented deployment"},{"line_number":41,"context_line":"pre-requisites to be dealt with by the external OpenStack deployment tool"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_1aef50b1","line":38,"updated":"2015-08-28 10:42:28.000000000","message":"s/interrpt/interrupt/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":37,"context_line":"There will be a number of more general host OS setup tasks that can be done"},{"line_number":38,"context_line":"too. For example, host interrpt affinity can be set to move avoid interrupt"},{"line_number":39,"context_line":"handlers running on pCPUs assigned to guests. Many of these setup tasks will"},{"line_number":40,"context_line":"not be the responsiblity of Nova, but rather documented deployment"},{"line_number":41,"context_line":"pre-requisites to be dealt with by the external OpenStack deployment tool"},{"line_number":42,"context_line":"or cloud operator."},{"line_number":43,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_ba13c4b7","line":40,"updated":"2015-08-28 10:42:28.000000000","message":"s/responsiblity/responsibility/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":17863,"name":"henning.schild","email":"henning.schild@siemens.com","username":"henning.schild"},"change_message_id":"1bbf79f1e0553b7966b29e6520e9c965507015a6","unresolved":false,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Proposed change"},{"line_number":65,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"The flavour extra specs will permit a new property to be set"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_adaaec17","line":66,"updated":"2015-08-27 12:19:47.000000000","message":"An instance of nova that is rt-capable needs to have at least one pCPUs for noise. Emulator and IO threads must not run on the same pCPUs as the vCPUs.\nNot only could that disturb the vCPUs, such scheduling could lead to lockups. i.e. high prio vCPU polling on device that is served by IOthread/emulator\n\nAn instance of nova that is rt-capable also needs a notion of non-rt (noise). And it needs to know which pCPUs to schedule it on.\n\nThe host system potentially has 4 classes of pCPUs.\n\n1. non-rt, no-nova\n2. rt, no-nova\n3. non-rt, nova\n4. rt, nova\n\nTo cover that we need a way to describe two sets of pCPUs in nova.conf (3 and 4).\n\nThe current nova.conf option \"vcpu_pin_set\" has a confusing name for what it actually does. And its description is wrong, since it just refers to vcpus and not the noise.\n\nProposed changes:\n\n* rename \"vcpu_pin_set\" to \"pcpu_set\"\n* introduce new set of pcpus where the rt-vcpus will be running \" pcpu_set_rt\"\n\nThe descriptions should be \"set of pcpus nova is allowed to use [for rt]\".\n\nThese changes touch the config format and therefore an important interface. I guess they should be discussed first.\n\nHere are some arguments cited from an earlier discussing why i think that interface change makes sense\n\n\u003e\u003e When we get additional requirements, like rt IO we could also choose to pin certain IO threads to a dedicated rt-CPU. We can have VMs where only some vcpus will be RT. In a smp guest one core will probably be best-effort anyways why burn a host rt-CPU for that vcpu? And we get mixed RT-/best-effort compute nodes \"for free\", because for RT we essentially need to implement mixed mode.","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"2a5a948be8cf37fd1305851733491f032807d352","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"The flavour extra specs will permit a new property to be set"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"This property is only valid if the \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie"},{"line_number":72,"context_line":"the guest must have exclusive pCPUs assigned to it, you cannot give it a real"}],"source_content_type":"text/x-rst","patch_set":6,"id":"ba15a1d1_e3e0c7fc","line":69,"updated":"2015-09-15 16:06:24.000000000","message":"We cannot just have an option like that. A realtime flavor should always have a topology defined with at least 2 vCPUs (one for realtime and one for IO, emulator... ) as well that topology should be defined on top of NUMA to isolate RT-vCPUs running on a single host NUMA node.\n\nIn complement with proposed change of henning.schild; Adding pcpu_set_rt which will isolate host CPUs for running realtime workloads. We should to provide a new topology like cpu_dedicated.N, with N refer guest vCPU which will be dedicated to a host CPU. This topology can run only with a dedicated policy.\n\n - cpu_policy \u003d dedicated\n - cpu_dedicated.0 \u003d realtime # will pick one from pcpu_set_rt\n - cpu_dedicated.1 \u003d cpu, emulator\n\nThat can easily run on top of NUMA\n\n  - numa_nodes \u003d 2\n  - numa_node.0 \u003d 0 # refer vcpu 1 which will be pinned on a RT CPU\n  - numa_node.1 \u003d 1 # refer vcpu 2 which will be pinned on a regular CPU","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Similarly this policy could also be set against the image metadata"},{"line_number":77,"context_line":"properties with the name \u0027hw_cpu_realtime\u0027. But the use of this"},{"line_number":78,"context_line":"propertie would only be honoured if the flavor already had"},{"line_number":79,"context_line":"cpu_realtime set with yes."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"When this setting is enabled at least two extra libvirt configuration options"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_1a7430fb","line":78,"updated":"2015-08-28 10:42:28.000000000","message":"s/properie/property/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":8865,"name":"Andrey Korolyov","email":"akorolev@mirantis.com","username":"xdel"},"change_message_id":"3ec5fa18283e6657cfd81054d2ea4df372c0e829","unresolved":false,"context_lines":[{"line_number":82,"context_line":"will be set."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"* Memory locking will be enabled"},{"line_number":85,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Alternatives"},{"line_number":88,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3a50d1a3_cbb0f345","line":85,"updated":"2015-07-28 16:30:37.000000000","message":"Would it bring any good over CFS if no other CPU-consuming services are running and CPU pinning overlap will be not created during pinning procedure? By my experience, giving sched_rt for guests can bring out a shiny constellation of I/O issues, from network to disk ones, with collateral hypervisor lockups. Also please give more details on how this setting supposed to reduce response times from \u0027pure\u0027 compute storage (e.g no cpu-consuming applications aside) comparing with cfs for everything.","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":82,"context_line":"will be set."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"* Memory locking will be enabled"},{"line_number":85,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Alternatives"},{"line_number":88,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_daa558b5","line":85,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":8865,"name":"Andrey Korolyov","email":"akorolev@mirantis.com","username":"xdel"},"change_message_id":"9bc3cec6279a2b895364bae066403161d29de150","unresolved":false,"context_lines":[{"line_number":82,"context_line":"will be set."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"* Memory locking will be enabled"},{"line_number":85,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Alternatives"},{"line_number":88,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3a50d1a3_bc260bcc","line":85,"in_reply_to":"3a50d1a3_967beebd","updated":"2015-07-28 17:34:04.000000000","message":"Ok, I see, with these considerations the sched_rt usage seems safe. Also I`m afraid that libvirt should introduce rt cgroup handlers in advance for effective management for such VMs. There also should be a lot of pre-considerations like proper choosing for a CPU % bandwidth which would be given for RT tasks, probably no VF devices and couple of other minor quirks. CFS *may* meet same criteria for isolated cores with proper cgroup management and no CPU oversubscription/core overlapping for VMs, at least I can`t spot the difference between those two classes with such preconditions in a thought experiment. Any other preconditions (overlapped vm-\u003ephy core mapping/execution time oversubscription) may fail even for sched_rt case as well.","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"8d25a78211d92f7f89b0649ff4c922e4343d7825","unresolved":false,"context_lines":[{"line_number":82,"context_line":"will be set."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"* Memory locking will be enabled"},{"line_number":85,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Alternatives"},{"line_number":88,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3a50d1a3_967beebd","line":85,"in_reply_to":"3a50d1a3_cbb0f345","updated":"2015-07-28 16:54:37.000000000","message":"The goal of real time is to ensure a guest has deterministic scheduling latencies. The standard CFS schedular is not deterministic and thus useless for tasks which need guaranteed worst case response. Note that a real time guest will have worse overall performance than a non-real time guest as the goal is to tradeoff overall performance in favour of deterministic scheduling. So you cannot equate a real time guest\u0027s operation with a non-real time guest that has dedicated CPU availability.\n\nNB, you cannot simply give any guest a real time scheduling policy and expect it to magically work - the guest OS has to be expecting this and has to set itself up accordingly to avoid certain tasks/features that will not work in combination with real time. eg you can\u0027t do disk I/O from cpu threads realtime tasks.  Part of the work will be documenting exactly what you can \u0026 can\u0027t do.","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":87,"context_line":"Alternatives"},{"line_number":88,"context_line":"------------"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"One option would be to always enable a real time schedular policy when the"},{"line_number":91,"context_line":"guest is using dedicated CPU pinning and always enable memory locking when"},{"line_number":92,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":93,"context_line":"real time schedular does have potential downsides too, for example, kernel"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_7a83ec04","line":90,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":90,"context_line":"One option would be to always enable a real time schedular policy when the"},{"line_number":91,"context_line":"guest is using dedicated CPU pinning and always enable memory locking when"},{"line_number":92,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":93,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":94,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":95,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":96,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_bab744d7","line":93,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":92,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":93,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":94,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":95,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":98,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_ba9664ee","line":95,"updated":"2015-08-28 10:42:28.000000000","message":"s/perferrable/preferable/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":95,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":98,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"},{"line_number":99,"context_line":"with dedicated pCPUs. Some of those latencies could be further mitigated by"},{"line_number":100,"context_line":"careful host OS configuration, but it will never match what\u0027s achievable by"},{"line_number":101,"context_line":"fully configuring real time."}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_faab7c1b","line":98,"updated":"2015-08-28 10:42:28.000000000","message":"s/inherant/inherent/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":132,"context_line":"------------------"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"There will be no new performance impact to Nova as a whole. This is building"},{"line_number":135,"context_line":"on the existing CPU pinning and huge pages features, so the schedular logic is"},{"line_number":136,"context_line":"already in place. Likewise the impact on the host is restricted to pCPUs which"},{"line_number":137,"context_line":"are already assigned to a guest."},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_9ab4c0d1","line":135,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":180,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"* The libvirt project needs to add support for the XML feature to enable"},{"line_number":183,"context_line":"  real time schedular priority for guests."},{"line_number":184,"context_line":"* The KVM project needs to produce recommendations for optimal host OS"},{"line_number":185,"context_line":"  setup"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_fac5bc3b","line":183,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":193,"context_line":""},{"line_number":194,"context_line":"The key functional testing requirement is around correct operation of"},{"line_number":195,"context_line":"the existing Nova CPU pinning and huge pages features and their"},{"line_number":196,"context_line":"schedular integration. This is outside the scope of this particular"},{"line_number":197,"context_line":"blueprint."},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_7ae8ccb4","line":196,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"d9c1eb81b1490ff17aece889b7a72619896225aa","unresolved":false,"context_lines":[{"line_number":200,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"The deployment documentation will need to be updated to describe how to setup"},{"line_number":203,"context_line":"a host to take advantage of real time schedular prioritization."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"References"},{"line_number":206,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_3ad33406","line":203,"updated":"2015-08-28 10:42:28.000000000","message":"s/schedular/scheduler/","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"},{"author":{"_account_id":6804,"name":"bruce-benjamin","email":"bruce.benjamin@jhuapl.edu","username":"bruce-benjamin"},"change_message_id":"e0708b7d05b60212ad7bd9ad5817a16226780635","unresolved":false,"context_lines":[{"line_number":204,"context_line":""},{"line_number":205,"context_line":"References"},{"line_number":206,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"None at this time"}],"source_content_type":"text/x-rst","patch_set":6,"id":"da20952f_5bf0c591","line":207,"updated":"2015-09-08 16:07:10.000000000","message":"You may want to consider adding these references:\n\nhttp://www.linux-kvm.org/images/2/24/01x02-Rik_van_Riel-KVM_realtime.pdf\n\nhttp://www.linux-kvm.org/images/8/87/02x09-Aspen-Jun_Nakajima-KVM_as_the_NFV_Hypervisor.pdf","commit_id":"9f1f09c352f9e87aa31a90d4e4004623e25bdfad"}],"specs/kilo/approved/libvirt-real-time.rst":[{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"263f1c81d014965032f899db5ae81817dcbba95d","unresolved":false,"context_lines":[{"line_number":34,"context_line":"QEMU emulator. Locking all these into RAM ensures that no part of the QEMU"},{"line_number":35,"context_line":"process can be swapped out too."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"There will be a number of more general host OS setup tasks that can be done"},{"line_number":38,"context_line":"too. For example, host interrpt affinity can be set to move avoid interrupt"},{"line_number":39,"context_line":"handlers running on pCPUs assigned to guests. Many of these setup tasks will"},{"line_number":40,"context_line":"not be the responsiblity of Nova, but rather documented deployment"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_67822254","line":37,"updated":"2014-12-08 09:27:16.000000000","message":"s/interrpt/interrupt","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":6772,"name":"Stephen Gordon","email":"sgordon@redhat.com","username":"sgordon"},"change_message_id":"2332f0cd9a3e8b55e409798a406d298bf155514b","unresolved":false,"context_lines":[{"line_number":44,"context_line":"Merely enabling real time in the host OS and guest emulator is not on its own"},{"line_number":45,"context_line":"sufficient. The guest workload and operating system must also be suitably"},{"line_number":46,"context_line":"configured to take advantage of it. Such configuration is considered to be out"},{"line_number":47,"context_line":"of scope for Nova, rather it the responsibility of the tenant user to setup."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Use Cases"},{"line_number":50,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_8cf62ed6","line":47,"updated":"2014-12-09 06:14:16.000000000","message":"s/it/it is/","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":6772,"name":"Stephen Gordon","email":"sgordon@redhat.com","username":"sgordon"},"change_message_id":"2332f0cd9a3e8b55e409798a406d298bf155514b","unresolved":false,"context_lines":[{"line_number":77,"context_line":"with the name \u0027hw_cpu_realtime\u0027."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"When this setting is enabled at least two extra libvirt configuration options"},{"line_number":80,"context_line":"will be set."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedlar priority will be set to realtime"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_ac03eab6","line":80,"updated":"2014-12-09 06:14:16.000000000","message":"Would be nice to have the explicit values and possibly a reference link.","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"263f1c81d014965032f899db5ae81817dcbba95d","unresolved":false,"context_lines":[{"line_number":80,"context_line":"will be set."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedlar priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_e7953289","line":83,"updated":"2014-12-08 09:27:16.000000000","message":"s/schedlar/schedular","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":6772,"name":"Stephen Gordon","email":"sgordon@redhat.com","username":"sgordon"},"change_message_id":"2332f0cd9a3e8b55e409798a406d298bf155514b","unresolved":false,"context_lines":[{"line_number":80,"context_line":"will be set."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedlar priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_cc0036ad","line":83,"updated":"2014-12-09 06:14:16.000000000","message":"s/schedlar/schedular/","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":6772,"name":"Stephen Gordon","email":"sgordon@redhat.com","username":"sgordon"},"change_message_id":"2332f0cd9a3e8b55e409798a406d298bf155514b","unresolved":false,"context_lines":[{"line_number":90,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":91,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":92,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":93,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":96,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_ec2c124d","line":93,"updated":"2014-12-09 06:14:16.000000000","message":"Question with regards to the above, given the potential impact on kernel threads is this something we want to place in the hands of the user via image properties in the default case or should it be flavor only (with the admin free to enable the ability to set it on image properties via policy if that\u0027s feasible?)?","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"263f1c81d014965032f899db5ae81817dcbba95d","unresolved":false,"context_lines":[{"line_number":144,"context_line":"certain set of compute nodes for use in combination with huge pages and CPU"},{"line_number":145,"context_line":"pinning. This is a pre-existing impact from those features, and real time does"},{"line_number":146,"context_line":"not alter that."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"Developer impact"},{"line_number":149,"context_line":"----------------"},{"line_number":150,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_124b5a5a","line":147,"updated":"2014-12-08 09:27:16.000000000","message":"Does the host need to be configured with a kernel-rt ? - We probably need to have some reserved CPUs for the host?","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":6772,"name":"Stephen Gordon","email":"sgordon@redhat.com","username":"sgordon"},"change_message_id":"2332f0cd9a3e8b55e409798a406d298bf155514b","unresolved":false,"context_lines":[{"line_number":189,"context_line":"characteristics of guests deployed by Nova, which is what would be needed to"},{"line_number":190,"context_line":"validate this feature."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"The key functional testing requirement is around correct opertion of the"},{"line_number":193,"context_line":"existing Nova CPU pinning and huge pages features and their schedular"},{"line_number":194,"context_line":"integration. This is outside the scope of this particular blueprint."},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a961159_0c323ea7","line":192,"updated":"2014-12-09 06:14:16.000000000","message":"s/opertion/operation/","commit_id":"9cbf5b5ae4990908d72cb1f2ea5930b9bab385c3"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f21e1885af35de7a17bdf80869ea8c5090ec17a3","unresolved":false,"context_lines":[{"line_number":13,"context_line":"The CPU pinning feature added to the ability to assigned guest virtual CPUs"},{"line_number":14,"context_line":"to dedicated host CPUs, providing guarantees for CPU time and improved worst"},{"line_number":15,"context_line":"case latency for CPU scheduling. The real time feature builds on that work"},{"line_number":16,"context_line":"to provide stronger guarantees for worst case schedular latency for vCPUs."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_c971790d","line":16,"range":{"start_line":16,"start_character":46,"end_line":16,"end_character":55},"updated":"2014-12-12 16:11:16.000000000","message":"\"scheduler\" is misspelled.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"d295a3adc866cc5a81132972c1ba83ffbaf68d6c","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"This property is only valid if the \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie"},{"line_number":72,"context_line":"the guest must have exclusive pCPUs assigned to it, you cannot give it a real"},{"line_number":73,"context_line":"time policy if it is sharing pCPUs with other guests as that would be unfairly"},{"line_number":74,"context_line":"advantageous."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_aec2a931","line":71,"updated":"2014-12-11 08:48:23.000000000","message":"A suggestion, can we use cpu_policy insteadof introduce a new one?\n\n  hw:cpu_policy\u003dshared|dedicated|realtime\n\nThis will avoid to set cpu_policy to dedicated and realtime to yes but in other hand can be discussed since realtime also means a different behavior for the memory.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"2d0842246ce9110ffd8ef4932dc8eed852917fee","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"This property is only valid if the \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie"},{"line_number":72,"context_line":"the guest must have exclusive pCPUs assigned to it, you cannot give it a real"},{"line_number":73,"context_line":"time policy if it is sharing pCPUs with other guests as that would be unfairly"},{"line_number":74,"context_line":"advantageous."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_5e04dd7d","line":71,"in_reply_to":"3a961159_0932a15e","updated":"2014-12-17 12:14:07.000000000","message":"If Nova hasn\u0027t assigned a dedicated CPU to the guest, it is pretty difficult, if not impossible, to provide any meaningful real-time performance guarantee to the guest, since its vCPUs will suffer from \u0027steal time\u0027 when other guests run on the same host CPUs. This steal time will destroy any latency guarantees, making the feature worthless. So only supporting this when the CPU policy is \u0027dedicated\u0027 is the only sane thing todo IMHO.\n\nI don\u0027t think that \u0027hw:cpu_realtime\u0027 is Linux specific at all. At the Linux impl level, you\u0027re actually choosing between a variety of schedular types in sched_setaffinity, and I explicitly avoided exposing that as a config option to the user, since it is a low level impl detail.  By having a simple  \u0027cpu_realtime\u003dyes|no\u0027 boolean we leave the Nova hypervisor drivers complete flexibility to choose how they provide real time scheduling performance guarantees to their guests.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f21e1885af35de7a17bdf80869ea8c5090ec17a3","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"This property is only valid if the \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie"},{"line_number":72,"context_line":"the guest must have exclusive pCPUs assigned to it, you cannot give it a real"},{"line_number":73,"context_line":"time policy if it is sharing pCPUs with other guests as that would be unfairly"},{"line_number":74,"context_line":"advantageous."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_0932a15e","line":71,"in_reply_to":"3a961159_3425265c","updated":"2014-12-12 16:11:16.000000000","message":"Can we get a read from the Xen/VMware/HyperV folks about how supportable this is for them? Presumably for LXC-based things, this is also doable. However, I wonder if it makes more sense to name this something a little less linux-kernel-y like \"hw:cpu_dedicated\u003dstrict\" or something like that? Also, I can see other drivers having more stringent requirements for this, requiring more than a yes/no, and also maybe not wanting to limit the use of this property to cases where the cpu_policy is dedicated. Presumably some system could want non-preemptible timeslices without requiring a pinned CPU?","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"561000ce4b1922d68b11b8fe16365425412a623c","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"This property is only valid if the \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie"},{"line_number":72,"context_line":"the guest must have exclusive pCPUs assigned to it, you cannot give it a real"},{"line_number":73,"context_line":"time policy if it is sharing pCPUs with other guests as that would be unfairly"},{"line_number":74,"context_line":"advantageous."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_3425265c","line":71,"in_reply_to":"3a961159_aec2a931","updated":"2014-12-11 09:51:26.000000000","message":"That doesn\u0027t work, because of the way permissioning works. For example if the admin sets  flavour  cpu_policy to \u0027dedicated\u0027, the code doesn\u0027t allow any change of cpu policy via image properties, but we *do* want to allow the user to ask for realtime. So we must keep the properties separate.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f21e1885af35de7a17bdf80869ea8c5090ec17a3","unresolved":false,"context_lines":[{"line_number":80,"context_line":"will be set."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_897a1122","line":83,"range":{"start_line":83,"start_character":6,"end_line":83,"end_character":15},"updated":"2014-12-12 16:11:16.000000000","message":"\"scheduler\" is misspelled.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"1fb426a89509d9100e5eda3d232b4120fa9b3a9c","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_3662cead","line":84,"updated":"2014-12-09 17:56:19.000000000","message":"An issue I have with this work is that the result is not exposed to a user.  If an instance is booted such that these libvirt config options are set and later the flavor is deleted then there\u0027s no way to check if the instance had this setup on boot.\n\nIt might be nice to update the instance metadata in some way to indicate this condition.  But this is also a problem outside the scope of this change.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"5a4d446c984a0ce6153afbedfbd8a24a8a672a8a","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_dded13f1","line":84,"in_reply_to":"3a961159_3662cead","updated":"2014-12-10 09:38:38.000000000","message":"The problem you describe doesn\u0027t seem specific to this feature - it sounds like it affects basically any use of flavours","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"561000ce4b1922d68b11b8fe16365425412a623c","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_74e96ee5","line":84,"in_reply_to":"3a961159_6b8fe4cc","updated":"2014-12-11 09:51:26.000000000","message":"I really think that should just be done as a standalone bug fix or blueprint and not at all tied to this work","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"98e214813edb61368c641aa1ca46291dc08cbd48","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_6b8fe4cc","line":84,"in_reply_to":"3a961159_dded13f1","updated":"2014-12-10 19:59:16.000000000","message":"Yes.  That\u0027s what I meant with the last line of my comment.\n\nBut we could start doing something about it here by setting metadata or system_metadata.  It could also be left up to the user to set metadata if they wanted.  Just throwing it out there to think about.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f21e1885af35de7a17bdf80869ea8c5090ec17a3","unresolved":false,"context_lines":[{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"One option would be to always enable a real time schedular policy when the"},{"line_number":89,"context_line":"guest is using dedicated CPU pinning and always enable memory locking when"},{"line_number":90,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":91,"context_line":"real time schedular does have potential downsides too, for example, kernel"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_4938293d","line":88,"range":{"start_line":88,"start_character":49,"end_line":88,"end_character":58},"updated":"2014-12-12 16:11:16.000000000","message":"\"scheduler\" is misspelled.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f21e1885af35de7a17bdf80869ea8c5090ec17a3","unresolved":false,"context_lines":[{"line_number":88,"context_line":"One option would be to always enable a real time schedular policy when the"},{"line_number":89,"context_line":"guest is using dedicated CPU pinning and always enable memory locking when"},{"line_number":90,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":91,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":92,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":93,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":94,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_893db12d","line":91,"range":{"start_line":91,"start_character":10,"end_line":91,"end_character":19},"updated":"2014-12-12 16:11:16.000000000","message":"yep.","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":6772,"name":"Stephen Gordon","email":"sgordon@redhat.com","username":"sgordon"},"change_message_id":"43d21b3d3a43a2ff99ea97e912ec8e2ce856264f","unresolved":false,"context_lines":[{"line_number":90,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":91,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":92,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":93,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":96,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_9fed5c28","line":93,"updated":"2014-12-09 16:11:52.000000000","message":"Question with regards to the above, given the potential impact on kernel threads is this something we want to place in the hands of the user via image properties in the default case or should it be flavor only (with the admin free to enable the ability to set it on image properties via policy if that\u0027s feasible?)?","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"9a7f1ac7ab53de659e352a0c6b50f520329f8cce","unresolved":false,"context_lines":[{"line_number":90,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":91,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":92,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":93,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":96,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_ffe74010","line":93,"in_reply_to":"3a961159_9fed5c28","updated":"2014-12-09 16:17:51.000000000","message":"The issue with kernel threads is more that the kernel threads can negatively impact on the guest performance, causing the guest to have latency spikes, not that the guest can impact kernel. For dedicated CPU pinning we already assume the admin has reserved some portion of host CPUs for just OS services, so if the guest CPU is busy the kernel will just run its relevant threads on the reserved CPUs. So I don\u0027t think we need worry about this. The existing controls over use of dedicated CPU pinning is sufficient","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f21e1885af35de7a17bdf80869ea8c5090ec17a3","unresolved":false,"context_lines":[{"line_number":93,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":96,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"},{"line_number":97,"context_line":"with dedicated pCPUs. Some of those latencies could be further mitigated by"},{"line_number":98,"context_line":"careful host OS configuration, but it will never match what\u0027s achievable by"},{"line_number":99,"context_line":"fully configuring real time."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_692965e5","line":96,"range":{"start_line":96,"start_character":34,"end_line":96,"end_character":42},"updated":"2014-12-12 16:11:16.000000000","message":"\"inherent\" is misspelled","commit_id":"d759030f40753e727bead2f748ce6f6e82320836"}],"specs/liberty/approved/libvirt-real-time.rst":[{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"35e27f7ccb80712a3b38e9eeb7d7562ffaf27eb8","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/libvirt-real-time"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"The CPU pinning feature added to the ability to assigned guest virtual CPUs"},{"line_number":14,"context_line":"to dedicated host CPUs, providing guarantees for CPU time and improved worst"},{"line_number":15,"context_line":"case latency for CPU scheduling. The real time feature builds on that work"},{"line_number":16,"context_line":"to provide stronger guarantees for worst case schedular latency for vCPUs."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_7a38cba0","line":13,"updated":"2015-07-01 06:56:46.000000000","message":"assigned \u003d\u003e assign","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"873e5d1750abf7155724767820c4b8d0bea795e0","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/libvirt-real-time"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"The CPU pinning feature added to the ability to assigned guest virtual CPUs"},{"line_number":14,"context_line":"to dedicated host CPUs, providing guarantees for CPU time and improved worst"},{"line_number":15,"context_line":"case latency for CPU scheduling. The real time feature builds on that work"},{"line_number":16,"context_line":"to provide stronger guarantees for worst case schedular latency for vCPUs."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_a0a5c4b5","line":13,"in_reply_to":"ba3cc151_7a38cba0","updated":"2015-07-01 07:10:04.000000000","message":"Done","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"0c4c16465c985a9a23d298574aa2ff207bacb8b0","unresolved":false,"context_lines":[{"line_number":74,"context_line":"advantageous."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Similarly this policy could also be set against the image metadata properties"},{"line_number":77,"context_line":"with the name \u0027hw_cpu_realtime\u0027."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"When this setting is enabled at least two extra libvirt configuration options"},{"line_number":80,"context_line":"will be set."}],"source_content_type":"text/x-rst","patch_set":4,"id":"fa32b979_cdac788a","line":77,"updated":"2015-06-24 14:46:16.000000000","message":"Does this image properties limited by flavor extra specs? There can be different account.","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"873e5d1750abf7155724767820c4b8d0bea795e0","unresolved":false,"context_lines":[{"line_number":74,"context_line":"advantageous."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Similarly this policy could also be set against the image metadata properties"},{"line_number":77,"context_line":"with the name \u0027hw_cpu_realtime\u0027."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"When this setting is enabled at least two extra libvirt configuration options"},{"line_number":80,"context_line":"will be set."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_409ab864","line":77,"in_reply_to":"ba3cc151_42eaeb10","updated":"2015-07-01 07:10:04.000000000","message":"This is the policy we use all the time, but yes perhaps we should add a note. Thanks","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"b3de0491190fb19d4090e92b3e33b69c44eae445","unresolved":false,"context_lines":[{"line_number":74,"context_line":"advantageous."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Similarly this policy could also be set against the image metadata properties"},{"line_number":77,"context_line":"with the name \u0027hw_cpu_realtime\u0027."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"When this setting is enabled at least two extra libvirt configuration options"},{"line_number":80,"context_line":"will be set."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_42eaeb10","line":77,"in_reply_to":"ba3cc151_4f31e40a","updated":"2015-06-30 12:43:42.000000000","message":"thanks, better to describe it.","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"ef42992d7d2d84fa4f480786054b21ada4c2c593","unresolved":false,"context_lines":[{"line_number":74,"context_line":"advantageous."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Similarly this policy could also be set against the image metadata properties"},{"line_number":77,"context_line":"with the name \u0027hw_cpu_realtime\u0027."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"When this setting is enabled at least two extra libvirt configuration options"},{"line_number":80,"context_line":"will be set."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_4f31e40a","line":77,"in_reply_to":"fa32b979_cdac788a","updated":"2015-06-29 14:46:28.000000000","message":"Yes, if the flavor extra specs is set with \u0027hw:cpu_realtime\u003dyes\u0027 so user can use image properties to enable or disable that feature for the guest.","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"35e27f7ccb80712a3b38e9eeb7d7562ffaf27eb8","unresolved":false,"context_lines":[{"line_number":80,"context_line":"will be set."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedlar priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_daff97a8","line":83,"updated":"2015-07-01 06:56:46.000000000","message":"schedlar  \u003d\u003e schedular","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"873e5d1750abf7155724767820c4b8d0bea795e0","unresolved":false,"context_lines":[{"line_number":80,"context_line":"will be set."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Memory locking will be enabled"},{"line_number":83,"context_line":"* CPU schedlar priority will be set to realtime"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_80a200ad","line":83,"in_reply_to":"ba3cc151_daff97a8","updated":"2015-07-01 07:10:04.000000000","message":"Done","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"07a2135aa32d83da5d269cba824ce9774f282783","unresolved":false,"context_lines":[{"line_number":106,"context_line":"REST API impact"},{"line_number":107,"context_line":"---------------"},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"None required"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"Security impact"},{"line_number":112,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"fa32b979_2805d20f","line":109,"updated":"2015-06-24 14:48:54.000000000","message":"A little impact. When \u0027hw:cpu_policy\u0027 is not dedicated. there 400 should return","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"b3de0491190fb19d4090e92b3e33b69c44eae445","unresolved":false,"context_lines":[{"line_number":106,"context_line":"REST API impact"},{"line_number":107,"context_line":"---------------"},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"None required"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"Security impact"},{"line_number":112,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_82e8f311","line":109,"in_reply_to":"ba3cc151_1209bf2c","updated":"2015-06-30 12:43:42.000000000","message":"ok, just want to doc the error code we used. But it\u0027s fine as you said.","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"ef42992d7d2d84fa4f480786054b21ada4c2c593","unresolved":false,"context_lines":[{"line_number":106,"context_line":"REST API impact"},{"line_number":107,"context_line":"---------------"},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"None required"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"Security impact"},{"line_number":112,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_1209bf2c","line":109,"in_reply_to":"fa32b979_2805d20f","updated":"2015-06-29 14:46:28.000000000","message":"I do not think we usually fill this section for this kind of cases.","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"35e27f7ccb80712a3b38e9eeb7d7562ffaf27eb8","unresolved":false,"context_lines":[{"line_number":124,"context_line":"Other end user impact"},{"line_number":125,"context_line":"---------------------"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"The tenant will have the ability to request real time via an image property."},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Performance Impact"},{"line_number":130,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_fae85be4","line":127,"updated":"2015-07-01 06:56:46.000000000","message":"So the tenant will need to update the image metadata property? What if the image is shared between tenants?","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"873e5d1750abf7155724767820c4b8d0bea795e0","unresolved":false,"context_lines":[{"line_number":124,"context_line":"Other end user impact"},{"line_number":125,"context_line":"---------------------"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"The tenant will have the ability to request real time via an image property."},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Performance Impact"},{"line_number":130,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_800680cc","line":127,"in_reply_to":"ba3cc151_fae85be4","updated":"2015-07-01 07:10:04.000000000","message":"So if the other tenant try to start the image with a flavor that does not enable realtime, an error will be raised. - The kernel image is updated to run in realtime I do not think we want make able this image to boot.","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"35e27f7ccb80712a3b38e9eeb7d7562ffaf27eb8","unresolved":false,"context_lines":[{"line_number":180,"context_line":"* The libvirt project needs to add support for the XML feature to enable"},{"line_number":181,"context_line":"  real time schedular priority for guests."},{"line_number":182,"context_line":"* The KVM project needs to produce recommendations for optimal host OS"},{"line_number":183,"context_line":"  setup"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"Testing"},{"line_number":186,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_bac9737a","line":183,"updated":"2015-07-01 06:56:46.000000000","message":"is this supported by all versions of libvirt? Will this require a version cap of sorts?","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"dba585dbe5f3abeb1d066f2bf4fbf17f87ce6439","unresolved":false,"context_lines":[{"line_number":180,"context_line":"* The libvirt project needs to add support for the XML feature to enable"},{"line_number":181,"context_line":"  real time schedular priority for guests."},{"line_number":182,"context_line":"* The KVM project needs to produce recommendations for optimal host OS"},{"line_number":183,"context_line":"  setup"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"Testing"},{"line_number":186,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_d9e889b5","line":183,"in_reply_to":"ba3cc151_80bbe0e4","updated":"2015-07-03 11:01:47.000000000","message":"Yeah like I said, I don\u0027t think it\u0027s worth mentioning that here","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"873e5d1750abf7155724767820c4b8d0bea795e0","unresolved":false,"context_lines":[{"line_number":180,"context_line":"* The libvirt project needs to add support for the XML feature to enable"},{"line_number":181,"context_line":"  real time schedular priority for guests."},{"line_number":182,"context_line":"* The KVM project needs to produce recommendations for optimal host OS"},{"line_number":183,"context_line":"  setup"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"Testing"},{"line_number":186,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_80bbe0e4","line":183,"in_reply_to":"ba3cc151_bac9737a","updated":"2015-07-01 07:10:04.000000000","message":"Nothing particular here, the support to configure scheduler priority of a guest comes with version 1.2.13","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"338857fa88c6bbd0c4fa73fef25c6b3f5341110b","unresolved":false,"context_lines":[{"line_number":180,"context_line":"* The libvirt project needs to add support for the XML feature to enable"},{"line_number":181,"context_line":"  real time schedular priority for guests."},{"line_number":182,"context_line":"* The KVM project needs to produce recommendations for optimal host OS"},{"line_number":183,"context_line":"  setup"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"Testing"},{"line_number":186,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_79613d04","line":183,"in_reply_to":"ba3cc151_d9e889b5","updated":"2015-07-03 11:11:30.000000000","message":"Any min version requirements will be determined at time of impl. We don\u0027t mandate a new libvirt version in general. As with features done before, we only check the min version when the feature in question is used.","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"35e27f7ccb80712a3b38e9eeb7d7562ffaf27eb8","unresolved":false,"context_lines":[{"line_number":189,"context_line":"characteristics of guests deployed by Nova, which is what would be needed to"},{"line_number":190,"context_line":"validate this feature."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"The key functional testing requirement is around correct opertion of the"},{"line_number":193,"context_line":"existing Nova CPU pinning and huge pages features and their schedular"},{"line_number":194,"context_line":"integration. This is outside the scope of this particular blueprint."},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_75321c12","line":192,"range":{"start_line":192,"start_character":57,"end_line":192,"end_character":65},"updated":"2015-07-01 06:56:46.000000000","message":"opertion \u003d\u003e opertaion","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"873e5d1750abf7155724767820c4b8d0bea795e0","unresolved":false,"context_lines":[{"line_number":189,"context_line":"characteristics of guests deployed by Nova, which is what would be needed to"},{"line_number":190,"context_line":"validate this feature."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"The key functional testing requirement is around correct opertion of the"},{"line_number":193,"context_line":"existing Nova CPU pinning and huge pages features and their schedular"},{"line_number":194,"context_line":"integration. This is outside the scope of this particular blueprint."},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba3cc151_40b5d8eb","line":192,"in_reply_to":"ba3cc151_75321c12","updated":"2015-07-01 07:10:04.000000000","message":"Done","commit_id":"5300549e5612c9f0199f8b1509e79e978995d1d1"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":35,"context_line":"process can be swapped out too."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"There will be a number of more general host OS setup tasks that can be done"},{"line_number":38,"context_line":"too. For example, host interrpt affinity can be set to move avoid interrupt"},{"line_number":39,"context_line":"handlers running on pCPUs assigned to guests. Many of these setup tasks will"},{"line_number":40,"context_line":"not be the responsiblity of Nova, but rather documented deployment"},{"line_number":41,"context_line":"pre-requisites to be dealt with by the external OpenStack deployment tool"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_283a44e8","line":38,"updated":"2015-07-06 15:25:42.000000000","message":"s/interrpt/interrupt/\n\n\"set to move avoid\"?","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"651343f2c7ab8f22dd7a255c531a42c1538043ff","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"This property is only valid if the \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie"},{"line_number":72,"context_line":"the guest must have exclusive pCPUs assigned to it, you cannot give it a real"},{"line_number":73,"context_line":"time policy if it is sharing pCPUs with other guests as that would be unfairly"},{"line_number":74,"context_line":"advantageous."}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_22f58e82","line":71,"updated":"2015-07-06 11:02:57.000000000","message":"Hmm, in a way, it would be nice to skip all lots of the settings and just put \"cpu_realtime\" and it does everything.\n\nAnyways, I am tempted to punt on that till the code.","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"11b19ef423e647a06506b33ce14fba709778fbc8","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"This property is only valid if the \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie"},{"line_number":72,"context_line":"the guest must have exclusive pCPUs assigned to it, you cannot give it a real"},{"line_number":73,"context_line":"time policy if it is sharing pCPUs with other guests as that would be unfairly"},{"line_number":74,"context_line":"advantageous."}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_5611705c","line":71,"in_reply_to":"ba3cc151_22f58e82","updated":"2015-07-06 12:46:35.000000000","message":"We can probably do something like that yes.","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"651343f2c7ab8f22dd7a255c531a42c1538043ff","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Similarly this policy could also be set against the image metadata"},{"line_number":77,"context_line":"properties with the name \u0027hw_cpu_realtime\u0027. But the use of this"},{"line_number":78,"context_line":"propertie would only be honoured if the flavor already had"},{"line_number":79,"context_line":"cpu_realtime set with yes."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"When this setting is enabled at least two extra libvirt configuration options"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_62395641","line":78,"updated":"2015-07-06 11:02:57.000000000","message":"Nit: s/propertie/property/","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"11b19ef423e647a06506b33ce14fba709778fbc8","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Similarly this policy could also be set against the image metadata"},{"line_number":77,"context_line":"properties with the name \u0027hw_cpu_realtime\u0027. But the use of this"},{"line_number":78,"context_line":"propertie would only be honoured if the flavor already had"},{"line_number":79,"context_line":"cpu_realtime set with yes."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"When this setting is enabled at least two extra libvirt configuration options"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_7687d4d7","line":78,"in_reply_to":"ba3cc151_62395641","updated":"2015-07-06 12:46:35.000000000","message":"I will update such nit when the spec is approved :-)","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":82,"context_line":"will be set."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"* Memory locking will be enabled"},{"line_number":85,"context_line":"* CPU schedular priority will be set to realtime"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Alternatives"},{"line_number":88,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_2e31edec","line":85,"updated":"2015-07-06 15:25:42.000000000","message":"s/schedular/scheduler/","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":87,"context_line":"Alternatives"},{"line_number":88,"context_line":"------------"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"One option would be to always enable a real time schedular policy when the"},{"line_number":91,"context_line":"guest is using dedicated CPU pinning and always enable memory locking when"},{"line_number":92,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":93,"context_line":"real time schedular does have potential downsides too, for example, kernel"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_4e41d138","line":90,"updated":"2015-07-06 15:25:42.000000000","message":"s/schedular/scheduler/","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":90,"context_line":"One option would be to always enable a real time schedular policy when the"},{"line_number":91,"context_line":"guest is using dedicated CPU pinning and always enable memory locking when"},{"line_number":92,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":93,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":94,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":95,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":96,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_8e6d99ce","line":93,"updated":"2015-07-06 15:25:42.000000000","message":"s/schedular/scheduler/","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":92,"context_line":"the guest has huge pages. This is considered undesirable though because a"},{"line_number":93,"context_line":"real time schedular does have potential downsides too, for example, kernel"},{"line_number":94,"context_line":"threads may suffer worse latency overall. As a result it is considered"},{"line_number":95,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":98,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_aea97de1","line":95,"updated":"2015-07-06 15:25:42.000000000","message":"s/perferrable/preferable/","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":95,"context_line":"perferrable to have an opt-in mechanism for enabling real time."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Do nothing is always an option. In the event of doing nothing, guests would"},{"line_number":98,"context_line":"have to put up with the latencies inherant in non-real time scheduling, even"},{"line_number":99,"context_line":"with dedicated pCPUs. Some of those latencies could be further mitigated by"},{"line_number":100,"context_line":"careful host OS configuration, but it will never match what\u0027s achievable by"},{"line_number":101,"context_line":"fully configuring real time."}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_ce81614f","line":98,"updated":"2015-07-06 15:25:42.000000000","message":"s/inherant/inherent/","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":132,"context_line":"------------------"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"There will be no new performance impact to Nova as a whole. This is building"},{"line_number":135,"context_line":"on the existing CPU pinning and huge pages features, so the schedular logic is"},{"line_number":136,"context_line":"already in place. Likewise the impact on the host is restricted to pCPUs which"},{"line_number":137,"context_line":"are already assigned to a guest."},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_2ee48d39","line":135,"updated":"2015-07-06 15:25:42.000000000","message":"s/schedular/scheduler/","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"28f1048450741da622c477159edbda035305e5cd","unresolved":false,"context_lines":[{"line_number":180,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"* The libvirt project needs to add support for the XML feature to enable"},{"line_number":183,"context_line":"  real time schedular priority for guests."},{"line_number":184,"context_line":"* The KVM project needs to produce recommendations for optimal host OS"},{"line_number":185,"context_line":"  setup"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_ce5de1ed","line":183,"updated":"2015-07-06 15:25:42.000000000","message":"Does there happen to be a link to the libvirt project\u0027s issue tracker for this request?","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"8d52647cbfc94370e632c59dbb2b011e46e1e42b","unresolved":false,"context_lines":[{"line_number":180,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"* The libvirt project needs to add support for the XML feature to enable"},{"line_number":183,"context_line":"  real time schedular priority for guests."},{"line_number":184,"context_line":"* The KVM project needs to produce recommendations for optimal host OS"},{"line_number":185,"context_line":"  setup"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba3cc151_8ee57ca9","line":183,"in_reply_to":"ba3cc151_ce5de1ed","updated":"2015-07-07 05:22:42.000000000","message":"Scheduler policy has been introduced in libvirt 1.2.13 and memory locking in libvirt 1.0.6","commit_id":"7153c3918000219adbd999af2db7b6c13b2f24e6"}],"specs/mitaka/approved/libvirt-real-time.rst":[{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"55b6c1c1f0fe2a3c77c5743f3f14049efc6ede06","unresolved":false,"context_lines":[{"line_number":103,"context_line":"those which do not. There will then need to be a property available on"},{"line_number":104,"context_line":"the flavour"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"which will indicate whether instances booted with that flavour will be"},{"line_number":109,"context_line":"run with a realtime policy. Flavours with this property set to \u0027yes\u0027"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_77e82e17","line":106,"updated":"2015-09-21 10:19:57.000000000","message":"Suggestion; Instead of a boolean we can have a cpuset of host isolated CPUs for reatlime. By that way we can pick from here host CPUs to pin them to guest vCPUs for realtime and continue to pick from vcpu_pin_set guest vCPUs  non-realtime.\n\nThis will bring a better partitioning but running that on top of NUMA also make computing the schedule more difficult.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"5422aa659a199ed68d820257931138897f35d849","unresolved":false,"context_lines":[{"line_number":103,"context_line":"those which do not. There will then need to be a property available on"},{"line_number":104,"context_line":"the flavour"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"which will indicate whether instances booted with that flavour will be"},{"line_number":109,"context_line":"run with a realtime policy. Flavours with this property set to \u0027yes\u0027"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_b7fb2647","line":106,"in_reply_to":"ba15a1d1_77e82e17","updated":"2015-09-21 10:41:30.000000000","message":"At a host level, we need to distinguish between which CPUs are used for running the OS and which CPUs are reserved for use by the guest instances. The vcpu_pin_set setting in nova.conf achieves this.\n\nSince we are saying that hosts will be dedicated for use by realtime guests only, I don\u0027t see any need to have a further option to allow some host pCPUs to be used for realtime and some for non-realtime.\n\nEach guest will have some vCPUs used for realtime, and some vCPUs used for non-realtime, but we already represent that with the hw:cpu_realtime_mask\u003d^0-1 setting I mention later on. \n\nThe scheduler though, doesn\u0027t need that info as all vCPUs are dedicated, with no overcommit, it merely needs to know how many host pCPUs are available for running guests, which vcpu_pin_set tells it. Only the actual libvirt driver needs to care which vCPUs are doing realtime.\n\nIOW, I\u0027m not seeing a need for this to be anything other than a boolean setting.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"f8c1dc704c83174c455902786e3c8300d489a16a","unresolved":false,"context_lines":[{"line_number":115,"context_line":"time guests must have exclusive pCPUs assigned to them. You cannot give"},{"line_number":116,"context_line":"a real time policy to vCPUs that are susceptible to overcommit, as that"},{"line_number":117,"context_line":"would lead to starvation of the other guests on that pCPU, as well as"},{"line_number":118,"context_line":"degrading the latency guarantees."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"The precise actions that a hypevisor driver takes to configure a guest"},{"line_number":121,"context_line":"when real time is enabled are implementation defined. Different hypevisors"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_b476c404","line":118,"updated":"2015-09-21 10:09:23.000000000","message":"An other pre-requisite: Hosts will be dedicated to run realtime only. Operator will have to configure these hosts and so isolate CPUs.\n\nIn this context it\u0027s also important for operator to set \u0027vcpu_pin_set\u0027; Nova should only build guests on host isolated CPUs.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"838dc1d09a4d4d8a587108fdd8ea3842096e8bd5","unresolved":false,"context_lines":[{"line_number":115,"context_line":"time guests must have exclusive pCPUs assigned to them. You cannot give"},{"line_number":116,"context_line":"a real time policy to vCPUs that are susceptible to overcommit, as that"},{"line_number":117,"context_line":"would lead to starvation of the other guests on that pCPU, as well as"},{"line_number":118,"context_line":"degrading the latency guarantees."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"The precise actions that a hypevisor driver takes to configure a guest"},{"line_number":121,"context_line":"when real time is enabled are implementation defined. Different hypevisors"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_5745ea14","line":118,"in_reply_to":"ba15a1d1_9703820d","updated":"2015-09-21 10:22:27.000000000","message":"My complaint was about to mention vcpu_pin_set. OK","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"b4a372e5ede15f33334b644315a691cae740d17f","unresolved":false,"context_lines":[{"line_number":115,"context_line":"time guests must have exclusive pCPUs assigned to them. You cannot give"},{"line_number":116,"context_line":"a real time policy to vCPUs that are susceptible to overcommit, as that"},{"line_number":117,"context_line":"would lead to starvation of the other guests on that pCPU, as well as"},{"line_number":118,"context_line":"degrading the latency guarantees."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"The precise actions that a hypevisor driver takes to configure a guest"},{"line_number":121,"context_line":"when real time is enabled are implementation defined. Different hypevisors"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_9703820d","line":118,"in_reply_to":"ba15a1d1_b476c404","updated":"2015-09-21 10:16:03.000000000","message":"I mention that a few paragraphs earlier where I say you need to setup host aggregates for all compute hosts doing realtime.  WRT  vcpu_pin_set, I didn\u0027t want to mention that here, since that\u0027s just one small impl detail we\u0027ll deal with when writing docs on how to configure the host OS.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":17863,"name":"henning.schild","email":"henning.schild@siemens.com","username":"henning.schild"},"change_message_id":"c8546e63ae539f930aed0e95ee20006407b370dc","unresolved":false,"context_lines":[{"line_number":139,"context_line":"realtime, guests must have at least 2 vCPUs. Although guests will not"},{"line_number":140,"context_line":"use all vCPUs for real-time workloads, from the KVM host POV there is"},{"line_number":141,"context_line":"not neccessarily any downside to giving all vCPUs a realtime schedular"},{"line_number":142,"context_line":"priority if nothing else will run on the corresponding pCPUs."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"As well as the vCPU workload, most hypervisors have one or more other"},{"line_number":145,"context_line":"threads running in the control plane which do work on behalf of the"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_7d5faf75","line":142,"updated":"2015-09-21 11:33:36.000000000","message":"If you run Linux with PREEMPT-RT as a guest people will probably take 2vCPUs and use one for OS and housekeeping. But that really is just an assumption and no given fact. All amounts of rt vCPUs should be possible, especially also 1. Whether the RTOS will partition that set further should not be the business of OpenStack. Unless we want to be able to say for every vCPU, whether it should be rt or not.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"ef15e76373516280ecf4c08817181bcb6100b342","unresolved":false,"context_lines":[{"line_number":139,"context_line":"realtime, guests must have at least 2 vCPUs. Although guests will not"},{"line_number":140,"context_line":"use all vCPUs for real-time workloads, from the KVM host POV there is"},{"line_number":141,"context_line":"not neccessarily any downside to giving all vCPUs a realtime schedular"},{"line_number":142,"context_line":"priority if nothing else will run on the corresponding pCPUs."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"As well as the vCPU workload, most hypervisors have one or more other"},{"line_number":145,"context_line":"threads running in the control plane which do work on behalf of the"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_7d49ef8d","line":142,"in_reply_to":"ba15a1d1_7d5faf75","updated":"2015-09-21 11:40:18.000000000","message":"See 2 paragraphs later where I describe the  hw:cpu_realtime_mask, which lets the user say which vCPUs will be used for RT tasks.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":17863,"name":"henning.schild","email":"henning.schild@siemens.com","username":"henning.schild"},"change_message_id":"c6b960ac2c9c5c59569fb4f94a4b2b2095ecb9ab","unresolved":false,"context_lines":[{"line_number":151,"context_line":"the case of realtime, because these emulator threads will be"},{"line_number":152,"context_line":"doing work that can impact latency guarantees. There is thus a"},{"line_number":153,"context_line":"need to place emulator threads in a more fine precise fashion."},{"line_number":154,"context_line":"As noted above, every guest OS will have at least one vCPU that is"},{"line_number":155,"context_line":"to be used for non-realtime house keeping tasks. One option is to"},{"line_number":156,"context_line":"thus place the emulator threads on the same pCPU(s) that these"},{"line_number":157,"context_line":"non-realtime vCPUs run on."}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_e819bf7a","line":154,"updated":"2015-09-21 11:44:29.000000000","message":"Again! That is a very strong and wrong assumption.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":17863,"name":"henning.schild","email":"henning.schild@siemens.com","username":"henning.schild"},"change_message_id":"bbb58b10838d4009811337783f4eb64d33e809db","unresolved":false,"context_lines":[{"line_number":151,"context_line":"the case of realtime, because these emulator threads will be"},{"line_number":152,"context_line":"doing work that can impact latency guarantees. There is thus a"},{"line_number":153,"context_line":"need to place emulator threads in a more fine precise fashion."},{"line_number":154,"context_line":"As noted above, every guest OS will have at least one vCPU that is"},{"line_number":155,"context_line":"to be used for non-realtime house keeping tasks. One option is to"},{"line_number":156,"context_line":"thus place the emulator threads on the same pCPU(s) that these"},{"line_number":157,"context_line":"non-realtime vCPUs run on."}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_48430b6d","line":154,"in_reply_to":"ba15a1d1_0868b3b9","updated":"2015-09-21 12:10:05.000000000","message":"How would a legacy uniprocessor guest make sure its vCPU is not colocated with the emulator? If the guest is SMP the image needs to know which vCPUs can not be trusted, that is OK. But the guest needs to be SMP is certainly not OK.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"6e70090bbbe19a5338777f59654f4bcb0b19e4e9","unresolved":false,"context_lines":[{"line_number":151,"context_line":"the case of realtime, because these emulator threads will be"},{"line_number":152,"context_line":"doing work that can impact latency guarantees. There is thus a"},{"line_number":153,"context_line":"need to place emulator threads in a more fine precise fashion."},{"line_number":154,"context_line":"As noted above, every guest OS will have at least one vCPU that is"},{"line_number":155,"context_line":"to be used for non-realtime house keeping tasks. One option is to"},{"line_number":156,"context_line":"thus place the emulator threads on the same pCPU(s) that these"},{"line_number":157,"context_line":"non-realtime vCPUs run on."}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_83810450","line":154,"in_reply_to":"ba15a1d1_48430b6d","updated":"2015-09-21 12:16:23.000000000","message":"This initial spec requires that the guest be SMP, since it has the requirement that the emulator threads be co-located with one of the guest vCPUs that is used for non-RT tasks.\n\nWe need to be able to deal with emulator threads being placed on host CPUs which are *not* used by any vCPU, but that is a more involved bit of work since it will have schedular impact. It also does not depend on the real-time blueprint, since it is something we want to be able todo even in non-realtime cases. I\u0027m intending to submit a separate blueprint for that work.  Once that blueprint is done, then the restriction that all realtime guests must be SMP will be relaxed to all UP guests.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"a67aca5cf82fa73595571fc8484df502fe4e7612","unresolved":false,"context_lines":[{"line_number":151,"context_line":"the case of realtime, because these emulator threads will be"},{"line_number":152,"context_line":"doing work that can impact latency guarantees. There is thus a"},{"line_number":153,"context_line":"need to place emulator threads in a more fine precise fashion."},{"line_number":154,"context_line":"As noted above, every guest OS will have at least one vCPU that is"},{"line_number":155,"context_line":"to be used for non-realtime house keeping tasks. One option is to"},{"line_number":156,"context_line":"thus place the emulator threads on the same pCPU(s) that these"},{"line_number":157,"context_line":"non-realtime vCPUs run on."}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_0868b3b9","line":154,"in_reply_to":"ba15a1d1_e819bf7a","updated":"2015-09-21 11:49:04.000000000","message":"No, it is not an assumption - it is just an illustration of the default, which is fully user overridable as below.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":17863,"name":"henning.schild","email":"henning.schild@siemens.com","username":"henning.schild"},"change_message_id":"c6b960ac2c9c5c59569fb4f94a4b2b2095ecb9ab","unresolved":false,"context_lines":[{"line_number":161,"context_line":"can be set either on the flavour, or on the image. This will"},{"line_number":162,"context_line":"indicate which vCPUs will have realtime policy enabled:"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"* hw:cpu_realtime_mask\u003d^0-1"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"This indicates that all vCPUs, except vCPUs 0 and 1 will have"},{"line_number":167,"context_line":"a realtime policy. ie vCPUs 0 and 1 will remain non-realtime."}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_4808eb28","line":164,"updated":"2015-09-21 11:44:29.000000000","message":"But 0 and 1 would still be \"dedicated\" to the instance. You might need a cpu_dedicated_mask as well.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"a67aca5cf82fa73595571fc8484df502fe4e7612","unresolved":false,"context_lines":[{"line_number":161,"context_line":"can be set either on the flavour, or on the image. This will"},{"line_number":162,"context_line":"indicate which vCPUs will have realtime policy enabled:"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"* hw:cpu_realtime_mask\u003d^0-1"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"This indicates that all vCPUs, except vCPUs 0 and 1 will have"},{"line_number":167,"context_line":"a realtime policy. ie vCPUs 0 and 1 will remain non-realtime."}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_487b0b7c","line":164,"in_reply_to":"ba15a1d1_4808eb28","updated":"2015-09-21 11:49:04.000000000","message":"The real time policy builds on the dedicated vCPUs policy, so all vCPUs are dedicated for use by the guest instance in question. As such we merely need to distinguish realtime vs non-realtime vCPUs, which this does.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":17863,"name":"henning.schild","email":"henning.schild@siemens.com","username":"henning.schild"},"change_message_id":"bbb58b10838d4009811337783f4eb64d33e809db","unresolved":false,"context_lines":[{"line_number":179,"context_line":"the emulator threads as essentially being a virtual CPU in their"},{"line_number":180,"context_line":"own right. Such an enhancement is considered out of scope for"},{"line_number":181,"context_line":"the initial implementation, in order to avoid the need to make"},{"line_number":182,"context_line":"any schedular modifications."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"A significant portion of the work required will be documenting the"},{"line_number":185,"context_line":"required compute host and guest OS setup, as much of this cannot be"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_a8d42747","line":182,"updated":"2015-09-21 12:10:05.000000000","message":"I think all emulator threads should be treated like any vCPU-thread that is not dedicated. Meaning they should run distributed over all pCPUs available to nova, excluding the ones that are dedicated to vCPU threads.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"6e70090bbbe19a5338777f59654f4bcb0b19e4e9","unresolved":false,"context_lines":[{"line_number":179,"context_line":"the emulator threads as essentially being a virtual CPU in their"},{"line_number":180,"context_line":"own right. Such an enhancement is considered out of scope for"},{"line_number":181,"context_line":"the initial implementation, in order to avoid the need to make"},{"line_number":182,"context_line":"any schedular modifications."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"A significant portion of the work required will be documenting the"},{"line_number":185,"context_line":"required compute host and guest OS setup, as much of this cannot be"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba15a1d1_a3868863","line":182,"in_reply_to":"ba15a1d1_a8d42747","updated":"2015-09-21 12:16:23.000000000","message":"See above, that will be a separate blueprint.","commit_id":"91151351b5cc38bd79f25c558fe06754fc55c4e2"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"daefd1148ea7050bc4c360f9a23c4eaf31515e50","unresolved":false,"context_lines":[{"line_number":309,"context_line":"though, merely a complementary piece of work to allow real-time to"},{"line_number":310,"context_line":"be used in a broader range of scenarios."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"* https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads"},{"line_number":313,"context_line":"* https://review.openstack.org/225893"},{"line_number":314,"context_line":""},{"line_number":315,"context_line":"Testing"}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba15a1d1_71c0f67c","line":312,"updated":"2015-09-22 14:58:43.000000000","message":"https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads-policy","commit_id":"169e49b401325209f5eea1ff3bbd273a89bb76aa"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"054318735b5dec9aff02b96218efce52777d155e","unresolved":false,"context_lines":[{"line_number":309,"context_line":"though, merely a complementary piece of work to allow real-time to"},{"line_number":310,"context_line":"be used in a broader range of scenarios."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"* https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads"},{"line_number":313,"context_line":"* https://review.openstack.org/225893"},{"line_number":314,"context_line":""},{"line_number":315,"context_line":"Testing"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba15a1d1_31378900","line":312,"updated":"2015-09-24 07:39:34.000000000","message":"We need to fix that link: https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads-policy","commit_id":"b4dcbfe701efbab4dd4a354a975be2b7b175012f"},{"author":{"_account_id":17789,"name":"Zhao Lei","email":"zhaolei@cn.fujitsu.com","username":"zhaolei"},"change_message_id":"91d5a65731ed31dafc749d7a8efc12108c12ad7e","unresolved":false,"context_lines":[{"line_number":38,"context_line":"Assigning huge pages to a guest ensures that guest RAM cannot be swapped out"},{"line_number":39,"context_line":"on the host, but there are still other arbitrary memory allocations for the"},{"line_number":40,"context_line":"QEMU emulator. If parts of QEMU get swapped out to disk, then can have an"},{"line_number":41,"context_line":"impact on the peformance of the realtime guest."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Enabling realtime is not without cost. In order to meet the strict worst"},{"line_number":44,"context_line":"case requirements for CPU latency, overall throughput of the system must"}],"source_content_type":"text/x-rst","patch_set":10,"id":"9a1a9d01_493516d8","line":41,"updated":"2015-09-28 08:12:47.000000000","message":"performance","commit_id":"4b93bf3bf9bf62e97faa3dc905b0893bf0343aa2"},{"author":{"_account_id":17789,"name":"Zhao Lei","email":"zhaolei@cn.fujitsu.com","username":"zhaolei"},"change_message_id":"91d5a65731ed31dafc749d7a8efc12108c12ad7e","unresolved":false,"context_lines":[{"line_number":124,"context_line":"of them will be providing vCPUs with an improved worst case latency"},{"line_number":125,"context_line":"guarantee, as compared to non-realtime instances. The tenant user does"},{"line_number":126,"context_line":"not need to know the details of how the requirements are met, merely"},{"line_number":127,"context_line":"that the cloud can support the neccessary latency guarantees."},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"In the case of the libvirt driver with the KVM hypervisor, it is expected"},{"line_number":130,"context_line":"that setting the real time flavour will result in the following guest"}],"source_content_type":"text/x-rst","patch_set":10,"id":"9a1a9d01_2951c20e","line":127,"updated":"2015-09-28 08:12:47.000000000","message":"necessary","commit_id":"4b93bf3bf9bf62e97faa3dc905b0893bf0343aa2"},{"author":{"_account_id":17789,"name":"Zhao Lei","email":"zhaolei@cn.fujitsu.com","username":"zhaolei"},"change_message_id":"91d5a65731ed31dafc749d7a8efc12108c12ad7e","unresolved":false,"context_lines":[{"line_number":171,"context_line":"cannot reserve a vCPU for real-time tasks. This would require"},{"line_number":172,"context_line":"the schedular to treat the emulator threads as essentially being"},{"line_number":173,"context_line":"a virtual CPU in their own right. Such an enhancement is considered"},{"line_number":174,"context_line":"out of scope for this blueprint in order to remove any dependancy"},{"line_number":175,"context_line":"on schedular modifications. It will be dealt with in a new blueprint"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads"}],"source_content_type":"text/x-rst","patch_set":10,"id":"9a1a9d01_89424e3d","line":174,"updated":"2015-09-28 08:12:47.000000000","message":"dependency","commit_id":"4b93bf3bf9bf62e97faa3dc905b0893bf0343aa2"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"b3e93a709e76c55d22a8be8a646b8d541df0a168","unresolved":false,"context_lines":[{"line_number":112,"context_line":"supporting realtime."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"A pre-requisite for enabling the realtime feature on a flavour is that"},{"line_number":115,"context_line":"it must also have \u0027hw:cpu_policy\u0027 is set to \u0027dedicated\u0027. ie all real"},{"line_number":116,"context_line":"time guests must have exclusive pCPUs assigned to them. You cannot give"},{"line_number":117,"context_line":"a real time policy to vCPUs that are susceptible to overcommit, as that"},{"line_number":118,"context_line":"would lead to starvation of the other guests on that pCPU, as well as"}],"source_content_type":"text/x-rst","patch_set":11,"id":"9a1a9d01_1103e910","line":115,"updated":"2015-10-02 14:54:15.000000000","message":"its tempting to just assume that, but thats a code detail in many ways.","commit_id":"cdcfda56ba0ae270a831567e0412e8d730cc76c6"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"b3e93a709e76c55d22a8be8a646b8d541df0a168","unresolved":false,"context_lines":[{"line_number":172,"context_line":"the schedular to treat the emulator threads as essentially being"},{"line_number":173,"context_line":"a virtual CPU in their own right. Such an enhancement is considered"},{"line_number":174,"context_line":"out of scope for this blueprint in order to remove any dependency"},{"line_number":175,"context_line":"on schedular modifications. It will be dealt with in a new blueprint"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads"},{"line_number":178,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"9a1a9d01_a7d2ff32","line":175,"updated":"2015-10-02 14:54:15.000000000","message":"OK, good split.","commit_id":"cdcfda56ba0ae270a831567e0412e8d730cc76c6"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"b3e93a709e76c55d22a8be8a646b8d541df0a168","unresolved":false,"context_lines":[{"line_number":174,"context_line":"out of scope for this blueprint in order to remove any dependency"},{"line_number":175,"context_line":"on schedular modifications. It will be dealt with in a new blueprint"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"A significant portion of the work required will be documenting the"},{"line_number":180,"context_line":"required compute host and guest OS setup, as much of this cannot be"}],"source_content_type":"text/x-rst","patch_set":11,"id":"9a1a9d01_a7a0bfa6","line":177,"range":{"start_line":177,"start_character":2,"end_line":177,"end_character":70},"updated":"2015-10-02 14:54:15.000000000","message":"This link doesn\u0027t appear to work? But we can fix that, so not blocking on it.","commit_id":"cdcfda56ba0ae270a831567e0412e8d730cc76c6"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"32442d6964401995b97851a2cfbd45af280f2207","unresolved":false,"context_lines":[{"line_number":174,"context_line":"out of scope for this blueprint in order to remove any dependency"},{"line_number":175,"context_line":"on schedular modifications. It will be dealt with in a new blueprint"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"A significant portion of the work required will be documenting the"},{"line_number":180,"context_line":"required compute host and guest OS setup, as much of this cannot be"}],"source_content_type":"text/x-rst","patch_set":11,"id":"9a1a9d01_22681dba","line":177,"in_reply_to":"9a1a9d01_a7a0bfa6","updated":"2015-10-02 15:02:43.000000000","message":"Should have been this one https://blueprints.launchpad.net/nova/+spec/libvirt-emulator-threads-policy","commit_id":"cdcfda56ba0ae270a831567e0412e8d730cc76c6"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a176cb8e20fc68661bf3b778c036f9dfd4568f8a","unresolved":false,"context_lines":[{"line_number":49,"context_line":""},{"line_number":50,"context_line":"As an indication of the benefits and tradeoffs of realtime, it is useful"},{"line_number":51,"context_line":"to consider some real performance numbers. With bare metal and dedicated"},{"line_number":52,"context_line":"CPUs but non-realtime schedular policy, worst case latency is on the order"},{"line_number":53,"context_line":"of 150 microseconds, and mean latency is approx 2 microseconds. With KVM"},{"line_number":54,"context_line":"and dedicated CPUs and a realtime schedular policy, worst case latency"},{"line_number":55,"context_line":"is 14 microseconds, and mean latency is \u003c 10 microseconds. This shows"}],"source_content_type":"text/x-rst","patch_set":12,"id":"9a1a9d01_1daf78e4","line":52,"range":{"start_line":52,"start_character":22,"end_line":52,"end_character":31},"updated":"2015-10-02 15:32:52.000000000","message":"\"scheduler\" is misspelled.","commit_id":"ea7dc53d78eeb275f0a1b70ce066aab8fe3a4c7a"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a176cb8e20fc68661bf3b778c036f9dfd4568f8a","unresolved":false,"context_lines":[{"line_number":51,"context_line":"to consider some real performance numbers. With bare metal and dedicated"},{"line_number":52,"context_line":"CPUs but non-realtime schedular policy, worst case latency is on the order"},{"line_number":53,"context_line":"of 150 microseconds, and mean latency is approx 2 microseconds. With KVM"},{"line_number":54,"context_line":"and dedicated CPUs and a realtime schedular policy, worst case latency"},{"line_number":55,"context_line":"is 14 microseconds, and mean latency is \u003c 10 microseconds. This shows"},{"line_number":56,"context_line":"that while realtime brings significant benefits in worst case latency,"},{"line_number":57,"context_line":"the mean latency is still significantly higher than that achieved on"}],"source_content_type":"text/x-rst","patch_set":12,"id":"9a1a9d01_bded2c27","line":54,"range":{"start_line":54,"start_character":34,"end_line":54,"end_character":43},"updated":"2015-10-02 15:32:52.000000000","message":"here too.","commit_id":"ea7dc53d78eeb275f0a1b70ce066aab8fe3a4c7a"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a176cb8e20fc68661bf3b778c036f9dfd4568f8a","unresolved":false,"context_lines":[{"line_number":55,"context_line":"is 14 microseconds, and mean latency is \u003c 10 microseconds. This shows"},{"line_number":56,"context_line":"that while realtime brings significant benefits in worst case latency,"},{"line_number":57,"context_line":"the mean latency is still significantly higher than that achieved on"},{"line_number":58,"context_line":"bare metal with non-realtime policy. This serves to re-inforce the point"},{"line_number":59,"context_line":"that realtime is not something to unconditionally use, it is only"},{"line_number":60,"context_line":"suitable for specific workloads that require latency guarantees. Many"},{"line_number":61,"context_line":"apps will find dedicated CPUs alone to be sufficient for their needs."}],"source_content_type":"text/x-rst","patch_set":12,"id":"9a1a9d01_3dc11c81","line":58,"range":{"start_line":58,"start_character":52,"end_line":58,"end_character":62},"updated":"2015-10-02 15:32:52.000000000","message":"The hyphenated case is \"re-enforce\", but I think \"reinforce\" is what you wanted here.","commit_id":"ea7dc53d78eeb275f0a1b70ce066aab8fe3a4c7a"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"a176cb8e20fc68661bf3b778c036f9dfd4568f8a","unresolved":false,"context_lines":[{"line_number":106,"context_line":""},{"line_number":107,"context_line":"* hw:cpu_realtime\u003dyes|no"},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"which will indicate whether instances booted with that flavour will be"},{"line_number":110,"context_line":"run with a realtime policy. Flavours with this property set to \u0027yes\u0027"},{"line_number":111,"context_line":"will need to be associated with the host aggregate that contains hosts"},{"line_number":112,"context_line":"supporting realtime."}],"source_content_type":"text/x-rst","patch_set":12,"id":"9a1a9d01_1d95383f","line":109,"range":{"start_line":109,"start_character":55,"end_line":109,"end_character":62},"updated":"2015-10-02 15:32:52.000000000","message":"I think we\u0027ve had the conversation about being consistent about spelling flavor with an \"or\" in our code and other docs. Since this is documentation, I think it should follow. There is only one other use of flavour in the specs tree.","commit_id":"ea7dc53d78eeb275f0a1b70ce066aab8fe3a4c7a"}]}