)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Artom Lifshitz \u003califshit@redhat.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2018-03-21 20:01:51 -0400"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"NUMA-aware live migration"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When an instance with NUMA characteristics is live-migrated, those"},{"line_number":10,"context_line":"characteristics are not recalculated on the destination compute host."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"df7087c5_31d67629","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":10},"updated":"2018-03-22 00:57:44.000000000","message":"can we rename this to EPA-aware live migration\nas cpu pinning is not a numa feature.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"51ab84c8671f41c1ba6566b20a3b43af26dd5d44","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Artom Lifshitz \u003califshit@redhat.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2018-03-21 20:01:51 -0400"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"NUMA-aware live migration"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When an instance with NUMA characteristics is live-migrated, those"},{"line_number":10,"context_line":"characteristics are not recalculated on the destination compute host."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"bf659307_6ce80316","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":10},"in_reply_to":"df7087c5_31d67629","updated":"2018-03-23 18:40:20.000000000","message":"and EPA is an Intel-specific thing, so -1 on that suggestion.\n\nhttps://networkbuilders.intel.com/network-technologies/enhancedplatformawareness","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":11,"context_line":"In the CPU pinning case, using the source host\u0027s pin mappings on the"},{"line_number":12,"context_line":"destination can lead to multiple instances being pinned to the same"},{"line_number":13,"context_line":"pCPUs. In the case of hugepage-backed instances, which are"},{"line_number":14,"context_line":"NUMA-localized, an instance needs to have its NUMA topology"},{"line_number":15,"context_line":"recalculated on the destination compute host during a live migration."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Change-Id: I1c75c555821e9eb7c51deb7eee17e3977cf45f18"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"df7087c5_d1b5baef","line":14,"range":{"start_line":14,"start_character":46,"end_line":14,"end_character":59},"updated":"2018-03-22 00:57:44.000000000","message":"virtual numa topology mappings","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"}],"specs/rocky/approved/live-migration-with-cpu-pinning.rst":[{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"88afe5df2b5c714c3044582c19da70fb7422d326","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When an instance with pinned CPUs is live-migrated, the pinning is not"},{"line_number":14,"context_line":"recalculated on the destination compute host. This can lead to multiple"},{"line_number":15,"context_line":"instances being pinned to the same pCPUs, violating their ``dedicated`` CPU"},{"line_number":16,"context_line":"policy and causing unpredictable performance degradation. This is a"},{"line_number":17,"context_line":"long-standing and well-known problem with a fix in progress since 2015. This"},{"line_number":18,"context_line":"spec assumes that the proposed fix has accumulated too much negative inertia to"},{"line_number":19,"context_line":"merge in its current state, and proposes a new solution."}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_5259638b","line":16,"range":{"start_line":15,"start_character":42,"end_line":16,"end_character":7},"updated":"2018-03-14 17:15:37.000000000","message":"only if that\u0027s the asked policy... ;-)","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"88afe5df2b5c714c3044582c19da70fb7422d326","unresolved":false,"context_lines":[{"line_number":14,"context_line":"recalculated on the destination compute host. This can lead to multiple"},{"line_number":15,"context_line":"instances being pinned to the same pCPUs, violating their ``dedicated`` CPU"},{"line_number":16,"context_line":"policy and causing unpredictable performance degradation. This is a"},{"line_number":17,"context_line":"long-standing and well-known problem with a fix in progress since 2015. This"},{"line_number":18,"context_line":"spec assumes that the proposed fix has accumulated too much negative inertia to"},{"line_number":19,"context_line":"merge in its current state, and proposes a new solution."},{"line_number":20,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_128a0bbe","line":17,"range":{"start_line":17,"start_character":37,"end_line":17,"end_character":70},"updated":"2018-03-14 17:15:37.000000000","message":"well, that\u0027s a tribal knowledge that I\u0027m not sure we should document it in a spec :-)\nIt\u0027s not just about merging a patch, it\u0027s about understanding all the implications that patch carries, hence that new spec.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"256245c11bcc617760326ecf8e32dfcc7b254868","unresolved":false,"context_lines":[{"line_number":14,"context_line":"recalculated on the destination compute host. This can lead to multiple"},{"line_number":15,"context_line":"instances being pinned to the same pCPUs, violating their ``dedicated`` CPU"},{"line_number":16,"context_line":"policy and causing unpredictable performance degradation. This is a"},{"line_number":17,"context_line":"long-standing and well-known problem with a fix in progress since 2015. This"},{"line_number":18,"context_line":"spec assumes that the proposed fix has accumulated too much negative inertia to"},{"line_number":19,"context_line":"merge in its current state, and proposes a new solution."},{"line_number":20,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_6206fe54","line":17,"range":{"start_line":17,"start_character":37,"end_line":17,"end_character":70},"in_reply_to":"df7087c5_128a0bbe","updated":"2018-03-15 10:59:34.000000000","message":"Agreed","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"256245c11bcc617760326ecf8e32dfcc7b254868","unresolved":false,"context_lines":[{"line_number":15,"context_line":"instances being pinned to the same pCPUs, violating their ``dedicated`` CPU"},{"line_number":16,"context_line":"policy and causing unpredictable performance degradation. This is a"},{"line_number":17,"context_line":"long-standing and well-known problem with a fix in progress since 2015. This"},{"line_number":18,"context_line":"spec assumes that the proposed fix has accumulated too much negative inertia to"},{"line_number":19,"context_line":"merge in its current state, and proposes a new solution."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_f656742a","line":18,"range":{"start_line":18,"start_character":51,"end_line":18,"end_character":76},"updated":"2018-03-15 10:59:34.000000000","message":"ooooh","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"88afe5df2b5c714c3044582c19da70fb7422d326","unresolved":false,"context_lines":[{"line_number":22,"context_line":"Problem description"},{"line_number":23,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Live migrating instances with pinned CPUs has been known to be problematic"},{"line_number":26,"context_line":"since `2015`_. The currently posted `fix`_ is stuck in review hell and is"},{"line_number":27,"context_line":"unlikely to be merged in its current state. It is a hard to understand patch"},{"line_number":28,"context_line":"that has foiled multitudes of reviewers over the years. The aim of this spec is"},{"line_number":29,"context_line":"to spell out, in English, a solution to the live migration with CPU pinning"},{"line_number":30,"context_line":"problem. The agreed upon solution will then be implemented in a series of"},{"line_number":31,"context_line":"small, hopefully easily digestible patches. The hope is that by agreeing to a"},{"line_number":32,"context_line":"solution in a spec, the actualy code patches will be easier to review."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"},{"line_number":35,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_1278cbc1","line":32,"range":{"start_line":25,"start_character":0,"end_line":32,"end_character":70},"updated":"2018-03-14 17:15:37.000000000","message":"While I nod on the explanation, I think the problem description should only consider commenting the technical problems we have, not the social ones.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":22,"context_line":"Problem description"},{"line_number":23,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Live migrating instances with pinned CPUs has been known to be problematic"},{"line_number":26,"context_line":"since `2015`_. The currently posted `fix`_ is stuck in review hell and is"},{"line_number":27,"context_line":"unlikely to be merged in its current state. It is a hard to understand patch"},{"line_number":28,"context_line":"that has foiled multitudes of reviewers over the years. The aim of this spec is"},{"line_number":29,"context_line":"to spell out, in English, a solution to the live migration with CPU pinning"},{"line_number":30,"context_line":"problem. The agreed upon solution will then be implemented in a series of"},{"line_number":31,"context_line":"small, hopefully easily digestible patches. The hope is that by agreeing to a"},{"line_number":32,"context_line":"solution in a spec, the actualy code patches will be easier to review."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"},{"line_number":35,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_731fb207","line":32,"range":{"start_line":25,"start_character":0,"end_line":32,"end_character":70},"in_reply_to":"df7087c5_1278cbc1","updated":"2018-03-15 12:42:04.000000000","message":"Agree with Sylvain. I would just limit the problem description to a discussion of the primary issue: the inability to assure a similar pinned topology on the destination node during live migration.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7a9783015d094c35b60b8aa67d186c95f7fe8df1","unresolved":false,"context_lines":[{"line_number":22,"context_line":"Problem description"},{"line_number":23,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Live migrating instances with pinned CPUs has been known to be problematic"},{"line_number":26,"context_line":"since `2015`_. The currently posted `fix`_ is stuck in review hell and is"},{"line_number":27,"context_line":"unlikely to be merged in its current state. It is a hard to understand patch"},{"line_number":28,"context_line":"that has foiled multitudes of reviewers over the years. The aim of this spec is"},{"line_number":29,"context_line":"to spell out, in English, a solution to the live migration with CPU pinning"},{"line_number":30,"context_line":"problem. The agreed upon solution will then be implemented in a series of"},{"line_number":31,"context_line":"small, hopefully easily digestible patches. The hope is that by agreeing to a"},{"line_number":32,"context_line":"solution in a spec, the actualy code patches will be easier to review."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"},{"line_number":35,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_57de11da","line":32,"range":{"start_line":25,"start_character":0,"end_line":32,"end_character":70},"in_reply_to":"df7087c5_731fb207","updated":"2018-03-15 17:20:59.000000000","message":"And let\u0027s not artificially limit this to CPU pinning, it comes into play due to hugepages (which are NUMA-specific) as well.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7a9783015d094c35b60b8aa67d186c95f7fe8df1","unresolved":false,"context_lines":[{"line_number":34,"context_line":"Use Cases"},{"line_number":35,"context_line":"---------"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"As a cloud administrator, I want to live migrate instances with CPU pinning"},{"line_number":38,"context_line":"without the pin mappings overlapping on the destination compute host."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Proposed change"},{"line_number":41,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_d7eb0175","line":38,"range":{"start_line":37,"start_character":0,"end_line":38,"end_character":69},"updated":"2018-03-15 17:20:59.000000000","message":"As mentioned above, we also need to recalculate resources for hugepage-backed instances (which are NUMA-localized, and so we need to recalculate which NUMA node to land on at the destination compute node).","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"As an oversimplification for the purposes of this spec, live migration with CPU"},{"line_number":44,"context_line":"pinning can be broken down into two phases: the scheduler phase and the compute"},{"line_number":45,"context_line":"phase. In the scheduler phase, the problem to solve is claiming pinned pCPUs on"},{"line_number":46,"context_line":"the destination compute host. In the compute phase, the problem to solve is"},{"line_number":47,"context_line":"calculating the exact pin mapping for the instance being migrated."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"In the scheduler phase, regardless of whether a destination host is specified"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_93a92691","line":46,"range":{"start_line":45,"start_character":7,"end_line":46,"end_character":29},"updated":"2018-03-15 12:42:04.000000000","message":"Unfortunately, this is not true. The scheduler does not *claim* any pCPU resources on the destination compute host, unlike other types of resources (VCPU, MEMORY_MB, DISK_GB). In fact, this is one of the root problems with all the NUMA/pinning resources in nova: the scheduler does *not* claim those resources. The NUMATopologyFilter calls hardware.numa_fit_instance_to_host(), immediately forgets the topology that was determined by that routine, and instead relies on the destination compute host to actually claim the resources by re-calling hardware.numa_fit_instance_to_host() and actually saving the resulting topology to the nova cell DB\u0027s compute_nodes.numa_topology field.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":47,"context_line":"calculating the exact pin mapping for the instance being migrated."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"In the scheduler phase, regardless of whether a destination host is specified"},{"line_number":50,"context_line":"in the API request by the administrator, resources are claimed on the"},{"line_number":51,"context_line":"destination compute host. If the destination is not specified, the claim"},{"line_number":52,"context_line":"happens in `select_destinations`_, called from `find_destination`_ in the"},{"line_number":53,"context_line":"live migrate task. If the destination is specified, the claim happens in the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_93b046df","line":50,"range":{"start_line":50,"start_character":41,"end_line":50,"end_character":50},"updated":"2018-03-15 12:42:04.000000000","message":"see above. this needs to be clarified. No *NUMA/pinning* resources are claimed in the scheduler. Only VCPU, MEMORY_MB, DISK_GB, VGPU and custom resource classes.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":51,"context_line":"destination compute host. If the destination is not specified, the claim"},{"line_number":52,"context_line":"happens in `select_destinations`_, called from `find_destination`_ in the"},{"line_number":53,"context_line":"live migrate task. If the destination is specified, the claim happens in the"},{"line_number":54,"context_line":"live migrate task `itself`_. Thus, once pCPUs and vCPUs are modelled in"},{"line_number":55,"context_line":"placement, claiming them for live migration should happen automagically. The"},{"line_number":56,"context_line":"`Support shared and dedicated VMs in one host`_ spec proposes to do just that."},{"line_number":57,"context_line":"This part of the problem is therefore not discussed here, and this spec is"},{"line_number":58,"context_line":"concurrently dependant on the *CPUs in placement* work item of the *shared and"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_b336aa50","line":55,"range":{"start_line":54,"start_character":29,"end_line":55,"end_character":72},"updated":"2018-03-15 12:42:04.000000000","message":"Clarification: placement never intends to store the actual pinsets for CPU topology constraints. Placement only intends to store a bucket of PCPU resource inventories for each NUMA node (with each NUMA node represented by a resource provider under the compute node parent provider). So, \"claiming\" of PCPU resources will be the simple process of deducting some integer amount of PCPU inventory from one or more NUMA node resource providers by allocating that amount to the instance. However, the actual mapping of vCPU -\u003e pCPU will absolutely *not* be happening in the placement service. I think it\u0027s important that this spec be explicit about those aims.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":55,"context_line":"placement, claiming them for live migration should happen automagically. The"},{"line_number":56,"context_line":"`Support shared and dedicated VMs in one host`_ spec proposes to do just that."},{"line_number":57,"context_line":"This part of the problem is therefore not discussed here, and this spec is"},{"line_number":58,"context_line":"concurrently dependant on the *CPUs in placement* work item of the *shared and"},{"line_number":59,"context_line":"dedicated VMs in one host* spec."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"In the compute phase, new pin mappings must be calculated on the destination"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_ce4ff168","line":58,"range":{"start_line":58,"start_character":13,"end_line":58,"end_character":22},"updated":"2018-03-15 12:42:04.000000000","message":"dependent","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"88afe5df2b5c714c3044582c19da70fb7422d326","unresolved":false,"context_lines":[{"line_number":56,"context_line":"`Support shared and dedicated VMs in one host`_ spec proposes to do just that."},{"line_number":57,"context_line":"This part of the problem is therefore not discussed here, and this spec is"},{"line_number":58,"context_line":"concurrently dependant on the *CPUs in placement* work item of the *shared and"},{"line_number":59,"context_line":"dedicated VMs in one host* spec."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"In the compute phase, new pin mappings must be calculated on the destination"},{"line_number":62,"context_line":"host in order to avoid pinning the migrating instance to pCPUs that already"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_f253d737","line":59,"updated":"2018-03-14 17:15:37.000000000","message":"Also note https://review.openstack.org/#/c/552924/ that\u0027s kinda related.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7a9783015d094c35b60b8aa67d186c95f7fe8df1","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"In the compute phase, new pin mappings must be calculated on the destination"},{"line_number":62,"context_line":"host in order to avoid pinning the migrating instance to pCPUs that already"},{"line_number":63,"context_line":"have vCPUs pinned to them. Currently, the compute manager `calls`_"},{"line_number":64,"context_line":"pre_live_migration on the destination host and obtains migrate_data, a"},{"line_number":65,"context_line":"LiveMigrateData object. This object is then passed to the virt driver in `its"},{"line_number":66,"context_line":"live_migrate call`_. The LiveMigrateData object will be modified to add a new"},{"line_number":67,"context_line":"object field that will contain a NUMACell object describing the pinning"},{"line_number":68,"context_line":"topology on the destination host. The libvirt driver on the migration source"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_372d9da7","line":65,"range":{"start_line":63,"start_character":27,"end_line":65,"end_character":23},"updated":"2018-03-15 17:20:59.000000000","message":"Not quite.  The conductor calls check_can_live_migrate_destination() on the destination, which calls check_can_live_migrate_source(), and then passes back the migrate data information.  This is then passed down to the source compute node in the live_migration() call, which passes it to the destination in the pre_live_migration() call.\n\nIn the previous patch, the resource claim was done on the destination compute node as part of check_can_live_migrate_destination(), and the resulting claimed resources were included in the migration data that was returned back.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"88afe5df2b5c714c3044582c19da70fb7422d326","unresolved":false,"context_lines":[{"line_number":67,"context_line":"object field that will contain a NUMACell object describing the pinning"},{"line_number":68,"context_line":"topology on the destination host. The libvirt driver on the migration source"},{"line_number":69,"context_line":"will use this information when generating the destination XML to include the"},{"line_number":70,"context_line":"new CPU pin mapping."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Alternatives"},{"line_number":73,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_b5a789bd","line":70,"updated":"2018-03-14 17:15:37.000000000","message":"assuming that if a compute service is older enough, then the LiveMigrateData object wouldn\u0027t include that field, right?\nIf so, what should do the source node ? Just pass the existing ?\n\nShould we also consider the opposite, ie. what if I\u0027m the destination, should I assume everything given by the source node as legit ?","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7a9783015d094c35b60b8aa67d186c95f7fe8df1","unresolved":false,"context_lines":[{"line_number":67,"context_line":"object field that will contain a NUMACell object describing the pinning"},{"line_number":68,"context_line":"topology on the destination host. The libvirt driver on the migration source"},{"line_number":69,"context_line":"will use this information when generating the destination XML to include the"},{"line_number":70,"context_line":"new CPU pin mapping."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Alternatives"},{"line_number":73,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_3234cbdd","line":70,"in_reply_to":"df7087c5_02c4e2f3","updated":"2018-03-15 17:20:59.000000000","message":"Are you suggesting that once we upgrade nova-conductor to this new version we\u0027d refuse to do live migration for instances with dedicated CPUs or hugepages until both compute nodes have been upgraded?","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":67,"context_line":"object field that will contain a NUMACell object describing the pinning"},{"line_number":68,"context_line":"topology on the destination host. The libvirt driver on the migration source"},{"line_number":69,"context_line":"will use this information when generating the destination XML to include the"},{"line_number":70,"context_line":"new CPU pin mapping."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Alternatives"},{"line_number":73,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_ce9db1bc","line":70,"in_reply_to":"df7087c5_02c4e2f3","updated":"2018-03-15 12:42:04.000000000","message":"agree with @stephenfin","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"256245c11bcc617760326ecf8e32dfcc7b254868","unresolved":false,"context_lines":[{"line_number":67,"context_line":"object field that will contain a NUMACell object describing the pinning"},{"line_number":68,"context_line":"topology on the destination host. The libvirt driver on the migration source"},{"line_number":69,"context_line":"will use this information when generating the destination XML to include the"},{"line_number":70,"context_line":"new CPU pin mapping."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Alternatives"},{"line_number":73,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_02c4e2f3","line":70,"in_reply_to":"df7087c5_b5a789bd","updated":"2018-03-15 10:59:34.000000000","message":"I\u0027d personally fail to schedule if we don\u0027t have a new enough version on both sides. This is already completely broken and making its brokenness more obvious seems like a good thing to do.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":80,"context_line":"LiveMigrateData with a new field to communicate pinning information from the"},{"line_number":81,"context_line":"destination to the source."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"Instead of sending new pinning information though LiveMigrateData from the"},{"line_number":84,"context_line":"destination to the source, it would be conceivable to re-use the result of"},{"line_number":85,"context_line":"`numa_fit_instance_to_host`_, as called from _schedule in the filter scheduler."},{"line_number":86,"context_line":"This would require modifying the Selection object to include pinning"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_8e7c9972","line":83,"range":{"start_line":83,"start_character":43,"end_line":83,"end_character":49},"updated":"2018-03-15 12:42:04.000000000","message":"through","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"256245c11bcc617760326ecf8e32dfcc7b254868","unresolved":false,"context_lines":[{"line_number":87,"context_line":"information, which will then be `passed into the conductor live migrate task`_,"},{"line_number":88,"context_line":"and from there down to the virt driver.  While this does technically save CPU"},{"line_number":89,"context_line":"cycles, the complexity of bubbling the pinning information all the way up and"},{"line_number":90,"context_line":"then all the way down makes this a poor alternative. In addition, this would"},{"line_number":91,"context_line":"only work if a destination host is not specified in the API request and"},{"line_number":92,"context_line":"find_destinations is called by the live migrate task."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"Data model impact"},{"line_number":95,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_62791ec6","line":92,"range":{"start_line":90,"start_character":53,"end_line":92,"end_character":53},"updated":"2018-03-15 10:59:34.000000000","message":"Also, it\u0027s highly likely that there will be race conditions if something claims those resources before we get to the destination. I guess that\u0027s going to be an issue with any solution though...","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"72bceff9db5435d2e68ff68ae003e5141477cdff","unresolved":false,"context_lines":[{"line_number":87,"context_line":"information, which will then be `passed into the conductor live migrate task`_,"},{"line_number":88,"context_line":"and from there down to the virt driver.  While this does technically save CPU"},{"line_number":89,"context_line":"cycles, the complexity of bubbling the pinning information all the way up and"},{"line_number":90,"context_line":"then all the way down makes this a poor alternative. In addition, this would"},{"line_number":91,"context_line":"only work if a destination host is not specified in the API request and"},{"line_number":92,"context_line":"find_destinations is called by the live migrate task."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"Data model impact"},{"line_number":95,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_f0cb1da2","line":92,"range":{"start_line":90,"start_character":53,"end_line":92,"end_character":53},"in_reply_to":"df7087c5_52b29f4a","updated":"2018-03-21 23:53:04.000000000","message":"Maybe do this in a follow-up? I feel like going from broken to working is a good first step ;)","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3823d8a10545a5b7b4c6c9be118595099b7b7f37","unresolved":false,"context_lines":[{"line_number":87,"context_line":"information, which will then be `passed into the conductor live migrate task`_,"},{"line_number":88,"context_line":"and from there down to the virt driver.  While this does technically save CPU"},{"line_number":89,"context_line":"cycles, the complexity of bubbling the pinning information all the way up and"},{"line_number":90,"context_line":"then all the way down makes this a poor alternative. In addition, this would"},{"line_number":91,"context_line":"only work if a destination host is not specified in the API request and"},{"line_number":92,"context_line":"find_destinations is called by the live migrate task."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"Data model impact"},{"line_number":95,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_6eda6578","line":92,"range":{"start_line":90,"start_character":53,"end_line":92,"end_character":53},"in_reply_to":"df7087c5_62791ec6","updated":"2018-03-15 12:42:04.000000000","message":"Right. This is what the resource provider generation guards against when claiming the other resource class inventories. If we did this alternative, we would need to add a generation to the compute_nodes table and perform the same kind of consistency checks that we do in placement -- and that would be a nightmare.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7a9783015d094c35b60b8aa67d186c95f7fe8df1","unresolved":false,"context_lines":[{"line_number":87,"context_line":"information, which will then be `passed into the conductor live migrate task`_,"},{"line_number":88,"context_line":"and from there down to the virt driver.  While this does technically save CPU"},{"line_number":89,"context_line":"cycles, the complexity of bubbling the pinning information all the way up and"},{"line_number":90,"context_line":"then all the way down makes this a poor alternative. In addition, this would"},{"line_number":91,"context_line":"only work if a destination host is not specified in the API request and"},{"line_number":92,"context_line":"find_destinations is called by the live migrate task."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"Data model impact"},{"line_number":95,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_52b29f4a","line":92,"range":{"start_line":90,"start_character":53,"end_line":92,"end_character":53},"in_reply_to":"df7087c5_6eda6578","updated":"2018-03-15 17:20:59.000000000","message":"As a performance optimization we could cache the result from the scheduler and optimistically try to use it on the compute node.  If the claim fails then we\u0027d do the full numa_fit_instance_to_host() again.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"88afe5df2b5c714c3044582c19da70fb7422d326","unresolved":false,"context_lines":[{"line_number":94,"context_line":"Data model impact"},{"line_number":95,"context_line":"-----------------"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"A new field is added to LiveMigrateData."},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"REST API impact"},{"line_number":100,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_15cd1565","line":97,"updated":"2018-03-14 17:15:37.000000000","message":"please be more precise : how do you plan to model the NUMA topology with that field ? of which type ?","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"256245c11bcc617760326ecf8e32dfcc7b254868","unresolved":false,"context_lines":[{"line_number":161,"context_line":"Testing"},{"line_number":162,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Live migration is currently tested in the gate without CPU pinning. Since no"},{"line_number":165,"context_line":"special hardware is required, it would be possible to change the job\u0027s"},{"line_number":166,"context_line":"configuration to enable CPU pinning and add a live migration CPU pinning"},{"line_number":167,"context_line":"tempest test."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"Documentation Impact"},{"line_number":170,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_6227decb","line":167,"range":{"start_line":164,"start_character":68,"end_line":167,"end_character":13},"updated":"2018-03-15 10:59:34.000000000","message":"Alas, you need nested virt (meaning hardware and OS support for VT-x or AMD\u0027s equivalent, iirc) for this to work. We can\u0027t rely on this hence why we don\u0027t currently test any of this in the gate. You\u0027re stuck with third-party CIs (Intel NFV CI) and test suites (whitebox tests) to validate this.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":6962,"name":"Kashyap Chamarthy","email":"kchamart@redhat.com","username":"kashyapc"},"change_message_id":"d911e3468317398d7f02c3d6b36f941a715f33ea","unresolved":false,"context_lines":[{"line_number":161,"context_line":"Testing"},{"line_number":162,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Live migration is currently tested in the gate without CPU pinning. Since no"},{"line_number":165,"context_line":"special hardware is required, it would be possible to change the job\u0027s"},{"line_number":166,"context_line":"configuration to enable CPU pinning and add a live migration CPU pinning"},{"line_number":167,"context_line":"tempest test."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"Documentation Impact"},{"line_number":170,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_f47ba888","line":167,"range":{"start_line":164,"start_character":68,"end_line":167,"end_character":13},"in_reply_to":"df7087c5_6227decb","updated":"2018-03-15 13:35:20.000000000","message":"On the KVM Nested Virt in the upstream gate thing — not an immediate solution,  but one of the hosting providers (OVH) was amenable to turning it on, after doing some testing.\n\nThere\u0027s no immediate time-line; but from the Cloud Providers Working Group meet-up at Dublin PTG, work is in progress to enable Nested Virt by at least a couple of hosting providers that the upstream Infra Team uses.","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7a9783015d094c35b60b8aa67d186c95f7fe8df1","unresolved":false,"context_lines":[{"line_number":161,"context_line":"Testing"},{"line_number":162,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Live migration is currently tested in the gate without CPU pinning. Since no"},{"line_number":165,"context_line":"special hardware is required, it would be possible to change the job\u0027s"},{"line_number":166,"context_line":"configuration to enable CPU pinning and add a live migration CPU pinning"},{"line_number":167,"context_line":"tempest test."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"Documentation Impact"},{"line_number":170,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_b258dbf9","line":167,"range":{"start_line":164,"start_character":68,"end_line":167,"end_character":13},"in_reply_to":"df7087c5_6227decb","updated":"2018-03-15 17:20:59.000000000","message":"There\u0027s a spec to enable NUMA with software qemu, which might help with this in the long term.  https://review.openstack.org/#/c/533077/","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"ef76959c56996a77deb41e2de93dea15cfb4e775","unresolved":false,"context_lines":[{"line_number":161,"context_line":"Testing"},{"line_number":162,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Live migration is currently tested in the gate without CPU pinning. Since no"},{"line_number":165,"context_line":"special hardware is required, it would be possible to change the job\u0027s"},{"line_number":166,"context_line":"configuration to enable CPU pinning and add a live migration CPU pinning"},{"line_number":167,"context_line":"tempest test."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"Documentation Impact"},{"line_number":170,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_0ad6f9f2","line":167,"range":{"start_line":164,"start_character":68,"end_line":167,"end_character":13},"in_reply_to":"df7087c5_b258dbf9","updated":"2018-03-19 02:53:27.000000000","message":"Thank you for picking up that spec, chris. Following your comments here and sean\u0027s comment[1] in that spec, I\u0027m thinking to support cpu pinning feature with software qemu as well in the spec. The patch is uploaded[2].\n\n[1] https://review.openstack.org/#/c/533077/5/specs/rocky/approved/libvirt-qemu-numa-support.rst@97\n[2] https://review.openstack.org/#/c/554076/","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"256245c11bcc617760326ecf8e32dfcc7b254868","unresolved":false,"context_lines":[{"line_number":178,"context_line":""},{"line_number":179,"context_line":".. _2015: https://bugs.launchpad.net/nova/+bug/1417667"},{"line_number":180,"context_line":".. _fix: https://review.openstack.org/#/c/244489/"},{"line_number":181,"context_line":".. _select_destinations: https://github.com/openstack/nova/blob/master/nova/scheduler/manager.py#L85"},{"line_number":182,"context_line":".. _find_destination: https://github.com/openstack/nova/blob/master/nova/conductor/tasks/live_migrate.py#L308"},{"line_number":183,"context_line":".. _itself: https://github.com/openstack/nova/blob/master/nova/conductor/tasks/live_migrate.py#L93"},{"line_number":184,"context_line":".. _Support shared and dedicated VMs in one host: https://review.openstack.org/#/c/543805/"},{"line_number":185,"context_line":".. _passed into the conductor live migrate task: https://github.com/openstack/nova/blob/master/nova/conductor/tasks/live_migrate.py#L319"},{"line_number":186,"context_line":".. _numa_fit_instance_to_host: https://github.com/openstack/nova/blob/master/nova/scheduler/host_manager.py#L297"},{"line_number":187,"context_line":".. _calls: https://github.com/openstack/nova/blob/master/nova/compute/manager.py#L5982"},{"line_number":188,"context_line":".. _its live_migrate call: https://github.com/openstack/nova/blob/master/nova/compute/manager.py#L6062"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"History"},{"line_number":191,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_0505cc10","line":188,"range":{"start_line":181,"start_character":0,"end_line":188,"end_character":102},"updated":"2018-03-15 10:59:34.000000000","message":"All of these are going to go out of date pretty quickly. You should reference them from a particular commit (press \u0027y\u0027 on a page) or, better yet, a tag (the latest stable release?)","commit_id":"f34b9819256350c0c9237217256a13ca00623dff"}],"specs/rocky/approved/numa-aware-live-migration.rst":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-live-migration"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When an instance with NUMA characteristics is live-migrated, those"},{"line_number":14,"context_line":"characteristics are not recalculated on the destination compute host. In the"},{"line_number":15,"context_line":"CPU pinning case, using the source host\u0027s pin mappings on the destination can"},{"line_number":16,"context_line":"lead to multiple instances being pinned to the same pCPUs. In the case of"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_3108b6d7","line":13,"range":{"start_line":13,"start_character":22,"end_line":13,"end_character":26},"updated":"2018-03-22 00:57:44.000000000","message":"this is a miss use of NUMA this should be EPA","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-live-migration"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When an instance with NUMA characteristics is live-migrated, those"},{"line_number":14,"context_line":"characteristics are not recalculated on the destination compute host. In the"},{"line_number":15,"context_line":"CPU pinning case, using the source host\u0027s pin mappings on the destination can"},{"line_number":16,"context_line":"lead to multiple instances being pinned to the same pCPUs. In the case of"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_609f6a03","line":13,"range":{"start_line":13,"start_character":22,"end_line":13,"end_character":26},"in_reply_to":"bf659307_81a520b8","updated":"2018-03-27 02:46:24.000000000","message":"I propose leaving the introduction (and therefore commit message) as is for brevity, and explaining our misuse of the term NUMA further down in Problem Description.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/numa-aware-live-migration"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When an instance with NUMA characteristics is live-migrated, those"},{"line_number":14,"context_line":"characteristics are not recalculated on the destination compute host. In the"},{"line_number":15,"context_line":"CPU pinning case, using the source host\u0027s pin mappings on the destination can"},{"line_number":16,"context_line":"lead to multiple instances being pinned to the same pCPUs. In the case of"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_81a520b8","line":13,"range":{"start_line":13,"start_character":22,"end_line":13,"end_character":26},"in_reply_to":"df7087c5_3108b6d7","updated":"2018-03-23 18:02:32.000000000","message":"Sean, we\u0027re talking about CPU pinning, hugepages, explicit numa_topology in the guest, etc.  Basically anything that involves the NUMATopologyFilter, hence my suggestion to just call it numa-aware.\n\nYes, it\u0027s not ideal, but I\u0027d suggest that EPA isn\u0027t really appropriate for an explicit guest NUMA topology without pinning or hugepages.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"When an instance with NUMA characteristics is live-migrated, those"},{"line_number":14,"context_line":"characteristics are not recalculated on the destination compute host. In the"},{"line_number":15,"context_line":"CPU pinning case, using the source host\u0027s pin mappings on the destination can"},{"line_number":16,"context_line":"lead to multiple instances being pinned to the same pCPUs. In the case of"},{"line_number":17,"context_line":"hugepage-backed instances, which are NUMA-localized, an instance needs to have"},{"line_number":18,"context_line":"its NUMA topology recalculated on the destination compute host during a live"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_91f8a2c4","line":15,"range":{"start_line":15,"start_character":0,"end_line":15,"end_character":12},"updated":"2018-03-22 00:57:44.000000000","message":"not a numa feature hence why i said EPA above.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":15,"context_line":"CPU pinning case, using the source host\u0027s pin mappings on the destination can"},{"line_number":16,"context_line":"lead to multiple instances being pinned to the same pCPUs. In the case of"},{"line_number":17,"context_line":"hugepage-backed instances, which are NUMA-localized, an instance needs to have"},{"line_number":18,"context_line":"its NUMA topology recalculated on the destination compute host during a live"},{"line_number":19,"context_line":"migration."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_f12a3e2a","line":18,"range":{"start_line":18,"start_character":4,"end_line":18,"end_character":17},"updated":"2018-03-22 00:57:44.000000000","message":"this is a little inacurate. the guest virtual numa topology is unmodified however the virtual numa node to phyical numa node mapping may change.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":15,"context_line":"CPU pinning case, using the source host\u0027s pin mappings on the destination can"},{"line_number":16,"context_line":"lead to multiple instances being pinned to the same pCPUs. In the case of"},{"line_number":17,"context_line":"hugepage-backed instances, which are NUMA-localized, an instance needs to have"},{"line_number":18,"context_line":"its NUMA topology recalculated on the destination compute host during a live"},{"line_number":19,"context_line":"migration."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_208972c9","line":18,"range":{"start_line":18,"start_character":4,"end_line":18,"end_character":17},"in_reply_to":"bf659307_01911053","updated":"2018-03-27 02:46:24.000000000","message":"Can I say \u0027NUMA mapping\u0027 then?","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":15,"context_line":"CPU pinning case, using the source host\u0027s pin mappings on the destination can"},{"line_number":16,"context_line":"lead to multiple instances being pinned to the same pCPUs. In the case of"},{"line_number":17,"context_line":"hugepage-backed instances, which are NUMA-localized, an instance needs to have"},{"line_number":18,"context_line":"its NUMA topology recalculated on the destination compute host during a live"},{"line_number":19,"context_line":"migration."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_01911053","line":18,"range":{"start_line":18,"start_character":4,"end_line":18,"end_character":17},"in_reply_to":"df7087c5_f12a3e2a","updated":"2018-03-23 18:02:32.000000000","message":"++","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"The problem is best described with two examples."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"The first example is live migration with CPU pinning. An instance with a"},{"line_number":27,"context_line":"``dedicated`` CPU policy and pinned CPUs is live-migrated.  Its pin mappings"},{"line_number":28,"context_line":"are naively copied over to the destination host. This creates two problems."},{"line_number":29,"context_line":"First, its pinned pCPUs aren\u0027t properly claimed on the destination. This means"},{"line_number":30,"context_line":"that, should a second instance with pinned CPUs land on the destination, both"},{"line_number":31,"context_line":"instances\u0027s vCPUs could be pinned to the same pCPUs. Second, any existing pin"},{"line_number":32,"context_line":"mappings on the destination are ignored. If another instance already exists on"},{"line_number":33,"context_line":"the destination, both instances\u0027s vCPUs could be pinned to the same pCPUs. In"},{"line_number":34,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"},{"line_number":35,"context_line":"unpredictable performance degradation."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"The second example is an instance with hugepages. Two identical instances are"},{"line_number":38,"context_line":"booted on two identical compute hosts. Both instances have NUMA topologies and"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_91cf427e","line":35,"range":{"start_line":26,"start_character":0,"end_line":35,"end_character":38},"updated":"2018-03-22 00:57:44.000000000","message":"numa is specifically about memory acess so this use case is not a numa usecase but is an epa usecase.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"The problem is best described with two examples."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"The first example is live migration with CPU pinning. An instance with a"},{"line_number":27,"context_line":"``dedicated`` CPU policy and pinned CPUs is live-migrated.  Its pin mappings"},{"line_number":28,"context_line":"are naively copied over to the destination host. This creates two problems."},{"line_number":29,"context_line":"First, its pinned pCPUs aren\u0027t properly claimed on the destination. This means"},{"line_number":30,"context_line":"that, should a second instance with pinned CPUs land on the destination, both"},{"line_number":31,"context_line":"instances\u0027s vCPUs could be pinned to the same pCPUs. Second, any existing pin"},{"line_number":32,"context_line":"mappings on the destination are ignored. If another instance already exists on"},{"line_number":33,"context_line":"the destination, both instances\u0027s vCPUs could be pinned to the same pCPUs. In"},{"line_number":34,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"},{"line_number":35,"context_line":"unpredictable performance degradation."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"The second example is an instance with hugepages. Two identical instances are"},{"line_number":38,"context_line":"booted on two identical compute hosts. Both instances have NUMA topologies and"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_405b4653","line":35,"range":{"start_line":26,"start_character":0,"end_line":35,"end_character":38},"in_reply_to":"df7087c5_91cf427e","updated":"2018-03-27 02:46:24.000000000","message":"Added a blurb about how we\u0027re misusing \u0027NUMA\u0027.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":34,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"},{"line_number":35,"context_line":"unpredictable performance degradation."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"The second example is an instance with hugepages. Two identical instances are"},{"line_number":38,"context_line":"booted on two identical compute hosts. Both instances have NUMA topologies and"},{"line_number":39,"context_line":"hugepages and are on their respective host\u0027s NUMA node 0. If one of the"},{"line_number":40,"context_line":"instances is live-migrated to the other compute host, it will stay on NUMA node"},{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_013b5038","line":41,"range":{"start_line":37,"start_character":0,"end_line":41,"end_character":18},"updated":"2018-03-23 18:02:32.000000000","message":"As Sean said, you should specify that each instance is using more than half the available hugepages on NUMA node 0.\n\n(You can run into the same problem with 4KB pages too, but if they\u0027re not preallocated you could run out of memory some time after the live migration instead of migration time.)","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":34,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"},{"line_number":35,"context_line":"unpredictable performance degradation."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"The second example is an instance with hugepages. Two identical instances are"},{"line_number":38,"context_line":"booted on two identical compute hosts. Both instances have NUMA topologies and"},{"line_number":39,"context_line":"hugepages and are on their respective host\u0027s NUMA node 0. If one of the"},{"line_number":40,"context_line":"instances is live-migrated to the other compute host, it will stay on NUMA node"},{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_51768a08","line":41,"range":{"start_line":37,"start_character":0,"end_line":41,"end_character":18},"updated":"2018-03-22 00:57:44.000000000","message":"as described this is missing detail.\nfor example we do not directly map pages to guest that is still done by the kernel/numactl so if there was enough hugepages on node0 on the second host this would not fail.\n\nwhat you need to say is more along the following lines.\n\ni have 2 host with 2 numa nodes each and 8G of hugepages per numa node. i boot 2 vms, each with 1 virtual numa node and 8G of hugepage memory. one vm is placed on each host useing all the hugepages on the first numa node(node0) on the host they land on. \n\nwhen instance 0 is migrated form host0 to host1 the numa topology filter determinis it should have enough hugepages memory on node1 of host1 to boot the vm. because the libvirt dirver enforces strict numa affinity for hugepange and the libvirt dirver does not re create the libvirt xml on migrate the migration fails as there are not enough resouces on node0 to fufill the request for vm0 as they are all allocated to vm1.\"\n\nyou can fix this in 3 ways. allow non strict numa affintiy, libvirt supports this in which case numactl will allocate teh hugepages from the remote numa node. this may reduce performace. alternitively regenerate the xml which is pobably the best option or last you can make the numa topology filter use the existing numa topology of the guest to more strictly filter hosts before migrating.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":34,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"},{"line_number":35,"context_line":"unpredictable performance degradation."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"The second example is an instance with hugepages. Two identical instances are"},{"line_number":38,"context_line":"booted on two identical compute hosts. Both instances have NUMA topologies and"},{"line_number":39,"context_line":"hugepages and are on their respective host\u0027s NUMA node 0. If one of the"},{"line_number":40,"context_line":"instances is live-migrated to the other compute host, it will stay on NUMA node"},{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_4337d0e4","line":41,"range":{"start_line":37,"start_character":0,"end_line":41,"end_character":18},"in_reply_to":"df7087c5_51768a08","updated":"2018-03-27 02:46:24.000000000","message":"Done","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":40,"context_line":"instances is live-migrated to the other compute host, it will stay on NUMA node"},{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Use Cases"},{"line_number":46,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_91550253","line":43,"range":{"start_line":43,"start_character":0,"end_line":43,"end_character":52},"updated":"2018-03-22 00:57:44.000000000","message":"there is also a third epa usecase which is migrating with pci devices \n\nagain we dont regenerate the xml, update the claims or reassert the numa affinity policy for the device on the dest node and just use the xml from the source node.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":40,"context_line":"instances is live-migrated to the other compute host, it will stay on NUMA node"},{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Use Cases"},{"line_number":46,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_419dc857","line":43,"range":{"start_line":43,"start_character":0,"end_line":43,"end_character":52},"in_reply_to":"df7087c5_91550253","updated":"2018-03-23 18:02:32.000000000","message":"Even if we did do those things, that\u0027s not sufficient for live-migration with PCI devices though, is it?  I mean wouldn\u0027t we need to either get guest cooperation or else hot-unplug the devices before the migration and then hot-plug them after?  (Since there is state inside the PCI device itself that we can\u0027t mirror over to the destination node.)","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":40,"context_line":"instances is live-migrated to the other compute host, it will stay on NUMA node"},{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Use Cases"},{"line_number":46,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_f206a986","line":43,"range":{"start_line":43,"start_character":0,"end_line":43,"end_character":52},"in_reply_to":"df7087c5_91550253","updated":"2018-03-27 02:46:24.000000000","message":"Let\u0027s limit this spec to the incorrectly named NUMA aware live migration.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Use Cases"},{"line_number":46,"context_line":"---------"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_c1723812","line":44,"updated":"2018-03-23 18:02:32.000000000","message":"There is a fourth problem (which happens now), which is simply that the NUMA node that the instance is currently running on may not actually *exist* on the destination compute node.  (Like it\u0027s running on host NUMA node 1 and the scheduler selects a destination with only a single NUMA node.)","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":41,"context_line":"0 and fail to run."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Both of these examples are known bugs `[1]`_ `[2]`_."},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Use Cases"},{"line_number":46,"context_line":"---------"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_e34aa453","line":44,"in_reply_to":"bf659307_c1723812","updated":"2018-03-27 02:46:24.000000000","message":"Ah, indeed. Added.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"As a cloud administrator, I want live migration of hugepage-backed instances to"},{"line_number":52,"context_line":"work and for the instances to successfully run on the destination compute host."},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Proposed change"},{"line_number":55,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":56,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_81cbe01c","line":53,"updated":"2018-03-23 18:02:32.000000000","message":"As a cloud administrator, I want to live migrate instances with explicit NUMA topology and for the instance to successfully run on the destination compute host.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"As a cloud administrator, I want live migration of hugepage-backed instances to"},{"line_number":52,"context_line":"work and for the instances to successfully run on the destination compute host."},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Proposed change"},{"line_number":55,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":56,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_439350a4","line":53,"in_reply_to":"bf659307_81cbe01c","updated":"2018-03-27 02:46:24.000000000","message":"Done","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"51ab84c8671f41c1ba6566b20a3b43af26dd5d44","unresolved":false,"context_lines":[{"line_number":54,"context_line":"Proposed change"},{"line_number":55,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Current, the scheduler does not claim any NUMA resources. While work has"},{"line_number":58,"context_line":"started to model NUMA topologies as resources providers in placement `[3]`_,"},{"line_number":59,"context_line":"this spec intentionally ignores that work and does not depend on it. Instead,"},{"line_number":60,"context_line":"the current method of claiming NUMA resources will continue to be used."}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_8c66b751","line":57,"range":{"start_line":57,"start_character":0,"end_line":57,"end_character":7},"updated":"2018-03-23 18:40:20.000000000","message":"Currently","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":54,"context_line":"Proposed change"},{"line_number":55,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Current, the scheduler does not claim any NUMA resources. While work has"},{"line_number":58,"context_line":"started to model NUMA topologies as resources providers in placement `[3]`_,"},{"line_number":59,"context_line":"this spec intentionally ignores that work and does not depend on it. Instead,"},{"line_number":60,"context_line":"the current method of claiming NUMA resources will continue to be used."}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_a3e90c1b","line":57,"range":{"start_line":57,"start_character":0,"end_line":57,"end_character":7},"in_reply_to":"bf659307_8c66b751","updated":"2018-03-27 02:46:24.000000000","message":"Done","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"51ab84c8671f41c1ba6566b20a3b43af26dd5d44","unresolved":false,"context_lines":[{"line_number":138,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":139,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"The interest of exhanging instance NUMA topologies early (in"},{"line_number":142,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) is to fail as"},{"line_number":143,"context_line":"fast as possible if the destination cannot fit the instance."},{"line_number":144,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_4ca3bf50","line":141,"range":{"start_line":141,"start_character":16,"end_line":141,"end_character":25},"updated":"2018-03-23 18:40:20.000000000","message":"\"The reason we exchange instance NUMA topologies early...\"","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":138,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":139,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"The interest of exhanging instance NUMA topologies early (in"},{"line_number":142,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) is to fail as"},{"line_number":143,"context_line":"fast as possible if the destination cannot fit the instance."},{"line_number":144,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_a3d26c5b","line":141,"range":{"start_line":141,"start_character":16,"end_line":141,"end_character":25},"in_reply_to":"bf659307_4ca3bf50","updated":"2018-03-27 02:46:24.000000000","message":"Ah, yeah, that\u0027s a Gallicism. Normally I do stuff the other way around - think in English and map direct to French. Fixed, though not exactly as you suggested, but I try to avoid the first person if I can help it :)","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":138,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":139,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"The interest of exhanging instance NUMA topologies early (in"},{"line_number":142,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) is to fail as"},{"line_number":143,"context_line":"fast as possible if the destination cannot fit the instance."},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Alternatives"},{"line_number":146,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_f1ec7e90","line":143,"range":{"start_line":141,"start_character":0,"end_line":143,"end_character":59},"updated":"2018-03-22 00:57:44.000000000","message":"we will have to do a similar check to the multiple port binding spec and only use this workflow if both nodes support this feature and use the legacy workflow otherwise else this wont work properly across upgrades/ computes of different releases","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":138,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":139,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"The interest of exhanging instance NUMA topologies early (in"},{"line_number":142,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) is to fail as"},{"line_number":143,"context_line":"fast as possible if the destination cannot fit the instance."},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Alternatives"},{"line_number":146,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_234bbc2b","line":143,"range":{"start_line":141,"start_character":0,"end_line":143,"end_character":59},"in_reply_to":"bf659307_61948ce9","updated":"2018-03-27 02:46:24.000000000","message":"In the Upgrade Impact the emerging consensus seems to be to use the new handshake if both computes support it, and just fail it either one is too old. Can I just link to that section here?","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":138,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":139,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"The interest of exhanging instance NUMA topologies early (in"},{"line_number":142,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) is to fail as"},{"line_number":143,"context_line":"fast as possible if the destination cannot fit the instance."},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Alternatives"},{"line_number":146,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_61948ce9","line":143,"range":{"start_line":141,"start_character":0,"end_line":143,"end_character":59},"in_reply_to":"df7087c5_f1ec7e90","updated":"2018-03-23 18:02:32.000000000","message":"++","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":149,"context_line":"the destination and the source, it would be conceivable to re-use the result of"},{"line_number":150,"context_line":"`numa_fit_instance_to_host`, as called from the scheduler before the live"},{"line_number":151,"context_line":"migration reaches the conductor. While this does technically save CPU cycles,"},{"line_number":152,"context_line":"there is currently no obvious mechanism for this sort of scheduler to conductor"},{"line_number":153,"context_line":"communication, unlike `migrate_data` between the source, destination and"},{"line_number":154,"context_line":"conductor. In addition, this would only work if a destination host is not"},{"line_number":155,"context_line":"specified in the API request."}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_91b18292","line":152,"range":{"start_line":152,"start_character":20,"end_line":152,"end_character":40},"updated":"2018-03-22 00:57:44.000000000","message":"am the schduler has the instance request spec object which is presumable returned back to the conductor at some point.\n\nin the numa topology filter could we not creat a ditionary of host to numa topology in the request spec for all valid hosts and then in the conductor retrive the topology for the selected host.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":149,"context_line":"the destination and the source, it would be conceivable to re-use the result of"},{"line_number":150,"context_line":"`numa_fit_instance_to_host`, as called from the scheduler before the live"},{"line_number":151,"context_line":"migration reaches the conductor. While this does technically save CPU cycles,"},{"line_number":152,"context_line":"there is currently no obvious mechanism for this sort of scheduler to conductor"},{"line_number":153,"context_line":"communication, unlike `migrate_data` between the source, destination and"},{"line_number":154,"context_line":"conductor. In addition, this would only work if a destination host is not"},{"line_number":155,"context_line":"specified in the API request."}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_a3dd8c15","line":152,"range":{"start_line":152,"start_character":20,"end_line":152,"end_character":40},"in_reply_to":"df7087c5_91b18292","updated":"2018-03-27 02:46:24.000000000","message":"So looking at this again, we sort of do this already with `limits` in the Selection object, but those limits are just CPU and RAM allocation ratios. We could add InstanceNUMATopology to Selection, and make the NUMA topology filter add it for every selection. This would get returned from the scheduler\u0027s select_destination to the conductor\u0027s live migrate task, which could add it to migrate_data, and the destination compute would still do the claim as previously described.\n\nIf that\u0027s all correct, it\u0027s a very viable alternative, I don\u0027t think I could decisively argue either for or against it, so I\u0027ll just throw it out there.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":151,"context_line":"migration reaches the conductor. While this does technically save CPU cycles,"},{"line_number":152,"context_line":"there is currently no obvious mechanism for this sort of scheduler to conductor"},{"line_number":153,"context_line":"communication, unlike `migrate_data` between the source, destination and"},{"line_number":154,"context_line":"conductor. In addition, this would only work if a destination host is not"},{"line_number":155,"context_line":"specified in the API request."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Data model impact"},{"line_number":158,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_a17f04aa","line":155,"range":{"start_line":154,"start_character":12,"end_line":155,"end_character":29},"updated":"2018-03-23 18:02:32.000000000","message":"I believe that even if the dest host is specified in the live migration request we will still go through the scheduler unless it is \"forced\" as well.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":151,"context_line":"migration reaches the conductor. While this does technically save CPU cycles,"},{"line_number":152,"context_line":"there is currently no obvious mechanism for this sort of scheduler to conductor"},{"line_number":153,"context_line":"communication, unlike `migrate_data` between the source, destination and"},{"line_number":154,"context_line":"conductor. In addition, this would only work if a destination host is not"},{"line_number":155,"context_line":"specified in the API request."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Data model impact"},{"line_number":158,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_a3058cc4","line":155,"range":{"start_line":154,"start_character":12,"end_line":155,"end_character":29},"in_reply_to":"bf659307_a17f04aa","updated":"2018-03-27 02:46:24.000000000","message":"Done.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"51ab84c8671f41c1ba6566b20a3b43af26dd5d44","unresolved":false,"context_lines":[{"line_number":151,"context_line":"migration reaches the conductor. While this does technically save CPU cycles,"},{"line_number":152,"context_line":"there is currently no obvious mechanism for this sort of scheduler to conductor"},{"line_number":153,"context_line":"communication, unlike `migrate_data` between the source, destination and"},{"line_number":154,"context_line":"conductor. In addition, this would only work if a destination host is not"},{"line_number":155,"context_line":"specified in the API request."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Data model impact"},{"line_number":158,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_ecb0938e","line":155,"range":{"start_line":154,"start_character":12,"end_line":155,"end_character":29},"in_reply_to":"bf659307_a17f04aa","updated":"2018-03-23 18:40:20.000000000","message":"that is correct, Chris.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":199,"context_line":""},{"line_number":200,"context_line":"None."},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"How a version mismatch between source and destination is handled depends on"},{"line_number":203,"context_line":"which of the two compute hosts is older."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"If the destination is older than the source, the source does not get an"},{"line_number":206,"context_line":"`InstanceNUMATopology` in `migrate_data` and can therefore choose to"},{"line_number":207,"context_line":"run an old-style live migration."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"If the source is older than the destination, the new field in `LiveMigrateData`"},{"line_number":210,"context_line":"is ignored and the source\u0027s old live migration runs without issues.  However,"},{"line_number":211,"context_line":"the destination has already claimed NUMA resources that the source does"},{"line_number":212,"context_line":"generate instance XML for. The destination could conceivably check the source\u0027s"},{"line_number":213,"context_line":"compute service version and fail the migration before claiming resources if the"},{"line_number":214,"context_line":"source doesn\u0027t support NUMA live migration. However, given the current broken"},{"line_number":215,"context_line":"state of NUMA live migration, a simpler solution is to refuse to perform a NUMA"},{"line_number":216,"context_line":"live migration unless the entire cloud has been upgraded to a version that"},{"line_number":217,"context_line":"supports it. This can be achieved by checking the cloud\u0027s minimum service"},{"line_number":218,"context_line":"version."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"Implementation"},{"line_number":221,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_111a527c","line":218,"range":{"start_line":202,"start_character":0,"end_line":218,"end_character":8},"updated":"2018-03-22 00:57:44.000000000","message":"we had the same converstation for the multiple neutron port binding spec. there we said just use the the new workflow if both nodes support it or the legacy workflow if they dont.\nthat makes the code and testing much simpler \n\nsee https://specs.openstack.org/openstack/nova-specs/specs/rocky/approved/neutron-new-port-binding-api.html#migration-across-mixed-software-versions","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":199,"context_line":""},{"line_number":200,"context_line":"None."},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"How a version mismatch between source and destination is handled depends on"},{"line_number":203,"context_line":"which of the two compute hosts is older."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"If the destination is older than the source, the source does not get an"},{"line_number":206,"context_line":"`InstanceNUMATopology` in `migrate_data` and can therefore choose to"},{"line_number":207,"context_line":"run an old-style live migration."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"If the source is older than the destination, the new field in `LiveMigrateData`"},{"line_number":210,"context_line":"is ignored and the source\u0027s old live migration runs without issues.  However,"},{"line_number":211,"context_line":"the destination has already claimed NUMA resources that the source does"},{"line_number":212,"context_line":"generate instance XML for. The destination could conceivably check the source\u0027s"},{"line_number":213,"context_line":"compute service version and fail the migration before claiming resources if the"},{"line_number":214,"context_line":"source doesn\u0027t support NUMA live migration. However, given the current broken"},{"line_number":215,"context_line":"state of NUMA live migration, a simpler solution is to refuse to perform a NUMA"},{"line_number":216,"context_line":"live migration unless the entire cloud has been upgraded to a version that"},{"line_number":217,"context_line":"supports it. This can be achieved by checking the cloud\u0027s minimum service"},{"line_number":218,"context_line":"version."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"Implementation"},{"line_number":221,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_c3bf60ce","line":218,"range":{"start_line":202,"start_character":0,"end_line":218,"end_character":8},"in_reply_to":"df7087c5_111a527c","updated":"2018-03-27 02:46:24.000000000","message":"I clarified that this section is essentially hypothetical, and we conclude that we\u0027re just not going to support NUMA live migration between version-mismatched computes.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"51ab84c8671f41c1ba6566b20a3b43af26dd5d44","unresolved":false,"context_lines":[{"line_number":224,"context_line":"-----------"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Primary assignee:"},{"line_number":227,"context_line":"  notartom"},{"line_number":228,"context_line":""},{"line_number":229,"context_line":"Work Items"},{"line_number":230,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_2c6c4b07","line":227,"range":{"start_line":227,"start_character":2,"end_line":227,"end_character":10},"updated":"2018-03-23 18:40:20.000000000","message":"heh","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":224,"context_line":"-----------"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Primary assignee:"},{"line_number":227,"context_line":"  notartom"},{"line_number":228,"context_line":""},{"line_number":229,"context_line":"Work Items"},{"line_number":230,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_43f4b0cf","line":227,"range":{"start_line":227,"start_character":2,"end_line":227,"end_character":10},"in_reply_to":"bf659307_2c6c4b07","updated":"2018-03-27 02:46:24.000000000","message":"It\u0027s serious :D https://launchpad.net/~notartom","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":242,"context_line":"Testing"},{"line_number":243,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"CPU pinning (without NUMA) can be used as a test case for NUMA live migration"},{"line_number":246,"context_line":"because CPU pinning does not require special hardware. Live migration is"},{"line_number":247,"context_line":"currently tested in the gate without CPU pinning. It would be possible to"},{"line_number":248,"context_line":"change the job\u0027s configuration to enable CPU pinning and add a live migration"},{"line_number":249,"context_line":"CPU pinning tempest test. However, the only assertable outcome would be the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_9412d057","line":246,"range":{"start_line":245,"start_character":0,"end_line":246,"end_character":54},"updated":"2018-03-22 00:57:44.000000000","message":"actully it requires nested virt because nova does not correct generate its xml when using qemu so the gate needs to run with kvm as the hypervirios for devstack vms.\n\nso this cant be tested in the upstream gate but could go possibly go into the intel nfv ci","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"51ab84c8671f41c1ba6566b20a3b43af26dd5d44","unresolved":false,"context_lines":[{"line_number":242,"context_line":"Testing"},{"line_number":243,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"CPU pinning (without NUMA) can be used as a test case for NUMA live migration"},{"line_number":246,"context_line":"because CPU pinning does not require special hardware. Live migration is"},{"line_number":247,"context_line":"currently tested in the gate without CPU pinning. It would be possible to"},{"line_number":248,"context_line":"change the job\u0027s configuration to enable CPU pinning and add a live migration"},{"line_number":249,"context_line":"CPU pinning tempest test. However, the only assertable outcome would be the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_ec75f3cd","line":246,"range":{"start_line":245,"start_character":0,"end_line":246,"end_character":54},"in_reply_to":"bf659307_012110b5","updated":"2018-03-23 18:40:20.000000000","message":"Chris is correct. Spec is here: https://review.openstack.org/#/c/533077/","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":242,"context_line":"Testing"},{"line_number":243,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"CPU pinning (without NUMA) can be used as a test case for NUMA live migration"},{"line_number":246,"context_line":"because CPU pinning does not require special hardware. Live migration is"},{"line_number":247,"context_line":"currently tested in the gate without CPU pinning. It would be possible to"},{"line_number":248,"context_line":"change the job\u0027s configuration to enable CPU pinning and add a live migration"},{"line_number":249,"context_line":"CPU pinning tempest test. However, the only assertable outcome would be the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_c398c01c","line":246,"range":{"start_line":245,"start_character":0,"end_line":246,"end_character":54},"in_reply_to":"bf659307_ec75f3cd","updated":"2018-03-27 02:46:24.000000000","message":"Done","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"77133c142d0551b9970936033d8f4ec8e9d5533a","unresolved":false,"context_lines":[{"line_number":242,"context_line":"Testing"},{"line_number":243,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"CPU pinning (without NUMA) can be used as a test case for NUMA live migration"},{"line_number":246,"context_line":"because CPU pinning does not require special hardware. Live migration is"},{"line_number":247,"context_line":"currently tested in the gate without CPU pinning. It would be possible to"},{"line_number":248,"context_line":"change the job\u0027s configuration to enable CPU pinning and add a live migration"},{"line_number":249,"context_line":"CPU pinning tempest test. However, the only assertable outcome would be the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_012110b5","line":246,"range":{"start_line":245,"start_character":0,"end_line":246,"end_character":54},"in_reply_to":"df7087c5_9412d057","updated":"2018-03-23 18:02:32.000000000","message":"I think there is currently a spec being reviewed to allow some of this to work without KVM.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":243,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"CPU pinning (without NUMA) can be used as a test case for NUMA live migration"},{"line_number":246,"context_line":"because CPU pinning does not require special hardware. Live migration is"},{"line_number":247,"context_line":"currently tested in the gate without CPU pinning. It would be possible to"},{"line_number":248,"context_line":"change the job\u0027s configuration to enable CPU pinning and add a live migration"},{"line_number":249,"context_line":"CPU pinning tempest test. However, the only assertable outcome would be the"},{"line_number":250,"context_line":"success of the live migration. It is explicitly out of tempest\u0027s scope to test"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_74305ce9","line":247,"range":{"start_line":246,"start_character":55,"end_line":247,"end_character":49},"updated":"2018-03-22 00:57:44.000000000","message":"the intel nfv ci multinode job is ment to test\ncpu pinning + numa topology + hugepages + ovs-dpdk/vhost user livemigration.\n\ni need to go check if the live migration test have been turned back on again.","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7da47d5abcea8634e73b90b0ca074f6c369f7336","unresolved":false,"context_lines":[{"line_number":247,"context_line":"currently tested in the gate without CPU pinning. It would be possible to"},{"line_number":248,"context_line":"change the job\u0027s configuration to enable CPU pinning and add a live migration"},{"line_number":249,"context_line":"CPU pinning tempest test. However, the only assertable outcome would be the"},{"line_number":250,"context_line":"success of the live migration. It is explicitly out of tempest\u0027s scope to test"},{"line_number":251,"context_line":"compute host internals, like CPU pin mappings in an instance\u0027s XML."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"Documentation Impact"},{"line_number":254,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"df7087c5_745e1c13","line":251,"range":{"start_line":250,"start_character":31,"end_line":251,"end_character":67},"updated":"2018-03-22 00:57:44.000000000","message":"we have a tempest plug to specificlaly test this.\n\nhttps://github.com/openstack/intel-nfv-ci-tests/blob/master/intel_nfv_ci_tests/tests/scenario/test_cpu_pinning.py\n\npatches are welcome :)","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"06c53c4e874d6e86c8dbef5d69ef156ec9fbd5e6","unresolved":false,"context_lines":[{"line_number":247,"context_line":"currently tested in the gate without CPU pinning. It would be possible to"},{"line_number":248,"context_line":"change the job\u0027s configuration to enable CPU pinning and add a live migration"},{"line_number":249,"context_line":"CPU pinning tempest test. However, the only assertable outcome would be the"},{"line_number":250,"context_line":"success of the live migration. It is explicitly out of tempest\u0027s scope to test"},{"line_number":251,"context_line":"compute host internals, like CPU pin mappings in an instance\u0027s XML."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"Documentation Impact"},{"line_number":254,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bf659307_23041c4a","line":251,"range":{"start_line":250,"start_character":31,"end_line":251,"end_character":67},"in_reply_to":"df7087c5_745e1c13","updated":"2018-03-27 02:46:24.000000000","message":"Done (and a shameless plug - we should probably consolidate those, at some point, maybe).","commit_id":"e0a8aa8a26cfa742c23f5106304af2b89c210e4d"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"4e169b63f3f44b4a9f12bb8f61190bb502087377","unresolved":false,"context_lines":[{"line_number":44,"context_line":"two NUMA nodes and 8GB of hugepage memory per node (for a total of 16GB). Two"},{"line_number":45,"context_line":"identical instances are booted on the two hosts. Their NUMA topology is one"},{"line_number":46,"context_line":"virtual NUMA node with 8GB of huegpage memory. They land on their respective"},{"line_number":47,"context_line":"host\u0027s NUMA node 0, filling up its memory. Ine instance is live-migrated to the"},{"line_number":48,"context_line":"other host. The libvirt driver enforces strict NUMA affinity and does not"},{"line_number":49,"context_line":"regenerate the instance XML. Both instances end up on the hosts NUMA node 0,"},{"line_number":50,"context_line":"and the live-migrated instance fails to run."}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_29af1de4","line":47,"range":{"start_line":47,"start_character":43,"end_line":47,"end_character":46},"updated":"2018-03-27 04:14:04.000000000","message":"One","commit_id":"c08f4a95c9ee573721652e1872265b0bcaaee029"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"17e74bd54c7c7df0161db17e5463a8a2f3e865c9","unresolved":false,"context_lines":[{"line_number":44,"context_line":"two NUMA nodes and 8GB of hugepage memory per node (for a total of 16GB). Two"},{"line_number":45,"context_line":"identical instances are booted on the two hosts. Their NUMA topology is one"},{"line_number":46,"context_line":"virtual NUMA node with 8GB of huegpage memory. They land on their respective"},{"line_number":47,"context_line":"host\u0027s NUMA node 0, filling up its memory. Ine instance is live-migrated to the"},{"line_number":48,"context_line":"other host. The libvirt driver enforces strict NUMA affinity and does not"},{"line_number":49,"context_line":"regenerate the instance XML. Both instances end up on the hosts NUMA node 0,"},{"line_number":50,"context_line":"and the live-migrated instance fails to run."}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_acaed095","line":47,"range":{"start_line":47,"start_character":43,"end_line":47,"end_character":46},"in_reply_to":"bf659307_29af1de4","updated":"2018-03-27 13:02:10.000000000","message":"Done","commit_id":"c08f4a95c9ee573721652e1872265b0bcaaee029"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"4e169b63f3f44b4a9f12bb8f61190bb502087377","unresolved":false,"context_lines":[{"line_number":49,"context_line":"regenerate the instance XML. Both instances end up on the hosts NUMA node 0,"},{"line_number":50,"context_line":"and the live-migrated instance fails to run."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"The this example is an instance with a virtual NUMA topology (but without"},{"line_number":53,"context_line":"hugepages). If an instance affined to its host\u0027s NUMA node 2 is live migrated"},{"line_number":54,"context_line":"to a host with only two NUMA nodes, and thus without a NUMA node 2, it will"},{"line_number":55,"context_line":"fail to run."}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_e9b8452b","line":52,"range":{"start_line":52,"start_character":4,"end_line":52,"end_character":8},"updated":"2018-03-27 04:14:04.000000000","message":"third","commit_id":"c08f4a95c9ee573721652e1872265b0bcaaee029"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"17e74bd54c7c7df0161db17e5463a8a2f3e865c9","unresolved":false,"context_lines":[{"line_number":49,"context_line":"regenerate the instance XML. Both instances end up on the hosts NUMA node 0,"},{"line_number":50,"context_line":"and the live-migrated instance fails to run."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"The this example is an instance with a virtual NUMA topology (but without"},{"line_number":53,"context_line":"hugepages). If an instance affined to its host\u0027s NUMA node 2 is live migrated"},{"line_number":54,"context_line":"to a host with only two NUMA nodes, and thus without a NUMA node 2, it will"},{"line_number":55,"context_line":"fail to run."}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_4c95b43f","line":52,"range":{"start_line":52,"start_character":4,"end_line":52,"end_character":8},"in_reply_to":"bf659307_e9b8452b","updated":"2018-03-27 13:02:10.000000000","message":"Done","commit_id":"c08f4a95c9ee573721652e1872265b0bcaaee029"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"4e169b63f3f44b4a9f12bb8f61190bb502087377","unresolved":false,"context_lines":[{"line_number":172,"context_line":"include `InstanceNUMATopology`. The NUMA topology filter could add an"},{"line_number":173,"context_line":"`InstanceNUMATopology` for every host that passes. That topology would"},{"line_number":174,"context_line":"eventually reach the conductor, which would put it in `migrate_data`. The"},{"line_number":175,"context_line":"destination compute host would then claim the resources as previously"},{"line_number":176,"context_line":"described."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"Data model impact"},{"line_number":179,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_29c63d9c","line":176,"range":{"start_line":175,"start_character":25,"end_line":176,"end_character":10},"updated":"2018-03-27 04:14:04.000000000","message":"I think it would have to *attempt* to claim the specified resources, then if runs into problems it could fall back to the existing code path.  But that\u0027s basically a performance optimization and not necessary in this spec.","commit_id":"c08f4a95c9ee573721652e1872265b0bcaaee029"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"17e74bd54c7c7df0161db17e5463a8a2f3e865c9","unresolved":false,"context_lines":[{"line_number":172,"context_line":"include `InstanceNUMATopology`. The NUMA topology filter could add an"},{"line_number":173,"context_line":"`InstanceNUMATopology` for every host that passes. That topology would"},{"line_number":174,"context_line":"eventually reach the conductor, which would put it in `migrate_data`. The"},{"line_number":175,"context_line":"destination compute host would then claim the resources as previously"},{"line_number":176,"context_line":"described."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"Data model impact"},{"line_number":179,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bf659307_ec8da831","line":176,"range":{"start_line":175,"start_character":25,"end_line":176,"end_character":10},"in_reply_to":"bf659307_29c63d9c","updated":"2018-03-27 13:02:10.000000000","message":"Well, it would attempt to claim, and if the claim fails, kick the migration back to the scheduler? Or do we fail it outright? I think the former... I\u0027ve added a blurb to that effect in the Proposed Change.","commit_id":"c08f4a95c9ee573721652e1872265b0bcaaee029"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3d544d5d377287240aac1a64019e337c2227ac1a","unresolved":false,"context_lines":[{"line_number":21,"context_line":"Problem description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"In the following paragrpahs the term NUMA is incorrectly used to signify any"},{"line_number":25,"context_line":"guest characteristic that is expressed in the `InstanceNUMATopology` object,"},{"line_number":26,"context_line":"for example CPU pinning and hugepages. CPU pinning can be achieved without a"},{"line_number":27,"context_line":"guest NUMA topology, but because no better term than NUMA is available it will"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_216de203","line":24,"range":{"start_line":24,"start_character":17,"end_line":24,"end_character":27},"updated":"2018-03-27 16:10:07.000000000","message":"paragraphs","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"804075a05cf797eb14cbb3012bf255e1260a077b","unresolved":false,"context_lines":[{"line_number":21,"context_line":"Problem description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"In the following paragrpahs the term NUMA is incorrectly used to signify any"},{"line_number":25,"context_line":"guest characteristic that is expressed in the `InstanceNUMATopology` object,"},{"line_number":26,"context_line":"for example CPU pinning and hugepages. CPU pinning can be achieved without a"},{"line_number":27,"context_line":"guest NUMA topology, but because no better term than NUMA is available it will"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_7c5ffc71","line":24,"range":{"start_line":24,"start_character":17,"end_line":24,"end_character":27},"in_reply_to":"bf659307_216de203","updated":"2018-03-27 17:36:08.000000000","message":"Done","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3d544d5d377287240aac1a64019e337c2227ac1a","unresolved":false,"context_lines":[{"line_number":34,"context_line":"are naively copied over to the destination host. This creates two problems."},{"line_number":35,"context_line":"First, its pinned pCPUs aren\u0027t properly claimed on the destination. This means"},{"line_number":36,"context_line":"that, should a second instance with pinned CPUs land on the destination, both"},{"line_number":37,"context_line":"instances\u0027s vCPUs could be pinned to the same pCPUs. Second, any existing pin"},{"line_number":38,"context_line":"mappings on the destination are ignored. If another instance already exists on"},{"line_number":39,"context_line":"the destination, both instances\u0027s vCPUs could be pinned to the same pCPUs. In"},{"line_number":40,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_814f0e91","line":37,"range":{"start_line":37,"start_character":0,"end_line":37,"end_character":11},"updated":"2018-03-27 16:10:07.000000000","message":"instances\u0027","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"804075a05cf797eb14cbb3012bf255e1260a077b","unresolved":false,"context_lines":[{"line_number":34,"context_line":"are naively copied over to the destination host. This creates two problems."},{"line_number":35,"context_line":"First, its pinned pCPUs aren\u0027t properly claimed on the destination. This means"},{"line_number":36,"context_line":"that, should a second instance with pinned CPUs land on the destination, both"},{"line_number":37,"context_line":"instances\u0027s vCPUs could be pinned to the same pCPUs. Second, any existing pin"},{"line_number":38,"context_line":"mappings on the destination are ignored. If another instance already exists on"},{"line_number":39,"context_line":"the destination, both instances\u0027s vCPUs could be pinned to the same pCPUs. In"},{"line_number":40,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_fc8d4cd3","line":37,"range":{"start_line":37,"start_character":0,"end_line":37,"end_character":11},"in_reply_to":"bf659307_814f0e91","updated":"2018-03-27 17:36:08.000000000","message":"Done","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3d544d5d377287240aac1a64019e337c2227ac1a","unresolved":false,"context_lines":[{"line_number":40,"context_line":"both cases, the ``dedicated`` CPU policy is violated, potentially leading to"},{"line_number":41,"context_line":"unpredictable performance degradation."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 8GB of hugepage memory per node (for a total of 16GB). Two"},{"line_number":45,"context_line":"identical instances are booted on the two hosts. Their NUMA topology is one"},{"line_number":46,"context_line":"virtual NUMA node with 8GB of huegpage memory. They land on their respective"},{"line_number":47,"context_line":"host\u0027s NUMA node 0, filling up its memory. One instance is live-migrated to the"},{"line_number":48,"context_line":"other host. The libvirt driver enforces strict NUMA affinity and does not"},{"line_number":49,"context_line":"regenerate the instance XML. Both instances end up on the hosts NUMA node 0,"},{"line_number":50,"context_line":"and the live-migrated instance fails to run."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"The third example is an instance with a virtual NUMA topology (but without"},{"line_number":53,"context_line":"hugepages). If an instance affined to its host\u0027s NUMA node 2 is live migrated"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_c129869b","line":50,"range":{"start_line":43,"start_character":0,"end_line":50,"end_character":44},"updated":"2018-03-27 16:10:07.000000000","message":"Well, I\u0027ll be damned. I didn\u0027t know this issue even existed.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"f2f3a191f7bae57db9e5d867c2b3318aa464e7b0","unresolved":false,"context_lines":[{"line_number":118,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_3b00b167","line":121,"range":{"start_line":121,"start_character":7,"end_line":121,"end_character":58},"updated":"2018-03-27 14:57:23.000000000","message":"like the existing, right?","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"2b52edc4a2c5b0f74507dcf884b7965dfcef5e25","unresolved":false,"context_lines":[{"line_number":118,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_46b58ccc","line":121,"range":{"start_line":121,"start_character":7,"end_line":121,"end_character":58},"in_reply_to":"bf659307_3b00b167","updated":"2018-03-27 15:22:51.000000000","message":"I don\u0027t think we do any old-style resource tracker claims during live migration, do we? So this would be new behaviour. Should we just keep it simple for now and fail the migration if the claim fails?","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3d544d5d377287240aac1a64019e337c2227ac1a","unresolved":false,"context_lines":[{"line_number":118,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_8c2bc580","line":121,"range":{"start_line":121,"start_character":7,"end_line":121,"end_character":58},"in_reply_to":"bf659307_46b58ccc","updated":"2018-03-27 16:10:07.000000000","message":"I can\u0027t speak for existing behavior, but keeping this whole things as simple as humanely possible would always be my preference","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"d579cf1de33d11b19957e7f4f550e48e0b47a132","unresolved":false,"context_lines":[{"line_number":118,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_8cbf45cf","line":121,"range":{"start_line":121,"start_character":7,"end_line":121,"end_character":58},"in_reply_to":"bf659307_46b58ccc","updated":"2018-03-27 16:18:22.000000000","message":"I think we should just fail the live migration if the claim fails. Since we\u0027re getting NUMA RPs soon-ish, we won\u0027t have to do resource tracker claims in the dest compute for much longer, so there\u0027s no point in optimising it to go back the scheduler if the claim fails.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"804075a05cf797eb14cbb3012bf255e1260a077b","unresolved":false,"context_lines":[{"line_number":118,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_9c63f01a","line":121,"range":{"start_line":121,"start_character":7,"end_line":121,"end_character":58},"in_reply_to":"bf659307_8cbf45cf","updated":"2018-03-27 17:36:08.000000000","message":"\u003e I think we should just fail the live migration if the claim fails.\n \u003e Since we\u0027re getting NUMA RPs soon-ish, we won\u0027t have to do resource\n \u003e tracker claims in the dest compute for much longer, so there\u0027s no\n \u003e point in optimising it to go back the scheduler if the claim fails.\n\nYeah, which is why in the version I pushed just as you were leaving your comment I proposed just failing the migration outright.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"d90519b3a5545ca04b2995dd228e5c3ca102aab2","unresolved":false,"context_lines":[{"line_number":118,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_dc4f28bb","line":121,"range":{"start_line":121,"start_character":7,"end_line":121,"end_character":58},"in_reply_to":"bf659307_8cbf45cf","updated":"2018-03-27 17:23:59.000000000","message":"I\u0027m not sure that NUMA RPs will cover everything that the claim does.  Until it does, the claim is still needed and could still fail.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"f2f3a191f7bae57db9e5d867c2b3318aa464e7b0","unresolved":false,"context_lines":[{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"},{"line_number":125,"context_line":"to be sent to the destination for the live migration. The proposed flow is"},{"line_number":126,"context_line":"summarised in the following diagram.::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_bbda8114","line":123,"range":{"start_line":122,"start_character":59,"end_line":123,"end_character":9},"updated":"2018-03-27 14:57:23.000000000","message":"from a cells v2 perspective, that could be a problem given the super-conductor can\u0027t be called from a cell (up-call, we say).\n\nIf you want to call a conductor, that would only be the cell conductor, so a separate method. \nStill fine ?","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"d579cf1de33d11b19957e7f4f550e48e0b47a132","unresolved":false,"context_lines":[{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"},{"line_number":125,"context_line":"to be sent to the destination for the live migration. The proposed flow is"},{"line_number":126,"context_line":"summarised in the following diagram.::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_0c233507","line":123,"range":{"start_line":122,"start_character":59,"end_line":123,"end_character":9},"in_reply_to":"bf659307_2646587e","updated":"2018-03-27 16:18:22.000000000","message":"Tried to clarify in the spec as well.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"2b52edc4a2c5b0f74507dcf884b7965dfcef5e25","unresolved":false,"context_lines":[{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"},{"line_number":125,"context_line":"to be sent to the destination for the live migration. The proposed flow is"},{"line_number":126,"context_line":"summarised in the following diagram.::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_e6f94080","line":123,"range":{"start_line":122,"start_character":59,"end_line":123,"end_character":9},"in_reply_to":"bf659307_bbda8114","updated":"2018-03-27 15:22:51.000000000","message":"You can\u0027t live migrate between cells, right? So it\u0027s the cell conductor that handles live migration, not the super conductor. Also, this is just adding data to the existing flow, there are no new method calls here. The conductor calls check_can_live_migrate_destination, and the dest compute returns migrate_data. We\u0027re just adding new stuff to migrate_data.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"507d0d32eb0b0b0b201e4de33e071c9c2c1a6d41","unresolved":false,"context_lines":[{"line_number":119,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":120,"context_line":"`NUMATopology` and claim the resources using the resource tracker. If the claim"},{"line_number":121,"context_line":"fails, the live migration is kicked back to the scheduler. The destination\u0027s"},{"line_number":122,"context_line":"virt driver will then send the updated `InstanceNUMATopoogy` back to the"},{"line_number":123,"context_line":"conductor, which will pass it on to the source. The source\u0027s libvirt driver"},{"line_number":124,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"},{"line_number":125,"context_line":"to be sent to the destination for the live migration. The proposed flow is"},{"line_number":126,"context_line":"summarised in the following diagram.::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_2646587e","line":123,"range":{"start_line":122,"start_character":59,"end_line":123,"end_character":9},"in_reply_to":"bf659307_e6f94080","updated":"2018-03-27 15:36:34.000000000","message":"thanks for clarifying","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3d544d5d377287240aac1a64019e337c2227ac1a","unresolved":false,"context_lines":[{"line_number":157,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":158,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"Exhanging instance NUMA topologies is done early (in"},{"line_number":161,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) in order to"},{"line_number":162,"context_line":"fail as fast as possible if the destination cannot fit the instance. What"},{"line_number":163,"context_line":"happens when the compute hosts are not both running the updated handshake code"},{"line_number":164,"context_line":"is discussed in ref:`upgrade-impact`."},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"Alternatives"},{"line_number":167,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_4c736db2","line":164,"range":{"start_line":160,"start_character":0,"end_line":164,"end_character":37},"updated":"2018-03-27 16:10:07.000000000","message":"Think I asked this before but aren\u0027t races a serious concern here? Assuming we can\u0027t prevent other instances from using the pCPUs we chose in the \"Fit InstanceNUMATopology to NUMATopology\" stage, is there a chance that those will be gone by time we try to use them?","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"804075a05cf797eb14cbb3012bf255e1260a077b","unresolved":false,"context_lines":[{"line_number":157,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":158,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"Exhanging instance NUMA topologies is done early (in"},{"line_number":161,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) in order to"},{"line_number":162,"context_line":"fail as fast as possible if the destination cannot fit the instance. What"},{"line_number":163,"context_line":"happens when the compute hosts are not both running the updated handshake code"},{"line_number":164,"context_line":"is discussed in ref:`upgrade-impact`."},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"Alternatives"},{"line_number":167,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_2f3ce8bd","line":164,"range":{"start_line":160,"start_character":0,"end_line":164,"end_character":37},"in_reply_to":"bf659307_4c736db2","updated":"2018-03-27 17:36:08.000000000","message":"Until we start using NUMA RPs in placement, where schedule + claim is essentially atomic, there will always be the potential for races, since we\u0027re scheduling first, and claiming later. I\u0027ve added something to that effect in the spec.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3d544d5d377287240aac1a64019e337c2227ac1a","unresolved":false,"context_lines":[{"line_number":166,"context_line":"Alternatives"},{"line_number":167,"context_line":"------------"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"It would be possible to reuse the result of `numa_fit_instance_to_host` as"},{"line_number":170,"context_line":"called from the scheduler before the live migration reaches the conductor."},{"line_number":171,"context_line":"`select_destinations` in the scheduler returns a list of `Selection` objects to"},{"line_number":172,"context_line":"the conductor\u0027s live migrate task. The `Selection` object could be modified to"},{"line_number":173,"context_line":"include `InstanceNUMATopology`. The NUMA topology filter could add an"},{"line_number":174,"context_line":"`InstanceNUMATopology` for every host that passes. That topology would"},{"line_number":175,"context_line":"eventually reach the conductor, which would put it in `migrate_data`. The"},{"line_number":176,"context_line":"destination compute host would then claim the resources as previously"},{"line_number":177,"context_line":"described."},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"Data model impact"},{"line_number":180,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_4cf60dfc","line":177,"range":{"start_line":169,"start_character":0,"end_line":177,"end_character":10},"updated":"2018-03-27 16:10:07.000000000","message":"I think I did something like this in a previous patch that ndipanov later reverted [1]. I\u0027ll check this out tomorrow morning but it\u0027s probably worth looking at that revert to see if it\u0027s related\n\n[1] 46e98cfeb8463c68f912052ace036b490774a89f","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"804075a05cf797eb14cbb3012bf255e1260a077b","unresolved":false,"context_lines":[{"line_number":166,"context_line":"Alternatives"},{"line_number":167,"context_line":"------------"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"It would be possible to reuse the result of `numa_fit_instance_to_host` as"},{"line_number":170,"context_line":"called from the scheduler before the live migration reaches the conductor."},{"line_number":171,"context_line":"`select_destinations` in the scheduler returns a list of `Selection` objects to"},{"line_number":172,"context_line":"the conductor\u0027s live migrate task. The `Selection` object could be modified to"},{"line_number":173,"context_line":"include `InstanceNUMATopology`. The NUMA topology filter could add an"},{"line_number":174,"context_line":"`InstanceNUMATopology` for every host that passes. That topology would"},{"line_number":175,"context_line":"eventually reach the conductor, which would put it in `migrate_data`. The"},{"line_number":176,"context_line":"destination compute host would then claim the resources as previously"},{"line_number":177,"context_line":"described."},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"Data model impact"},{"line_number":180,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_2faaa8d1","line":177,"range":{"start_line":169,"start_character":0,"end_line":177,"end_character":10},"in_reply_to":"bf659307_4cf60dfc","updated":"2018-03-27 17:36:08.000000000","message":"Leaving this unaddressed for now, need to read more.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"f2f3a191f7bae57db9e5d867c2b3318aa464e7b0","unresolved":false,"context_lines":[{"line_number":241,"context_line":"However, given the current broken state of NUMA live migration, a simpler"},{"line_number":242,"context_line":"solution is to refuse to perform a NUMA live migration unless the entire cloud"},{"line_number":243,"context_line":"has been upgraded to a version that supports it. This can be achieved by"},{"line_number":244,"context_line":"checking the cloud\u0027s minimum service version."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"Implementation"},{"line_number":247,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_db1c15c3","line":244,"range":{"start_line":244,"start_character":0,"end_line":244,"end_character":45},"updated":"2018-03-27 14:57:23.000000000","message":"where ? in the conductor before calling the source ?\n\nIf so, that would be a changed behaviour, right? Should we signal it using a microversion ? I dunno, thoughts ?","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"3d544d5d377287240aac1a64019e337c2227ac1a","unresolved":false,"context_lines":[{"line_number":241,"context_line":"However, given the current broken state of NUMA live migration, a simpler"},{"line_number":242,"context_line":"solution is to refuse to perform a NUMA live migration unless the entire cloud"},{"line_number":243,"context_line":"has been upgraded to a version that supports it. This can be achieved by"},{"line_number":244,"context_line":"checking the cloud\u0027s minimum service version."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"Implementation"},{"line_number":247,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_8c08a5d9","line":244,"range":{"start_line":244,"start_character":0,"end_line":244,"end_character":45},"in_reply_to":"bf659307_01377ee5","updated":"2018-03-27 16:10:07.000000000","message":"\u003e If so, that doesn\u0027t require a microversion\n\nThis is all RPC stuff anyway. We don\u0027t microversions there, right?","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d8df9b052cf43a84628870c2cd1c54d07cd0c9a5","unresolved":false,"context_lines":[{"line_number":241,"context_line":"However, given the current broken state of NUMA live migration, a simpler"},{"line_number":242,"context_line":"solution is to refuse to perform a NUMA live migration unless the entire cloud"},{"line_number":243,"context_line":"has been upgraded to a version that supports it. This can be achieved by"},{"line_number":244,"context_line":"checking the cloud\u0027s minimum service version."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"Implementation"},{"line_number":247,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_61ff5a01","line":244,"range":{"start_line":244,"start_character":0,"end_line":244,"end_character":45},"in_reply_to":"bf659307_0651d462","updated":"2018-03-27 15:33:23.000000000","message":"You just need to know if the source and dest computes can do the new thing, see:\n\nhttps://review.openstack.org/#/c/522537/13/nova/conductor/tasks/live_migrate.py\n\nIf the source can\u0027t, you\u0027re screwed. If the source can but no dest hosts can, then we exhaust the list of hosts and fail with NoValidHost.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"507d0d32eb0b0b0b201e4de33e071c9c2c1a6d41","unresolved":false,"context_lines":[{"line_number":241,"context_line":"However, given the current broken state of NUMA live migration, a simpler"},{"line_number":242,"context_line":"solution is to refuse to perform a NUMA live migration unless the entire cloud"},{"line_number":243,"context_line":"has been upgraded to a version that supports it. This can be achieved by"},{"line_number":244,"context_line":"checking the cloud\u0027s minimum service version."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"Implementation"},{"line_number":247,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_01377ee5","line":244,"range":{"start_line":244,"start_character":0,"end_line":244,"end_character":45},"in_reply_to":"bf659307_61ff5a01","updated":"2018-03-27 15:36:34.000000000","message":"If so, that doesn\u0027t require a microversion","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"804075a05cf797eb14cbb3012bf255e1260a077b","unresolved":false,"context_lines":[{"line_number":241,"context_line":"However, given the current broken state of NUMA live migration, a simpler"},{"line_number":242,"context_line":"solution is to refuse to perform a NUMA live migration unless the entire cloud"},{"line_number":243,"context_line":"has been upgraded to a version that supports it. This can be achieved by"},{"line_number":244,"context_line":"checking the cloud\u0027s minimum service version."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"Implementation"},{"line_number":247,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_cf63ecdb","line":244,"range":{"start_line":244,"start_character":0,"end_line":244,"end_character":45},"in_reply_to":"bf659307_8c08a5d9","updated":"2018-03-27 17:36:08.000000000","message":"Well, if an error makes its way to the API where there previously was no error, it could be a microversion, but I don\u0027t think we need one here, since previously we would just break.","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"2b52edc4a2c5b0f74507dcf884b7965dfcef5e25","unresolved":false,"context_lines":[{"line_number":241,"context_line":"However, given the current broken state of NUMA live migration, a simpler"},{"line_number":242,"context_line":"solution is to refuse to perform a NUMA live migration unless the entire cloud"},{"line_number":243,"context_line":"has been upgraded to a version that supports it. This can be achieved by"},{"line_number":244,"context_line":"checking the cloud\u0027s minimum service version."},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"Implementation"},{"line_number":247,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"bf659307_0651d462","line":244,"range":{"start_line":244,"start_character":0,"end_line":244,"end_character":45},"in_reply_to":"bf659307_db1c15c3","updated":"2018-03-27 15:22:51.000000000","message":"We could even do this in the API, I think, sort of like what we do for device tagging [1].\n\nAccording to [2], this wouldn\u0027t be a microversion, depending on how we interpret \"silently fail\" in the first step. Strictly speaking, a NUMA migration would \"work\", but we\u0027d have policy-violating CPU pinning, for example.\n\n[1] https://github.com/openstack/nova/blob/stable/queens/nova/api/openstack/compute/servers.py#L496\n[2] https://docs.openstack.org/nova/latest/_images/graphviz-e6b2dc4dee641e0f96697d8f2f83db213e43f198.png","commit_id":"d7ddc8d8311cbfb68458ea14e504cf69a2ee078c"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"8e945ca13af489c2eebfb4f0c32c82f12560e4b8","unresolved":false,"context_lines":[{"line_number":41,"context_line":"unpredictable performance degradation."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 8GB of hugepage memory per node (for a total of 16GB). Two"},{"line_number":45,"context_line":"identical instances are booted on the two hosts. Their NUMA topology is one"},{"line_number":46,"context_line":"virtual NUMA node with 8GB of huegpage memory. They land on their respective"},{"line_number":47,"context_line":"host\u0027s NUMA node 0, filling up its memory. One instance is live-migrated to the"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf659307_556ac9ed","line":44,"range":{"start_line":44,"start_character":19,"end_line":44,"end_character":51},"updated":"2018-03-29 17:38:17.000000000","message":"it\u0027s probably important to reword this. Hugepages are atomic things. It\u0027s not correct really to say you have 8GB of hugepage memory per node. Rather, you would have, say 8 1GB memory pages assigned to a node. Similarly, you might have 4096 2MB memory pages assigned to another node. Those are inventories of 8 and 4096 things, respectively.","commit_id":"d06bc6d5b9748f93c69ef6b6ba39f90e6ef11355"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"ef88202545c2f97275af54822a325f0bf1b7ca7e","unresolved":false,"context_lines":[{"line_number":41,"context_line":"unpredictable performance degradation."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 8GB of hugepage memory per node (for a total of 16GB). Two"},{"line_number":45,"context_line":"identical instances are booted on the two hosts. Their NUMA topology is one"},{"line_number":46,"context_line":"virtual NUMA node with 8GB of huegpage memory. They land on their respective"},{"line_number":47,"context_line":"host\u0027s NUMA node 0, filling up its memory. One instance is live-migrated to the"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf659307_90686f87","line":44,"range":{"start_line":44,"start_character":19,"end_line":44,"end_character":51},"in_reply_to":"bf659307_556ac9ed","updated":"2018-03-29 17:55:38.000000000","message":"Done","commit_id":"d06bc6d5b9748f93c69ef6b6ba39f90e6ef11355"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"8e945ca13af489c2eebfb4f0c32c82f12560e4b8","unresolved":false,"context_lines":[{"line_number":167,"context_line":""},{"line_number":168,"context_line":"Currently, only placement allocations are updated during a live migration. The"},{"line_number":169,"context_line":"proposed resource tracker claims mechanism will become obsolete once NUMA"},{"line_number":170,"context_line":"topology resources providers are implemented `[3]`_. Therefore, as a stopgap"},{"line_number":171,"context_line":"error handling method, the live migration can be failed if the resource claim"},{"line_number":172,"context_line":"does not succeed on the destination compute host. Once NUMA is handled by"},{"line_number":173,"context_line":"placement, the compute host will not have to do any resource claims."}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf659307_153e91d9","line":170,"range":{"start_line":170,"start_character":0,"end_line":170,"end_character":18},"updated":"2018-03-29 17:38:17.000000000","message":"s/topology resources/resource/","commit_id":"d06bc6d5b9748f93c69ef6b6ba39f90e6ef11355"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"ef88202545c2f97275af54822a325f0bf1b7ca7e","unresolved":false,"context_lines":[{"line_number":167,"context_line":""},{"line_number":168,"context_line":"Currently, only placement allocations are updated during a live migration. The"},{"line_number":169,"context_line":"proposed resource tracker claims mechanism will become obsolete once NUMA"},{"line_number":170,"context_line":"topology resources providers are implemented `[3]`_. Therefore, as a stopgap"},{"line_number":171,"context_line":"error handling method, the live migration can be failed if the resource claim"},{"line_number":172,"context_line":"does not succeed on the destination compute host. Once NUMA is handled by"},{"line_number":173,"context_line":"placement, the compute host will not have to do any resource claims."}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf659307_70e4bb3c","line":170,"range":{"start_line":170,"start_character":0,"end_line":170,"end_character":18},"in_reply_to":"bf659307_153e91d9","updated":"2018-03-29 17:55:38.000000000","message":"Done","commit_id":"d06bc6d5b9748f93c69ef6b6ba39f90e6ef11355"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4c7a27ec427bc303793896bdda017f8c421e7fb5","unresolved":false,"context_lines":[{"line_number":41,"context_line":"unpredictable performance degradation."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 4 2GB hugepages per node (for a total of 8GB of memory per"},{"line_number":45,"context_line":"node, or 16GB per host). Two identical instances are booted on the two hosts."},{"line_number":46,"context_line":"Their NUMA topology is one virtual NUMA node with 8GB of huegpage memory. They"},{"line_number":47,"context_line":"land on their respective host\u0027s NUMA node 0, filling up its memory. One"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_50dad74b","line":44,"range":{"start_line":44,"start_character":21,"end_line":44,"end_character":24},"updated":"2018-03-29 18:00:35.000000000","message":"While I\u0027m sure there are single memory pages greater than 1GB in size, it\u0027s extremely uncommon. Memory page sizes are typically 4K, 2M and 1G in size. So, please change the above to say 8 1G memory pages.","commit_id":"252ae49dea6cb6618e7a8c7a3687cf36f31d5ae7"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"05d5af17f756b43111b196385a6cedc4a2d9b223","unresolved":false,"context_lines":[{"line_number":41,"context_line":"unpredictable performance degradation."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 4 2GB hugepages per node (for a total of 8GB of memory per"},{"line_number":45,"context_line":"node, or 16GB per host). Two identical instances are booted on the two hosts."},{"line_number":46,"context_line":"Their NUMA topology is one virtual NUMA node with 8GB of huegpage memory. They"},{"line_number":47,"context_line":"land on their respective host\u0027s NUMA node 0, filling up its memory. One"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_30be43d5","line":44,"range":{"start_line":44,"start_character":21,"end_line":44,"end_character":24},"in_reply_to":"bf659307_50dad74b","updated":"2018-03-29 18:03:10.000000000","message":"Done","commit_id":"252ae49dea6cb6618e7a8c7a3687cf36f31d5ae7"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4c7a27ec427bc303793896bdda017f8c421e7fb5","unresolved":false,"context_lines":[{"line_number":41,"context_line":"unpredictable performance degradation."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 4 2GB hugepages per node (for a total of 8GB of memory per"},{"line_number":45,"context_line":"node, or 16GB per host). Two identical instances are booted on the two hosts."},{"line_number":46,"context_line":"Their NUMA topology is one virtual NUMA node with 8GB of huegpage memory. They"},{"line_number":47,"context_line":"land on their respective host\u0027s NUMA node 0, filling up its memory. One"},{"line_number":48,"context_line":"instance is live-migrated to the other host. The libvirt driver enforces strict"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_70191b20","line":45,"range":{"start_line":44,"start_character":44,"end_line":45,"end_character":23},"updated":"2018-03-29 18:00:35.000000000","message":"I would remove this part entirely. When talking about memory pages, it\u0027s incorrect to even think of things as \"8GB of memory per node\" or \"16GB per host\". That\u0027s not what is represented by a memory page. The memory page is a single, indivisible thing. It\u0027s unit is not in GB (or MB). It\u0027s unit is the page of memory itself. :)","commit_id":"252ae49dea6cb6618e7a8c7a3687cf36f31d5ae7"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"05d5af17f756b43111b196385a6cedc4a2d9b223","unresolved":false,"context_lines":[{"line_number":41,"context_line":"unpredictable performance degradation."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 4 2GB hugepages per node (for a total of 8GB of memory per"},{"line_number":45,"context_line":"node, or 16GB per host). Two identical instances are booted on the two hosts."},{"line_number":46,"context_line":"Their NUMA topology is one virtual NUMA node with 8GB of huegpage memory. They"},{"line_number":47,"context_line":"land on their respective host\u0027s NUMA node 0, filling up its memory. One"},{"line_number":48,"context_line":"instance is live-migrated to the other host. The libvirt driver enforces strict"}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_50c3b75d","line":45,"range":{"start_line":44,"start_character":44,"end_line":45,"end_character":23},"in_reply_to":"bf659307_70191b20","updated":"2018-03-29 18:03:10.000000000","message":"Done","commit_id":"252ae49dea6cb6618e7a8c7a3687cf36f31d5ae7"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4c7a27ec427bc303793896bdda017f8c421e7fb5","unresolved":false,"context_lines":[{"line_number":44,"context_line":"two NUMA nodes and 4 2GB hugepages per node (for a total of 8GB of memory per"},{"line_number":45,"context_line":"node, or 16GB per host). Two identical instances are booted on the two hosts."},{"line_number":46,"context_line":"Their NUMA topology is one virtual NUMA node with 8GB of huegpage memory. They"},{"line_number":47,"context_line":"land on their respective host\u0027s NUMA node 0, filling up its memory. One"},{"line_number":48,"context_line":"instance is live-migrated to the other host. The libvirt driver enforces strict"},{"line_number":49,"context_line":"NUMA affinity and does not regenerate the instance XML. Both instances end up"},{"line_number":50,"context_line":"on the hosts NUMA node 0, and the live-migrated instance fails to run."}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_f0da0b49","line":47,"range":{"start_line":47,"start_character":45,"end_line":47,"end_character":66},"updated":"2018-03-29 18:00:35.000000000","message":"s/filling up its memory/consuming a set amount of memory pages of a particular size/","commit_id":"252ae49dea6cb6618e7a8c7a3687cf36f31d5ae7"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"05d5af17f756b43111b196385a6cedc4a2d9b223","unresolved":false,"context_lines":[{"line_number":44,"context_line":"two NUMA nodes and 4 2GB hugepages per node (for a total of 8GB of memory per"},{"line_number":45,"context_line":"node, or 16GB per host). Two identical instances are booted on the two hosts."},{"line_number":46,"context_line":"Their NUMA topology is one virtual NUMA node with 8GB of huegpage memory. They"},{"line_number":47,"context_line":"land on their respective host\u0027s NUMA node 0, filling up its memory. One"},{"line_number":48,"context_line":"instance is live-migrated to the other host. The libvirt driver enforces strict"},{"line_number":49,"context_line":"NUMA affinity and does not regenerate the instance XML. Both instances end up"},{"line_number":50,"context_line":"on the hosts NUMA node 0, and the live-migrated instance fails to run."}],"source_content_type":"text/x-rst","patch_set":10,"id":"bf659307_f0b3eb08","line":47,"range":{"start_line":47,"start_character":45,"end_line":47,"end_character":66},"in_reply_to":"bf659307_f0da0b49","updated":"2018-03-29 18:03:10.000000000","message":"Done","commit_id":"252ae49dea6cb6618e7a8c7a3687cf36f31d5ae7"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"14dc7101282f3f7c8110413d24623c05d2eba446","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 8 1GB hugepages per node. Two identical instances are booted"},{"line_number":45,"context_line":"on the two hosts. Their NUMA topology is one virtual NUMA node with 8GB of"},{"line_number":46,"context_line":"huegpage memory. They land on their respective host\u0027s NUMA node 0, consuming"},{"line_number":47,"context_line":"all 8 of its pages. One instance is live-migrated to the other host. The"},{"line_number":48,"context_line":"libvirt driver enforces strict NUMA affinity and does not regenerate the"},{"line_number":49,"context_line":"instance XML. Both instances end up on the hosts NUMA node 0, and the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bf659307_f059ab8d","line":46,"range":{"start_line":45,"start_character":68,"end_line":46,"end_character":15},"updated":"2018-03-29 18:17:12.000000000","message":"heh :) this should be \"Their virtual NUMA topology is for one virtual NUMA node and 8 1GB memory pages.\"","commit_id":"2b1122b8f115e84cbafe1666759f5a3c37451367"},{"author":{"_account_id":8864,"name":"Artom Lifshitz","email":"notartom@gmail.com","username":"artom"},"change_message_id":"527b12fee1f0a428ca23c56444830596e040f1cc","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"The second example is instances with hugepages. There are two hosts, each with"},{"line_number":44,"context_line":"two NUMA nodes and 8 1GB hugepages per node. Two identical instances are booted"},{"line_number":45,"context_line":"on the two hosts. Their NUMA topology is one virtual NUMA node with 8GB of"},{"line_number":46,"context_line":"huegpage memory. They land on their respective host\u0027s NUMA node 0, consuming"},{"line_number":47,"context_line":"all 8 of its pages. One instance is live-migrated to the other host. The"},{"line_number":48,"context_line":"libvirt driver enforces strict NUMA affinity and does not regenerate the"},{"line_number":49,"context_line":"instance XML. Both instances end up on the hosts NUMA node 0, and the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bf659307_309723ef","line":46,"range":{"start_line":45,"start_character":68,"end_line":46,"end_character":15},"in_reply_to":"bf659307_f059ab8d","updated":"2018-03-29 18:19:16.000000000","message":"Done","commit_id":"2b1122b8f115e84cbafe1666759f5a3c37451367"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":25,"context_line":"guest characteristic that is expressed in the `InstanceNUMATopology` object,"},{"line_number":26,"context_line":"for example CPU pinning and hugepages. CPU pinning can be achieved without a"},{"line_number":27,"context_line":"guest NUMA topology, but because no better term than NUMA is available it will"},{"line_number":28,"context_line":"contine to be used."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"The problem can best be described with three examples."},{"line_number":31,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_33760d40","line":28,"range":{"start_line":28,"start_character":0,"end_line":28,"end_character":7},"updated":"2018-04-03 15:04:11.000000000","message":"continue *cough cough*","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":66,"context_line":"work and for the instances to successfully run on the destination compute host."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"As a cloud administrator, I want live migration of instances with an explicit"},{"line_number":69,"context_line":"NUMA topology to work and for the instaces to successfully run on the"},{"line_number":70,"context_line":"destination compute host."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_531b81c8","line":69,"range":{"start_line":69,"start_character":34,"end_line":69,"end_character":42},"updated":"2018-04-03 15:04:11.000000000","message":"instances *cough cough*","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":79,"context_line":"Specifically, NUMA resources will continue to be claimed by the compute host\u0027s"},{"line_number":80,"context_line":"resource tracker."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"At the cell conductor (live migration isn\u0027t supported between cells, so the"},{"line_number":83,"context_line":"superconductor is not involved) and compute level, the relevant parts of the"},{"line_number":84,"context_line":"current live migration flow can be summarized by the following oversimplified"},{"line_number":85,"context_line":"pseudo sequence diagram.::"},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_b381bdc5","line":83,"range":{"start_line":82,"start_character":22,"end_line":83,"end_character":31},"updated":"2018-04-03 15:04:11.000000000","message":"thanks for that addition","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":118,"context_line":"instance\u0027s existing NUMA topology in the `migrate_data` that its"},{"line_number":119,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":120,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":121,"context_line":"`NUMATopology` and claim the resources using the resource tracker. It will then"},{"line_number":122,"context_line":"send the updated `InstanceNUMATopoogy` back to the conductor as part of the"},{"line_number":123,"context_line":"existing `migrate_data` that `check_can_live_migrate_destination` returns. The"},{"line_number":124,"context_line":"updated `InstanceNUMATopology` will continue to be propagated as part of"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_d315514a","line":121,"range":{"start_line":121,"start_character":19,"end_line":121,"end_character":38},"updated":"2018-04-03 15:04:11.000000000","message":"technically, that will be the ResourceTracker which will claim those resources after the virt driver tells to the compute manager \"yup\".","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":119,"context_line":"`check_can_live_migrate_source` returns to the destination. The destination\u0027s"},{"line_number":120,"context_line":"virt driver will fit this `InstanceNUMATopology` to the destination\u0027s"},{"line_number":121,"context_line":"`NUMATopology` and claim the resources using the resource tracker. It will then"},{"line_number":122,"context_line":"send the updated `InstanceNUMATopoogy` back to the conductor as part of the"},{"line_number":123,"context_line":"existing `migrate_data` that `check_can_live_migrate_destination` returns. The"},{"line_number":124,"context_line":"updated `InstanceNUMATopology` will continue to be propagated as part of"},{"line_number":125,"context_line":"`migrate_data`, eventually reaching the source. The source\u0027s libvirt driver"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_b339bdf2","line":122,"range":{"start_line":122,"start_character":18,"end_line":122,"end_character":37},"updated":"2018-04-03 15:04:11.000000000","message":"InstanceNUMATopology *cough cough*","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":123,"context_line":"existing `migrate_data` that `check_can_live_migrate_destination` returns. The"},{"line_number":124,"context_line":"updated `InstanceNUMATopology` will continue to be propagated as part of"},{"line_number":125,"context_line":"`migrate_data`, eventually reaching the source. The source\u0027s libvirt driver"},{"line_number":126,"context_line":"will use this updated `InstanceNUMATopology` when generating the instance XML"},{"line_number":127,"context_line":"to be sent to the destination for the live migration. The proposed flow is"},{"line_number":128,"context_line":"summarised in the following diagram.::"},{"line_number":129,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_d27b0889","line":126,"range":{"start_line":126,"start_character":45,"end_line":126,"end_character":77},"updated":"2018-04-03 15:04:11.000000000","message":"given the destination node will claim the resources, the instance XML created by the source node shouldn\u0027t be raced.","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":159,"context_line":"          |                                                              |                                            |            |-----------------------------------| |"},{"line_number":160,"context_line":"          |                                                              |                                            |                                                  |"},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"Exhanging instance NUMA topologies is done early (in"},{"line_number":163,"context_line":"`check_can_live_migrate_source` rather than `pre_live_migration`) in order to"},{"line_number":164,"context_line":"fail as fast as possible if the destination cannot fit the instance. What"},{"line_number":165,"context_line":"happens when the compute hosts are not both running the updated handshake code"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_f2616cd4","line":162,"range":{"start_line":162,"start_character":0,"end_line":162,"end_character":9},"updated":"2018-04-03 15:04:11.000000000","message":"Exchanging *cough cough*","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"73cf9130e4ae745e8bacf8893a88e0eacd6f9b92","unresolved":false,"context_lines":[{"line_number":176,"context_line":"live migrated instance before the latter\u0027s destination compute host has a"},{"line_number":177,"context_line":"chance to claim them. Until NUMA resource providers are implemented `[3]`_ and"},{"line_number":178,"context_line":"allow for an essentially atomic schedule+claim operation, scheduling and"},{"line_number":179,"context_line":"claiming will keep being done at different times on different nodes. Thus, the"},{"line_number":180,"context_line":"potential for races will continue to exist."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"Alternatives"},{"line_number":183,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bf659307_92def03b","line":180,"range":{"start_line":179,"start_character":69,"end_line":180,"end_character":43},"updated":"2018-04-03 15:04:11.000000000","message":"sure, but once the claim is done, that shouldn\u0027t be impossible.","commit_id":"1eb0ae86a5bcaf83809f473e6858d5426adc7966"}]}