)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"08017b111a441c6f4bc6a9e4f8ef270912b774a9","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     songwenping \u003csongwenping@inspur.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2020-09-07 17:46:42 +0800"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Support vGPU in nova and cyborg interaction"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Now we have supported pGPU in nova and cyborg interaction[1], and we"},{"line_number":10,"context_line":"also support attaching virtual GPU devices to guests[2]."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"9f560f44_5f69f6bb","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":43},"updated":"2020-09-07 10:16:22.000000000","message":"Support vGPU management by Cyborg","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"7c92d731c5e77e35bbc5498b37e3b099cfbe0312","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     songwenping \u003csongwenping@inspur.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2020-09-07 17:46:42 +0800"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Support vGPU in nova and cyborg interaction"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Now we have supported pGPU in nova and cyborg interaction[1], and we"},{"line_number":10,"context_line":"also support attaching virtual GPU devices to guests[2]."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"9f560f44_8827d1c7","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":43},"in_reply_to":"9f560f44_5f69f6bb","updated":"2020-09-08 00:55:26.000000000","message":"Done","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"08017b111a441c6f4bc6a9e4f8ef270912b774a9","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"[1] https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html"},{"line_number":20,"context_line":"[2] https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Change-Id: Icaf5556417741a27c83c1e0fe5b5fc39fbac48b6"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"9f560f44_bf513211","line":21,"updated":"2020-09-07 10:16:22.000000000","message":"Part of blueprint cyborg-vgpu-support","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"7c92d731c5e77e35bbc5498b37e3b099cfbe0312","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"[1] https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html"},{"line_number":20,"context_line":"[2] https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Change-Id: Icaf5556417741a27c83c1e0fe5b5fc39fbac48b6"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"9f560f44_c81ee96e","line":21,"in_reply_to":"9f560f44_bf513211","updated":"2020-09-08 00:55:26.000000000","message":"Done","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"As the long-term goal, OpenStack Cyborg aims to provide a standard"},{"line_number":10,"context_line":"management framework for acceleration resources. GPU is one of the"},{"line_number":11,"context_line":"important accelerators that cyborg needs to enable if cyborg wanna be that"},{"line_number":12,"context_line":"kind of framework."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":15,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"1f621f24_fbd351ff","line":12,"range":{"start_line":11,"start_character":51,"end_line":12,"end_character":18},"updated":"2020-11-17 11:40:17.000000000","message":"nit: you can remove this.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"As the long-term goal, OpenStack Cyborg aims to provide a standard"},{"line_number":10,"context_line":"management framework for acceleration resources. GPU is one of the"},{"line_number":11,"context_line":"important accelerators that cyborg needs to enable if cyborg wanna be that"},{"line_number":12,"context_line":"kind of framework."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":15,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"526a9799_75f4fb58","line":12,"range":{"start_line":11,"start_character":51,"end_line":12,"end_character":18},"in_reply_to":"1f621f24_fbd351ff","updated":"2020-11-23 06:15:12.000000000","message":"Done","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":11,"context_line":"important accelerators that cyborg needs to enable if cyborg wanna be that"},{"line_number":12,"context_line":"kind of framework."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":15,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":16,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":17,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Co-Authored-By: YumengBao \u003cyumeng_bao@yahoo.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"1f621f24_1bf28d9a","line":16,"range":{"start_line":14,"start_character":11,"end_line":16,"end_character":27},"updated":"2020-11-17 11:40:17.000000000","message":"no, that\u0027s not true, the nova-libvirt driver already supports creating mdevs and attaching them to an instance domain, and that\u0027s from Queens.\n\n\nWhat you want to tell is that Cyborg doesn\u0027t support it, that\u0027s it.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":11,"context_line":"important accelerators that cyborg needs to enable if cyborg wanna be that"},{"line_number":12,"context_line":"kind of framework."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":15,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":16,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":17,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Co-Authored-By: YumengBao \u003cyumeng_bao@yahoo.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"56ea3510_cde6eaf7","line":16,"range":{"start_line":14,"start_character":11,"end_line":16,"end_character":27},"in_reply_to":"1f621f24_1bf28d9a","updated":"2020-11-23 06:15:12.000000000","message":"Oh, sorry about saying that and confusing you. That\u0027s really bad practice. But I\u0027m sure I didn\u0027t mean to say that. What I want to point out is exactly that cyborg doesn\u0027t support it, but part of the changes will land in nova. Please see my update in the new message and problem description part. :) Hope you guys can forgive my inappropriate remarks.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"932372c8345cedf9894f980ec77bc881fbf2191e","unresolved":true,"context_lines":[{"line_number":10,"context_line":"to an instance."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in allocating a cyborg owned vGPU to an instance. This spec proposes to support composing cyborg owned vGPU into domain XML in"},{"line_number":13,"context_line":"nova libvirt driver."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Co-Authored-By: YumengBao \u003cyumeng_bao@yahoo.com\u003e"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"6283179a_8ae58772","line":13,"updated":"2020-11-27 12:18:22.000000000","message":"please wrap long lines in the commit message.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":false,"context_lines":[{"line_number":10,"context_line":"to an instance."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in allocating a cyborg owned vGPU to an instance. This spec proposes to support composing cyborg owned vGPU into domain XML in"},{"line_number":13,"context_line":"nova libvirt driver."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Co-Authored-By: YumengBao \u003cyumeng_bao@yahoo.com\u003e"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"50a3b822_4aa0f15f","line":13,"in_reply_to":"6283179a_8ae58772","updated":"2020-12-07 09:34:16.000000000","message":"Done","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"}],"specs/wallaby/approved/support-vGPU-nova-cyborg-interaction.rst":[{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"08017b111a441c6f4bc6a9e4f8ef270912b774a9","unresolved":false,"context_lines":[{"line_number":140,"context_line":"References"},{"line_number":141,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":".. [1] https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html"},{"line_number":144,"context_line":".. [2] https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"History"},{"line_number":147,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f560f44_1fa13ef8","line":144,"range":{"start_line":143,"start_character":0,"end_line":144,"end_character":68},"updated":"2020-09-07 10:16:22.000000000","message":".. _[1]:https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html\n.. _[2]:https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html\n\nYou can use above instead, that why the doc test task failed.","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"7c92d731c5e77e35bbc5498b37e3b099cfbe0312","unresolved":false,"context_lines":[{"line_number":140,"context_line":"References"},{"line_number":141,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":".. [1] https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html"},{"line_number":144,"context_line":".. [2] https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"History"},{"line_number":147,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f560f44_083c61df","line":144,"range":{"start_line":143,"start_character":0,"end_line":144,"end_character":68},"in_reply_to":"9f560f44_1fa13ef8","updated":"2020-09-08 00:55:26.000000000","message":"Done","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"08017b111a441c6f4bc6a9e4f8ef270912b774a9","unresolved":false,"context_lines":[{"line_number":151,"context_line":""},{"line_number":152,"context_line":"   * - Release Name"},{"line_number":153,"context_line":"     - Description"},{"line_number":154,"context_line":"   * - wallaby"},{"line_number":155,"context_line":"     - Introduced"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f560f44_9f18ae76","line":154,"range":{"start_line":154,"start_character":7,"end_line":154,"end_character":8},"updated":"2020-09-07 10:16:22.000000000","message":"Wallaby","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"7c92d731c5e77e35bbc5498b37e3b099cfbe0312","unresolved":false,"context_lines":[{"line_number":151,"context_line":""},{"line_number":152,"context_line":"   * - Release Name"},{"line_number":153,"context_line":"     - Description"},{"line_number":154,"context_line":"   * - wallaby"},{"line_number":155,"context_line":"     - Introduced"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f560f44_482139b4","line":154,"range":{"start_line":154,"start_character":7,"end_line":154,"end_character":8},"in_reply_to":"9f560f44_9f18ae76","updated":"2020-09-08 00:55:26.000000000","message":"Done","commit_id":"2ad3f8e346e111dc9edf0f7da5f5d36e9755e7ae"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":16,"context_line":"Problem description"},{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Currently we have supported pGPU in nova and cyborg interaction[1]_, and we"},{"line_number":20,"context_line":"also support attaching virtual GPU devices to guests[2]_. The operator have"},{"line_number":21,"context_line":"to find vGPU types and set vGPU types in nova-cpu.conf before attaching"},{"line_number":22,"context_line":"virtual GPU devices to guests, which is not conveniently and complex."},{"line_number":23,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_0651ff0d","line":20,"range":{"start_line":19,"start_character":73,"end_line":20,"end_character":4},"updated":"2020-10-07 16:48:30.000000000","message":"afaik, only nova supports this.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Currently we have supported pGPU in nova and cyborg interaction[1]_, and we"},{"line_number":20,"context_line":"also support attaching virtual GPU devices to guests[2]_. The operator have"},{"line_number":21,"context_line":"to find vGPU types and set vGPU types in nova-cpu.conf before attaching"},{"line_number":22,"context_line":"virtual GPU devices to guests, which is not conveniently and complex."},{"line_number":23,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_e66e8bc6","line":20,"range":{"start_line":20,"start_character":12,"end_line":20,"end_character":35},"updated":"2020-10-07 16:48:30.000000000","message":"technically, we don\u0027t attach the vGPUs, we ask the kernel to create the mdevs on the fly (or we reuse them) only when creating or rebuilding an instance. In other words, we don\u0027t have a specific API for attaching/detaching vGPUs with existing live instances.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":19,"context_line":"Currently we have supported pGPU in nova and cyborg interaction[1]_, and we"},{"line_number":20,"context_line":"also support attaching virtual GPU devices to guests[2]_. The operator have"},{"line_number":21,"context_line":"to find vGPU types and set vGPU types in nova-cpu.conf before attaching"},{"line_number":22,"context_line":"virtual GPU devices to guests, which is not conveniently and complex."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_665a7b1e","line":22,"range":{"start_line":22,"start_character":31,"end_line":22,"end_character":69},"updated":"2020-10-07 16:48:30.000000000","message":"this is somehow documented by https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"5dfab90f2cf4d6ce6911bdaf0ebfa7f1726486ec","unresolved":false,"context_lines":[{"line_number":19,"context_line":"Currently we have supported pGPU in nova and cyborg interaction[1]_, and we"},{"line_number":20,"context_line":"also support attaching virtual GPU devices to guests[2]_. The operator have"},{"line_number":21,"context_line":"to find vGPU types and set vGPU types in nova-cpu.conf before attaching"},{"line_number":22,"context_line":"virtual GPU devices to guests, which is not conveniently and complex."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_a594698d","line":22,"range":{"start_line":22,"start_character":31,"end_line":22,"end_character":69},"in_reply_to":"9f560f44_665a7b1e","updated":"2020-10-15 12:27:45.000000000","message":"Yeah, this has put in the reference at line144","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":20,"context_line":"also support attaching virtual GPU devices to guests[2]_. The operator have"},{"line_number":21,"context_line":"to find vGPU types and set vGPU types in nova-cpu.conf before attaching"},{"line_number":22,"context_line":"virtual GPU devices to guests, which is not conveniently and complex."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"},{"line_number":26,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_66af1be0","line":23,"updated":"2020-10-07 16:48:30.000000000","message":"which problem are you then trying to solve ? Configuration complexity, I guess ?","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"c6028c50080563d8b1ea26c04e80371a11776d96","unresolved":false,"context_lines":[{"line_number":20,"context_line":"also support attaching virtual GPU devices to guests[2]_. The operator have"},{"line_number":21,"context_line":"to find vGPU types and set vGPU types in nova-cpu.conf before attaching"},{"line_number":22,"context_line":"virtual GPU devices to guests, which is not conveniently and complex."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"},{"line_number":26,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f65232a_77aaaa85","line":23,"in_reply_to":"9f560f44_0586f565","updated":"2020-10-21 05:03:08.000000000","message":"@Wenping, the problem is not about configuration. The configuration is fine, Although we can further improve it, that\u0027s not the point. The point is as the long-term goal, the cyborg project aims to provide a general-purpose management framework for acceleration resources and GPU is one of the important accelerators that cyborg needs to support if cyborg wanna be that kind of framework. In vGPU case, Cyborg provides management for vGPU(discover, report, binding, etc), nova does the GPU resource scheduler and we also need to integrate with nova to do the vGPU allocation in virt driver level (eg. writing a vGPU managed by cyborg into an instance\u0027s XML). IMO, virt driver change should be the main point we need to mention in this spec.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"5dfab90f2cf4d6ce6911bdaf0ebfa7f1726486ec","unresolved":false,"context_lines":[{"line_number":20,"context_line":"also support attaching virtual GPU devices to guests[2]_. The operator have"},{"line_number":21,"context_line":"to find vGPU types and set vGPU types in nova-cpu.conf before attaching"},{"line_number":22,"context_line":"virtual GPU devices to guests, which is not conveniently and complex."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"},{"line_number":26,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_0586f565","line":23,"in_reply_to":"9f560f44_66af1be0","updated":"2020-10-15 12:27:45.000000000","message":"Now the vgpu function is not easy to use.\n\nAs you said, a reason is configuration complexity,\n\nand another is management the vgpu by Cyborg, that we can mange the GPU and vGPU, now if we configurature the vGPU in nvoa-compute.conf, that we cannt support operations of suspend/cold migration/resume etc.[1]. Now we are trying to support more server operations[2] if the server boot with accelerators.\n\n[1]https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html#caveats\n[2]https://docs.openstack.org/api-guide/compute/accelerator-support.html","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"User wants to use vGPU in new way by cyborg. This is extremely useful for"},{"line_number":28,"context_line":"clouds where operators needn\u0027t find vGPU types and set them in"},{"line_number":29,"context_line":"nova-cpu.conf."},{"line_number":30,"context_line":"Like pGPU passthrough to guests by cyborg component, cyborg can also manage"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_e6f16b02","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":44},"updated":"2020-10-07 16:48:30.000000000","message":"this is not a use case, this is a technical solution.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"c6028c50080563d8b1ea26c04e80371a11776d96","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"User wants to use vGPU in new way by cyborg. This is extremely useful for"},{"line_number":28,"context_line":"clouds where operators needn\u0027t find vGPU types and set them in"},{"line_number":29,"context_line":"nova-cpu.conf."},{"line_number":30,"context_line":"Like pGPU passthrough to guests by cyborg component, cyborg can also manage"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f65232a_b7260216","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":44},"in_reply_to":"9f560f44_c5e99dab","updated":"2020-10-21 05:03:08.000000000","message":"maybe we can say: As an operator, I want to use general-purpose management software to manage heterogeneous accelerators such as GPUs in the platform.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"5dfab90f2cf4d6ce6911bdaf0ebfa7f1726486ec","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Use Cases"},{"line_number":25,"context_line":"---------"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"User wants to use vGPU in new way by cyborg. This is extremely useful for"},{"line_number":28,"context_line":"clouds where operators needn\u0027t find vGPU types and set them in"},{"line_number":29,"context_line":"nova-cpu.conf."},{"line_number":30,"context_line":"Like pGPU passthrough to guests by cyborg component, cyborg can also manage"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_c5e99dab","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":44},"in_reply_to":"9f560f44_e6f16b02","updated":"2020-10-15 12:27:45.000000000","message":"May we can say:\nAs a user, I would like to use vgpu conveniently, and the devices of the server can managed by accelerators, such as gpu, fpga etc., this can increase our maintenance cost of accelerators.\n\nAs a operators, I\u0027s appreciate to manage accelerators by one tools or a project, this can improve our ability to use accelerators.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"User wants to use vGPU in new way by cyborg. This is extremely useful for"},{"line_number":28,"context_line":"clouds where operators needn\u0027t find vGPU types and set them in"},{"line_number":29,"context_line":"nova-cpu.conf."},{"line_number":30,"context_line":"Like pGPU passthrough to guests by cyborg component, cyborg can also manage"},{"line_number":31,"context_line":"vGPU devices and make the operators use vGPU conveniently like other"},{"line_number":32,"context_line":"accelerator devices such as pGPU, FPGA and so on."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_6688bb6a","line":29,"updated":"2020-10-07 16:48:30.000000000","message":"this can be managed by any config management system or by vendor tooling, like how Red Hat OSP does this at the moment : https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/16.1/html/instances_and_images_guide/ch-virtual_gpu#","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":30,"context_line":"Like pGPU passthrough to guests by cyborg component, cyborg can also manage"},{"line_number":31,"context_line":"vGPU devices and make the operators use vGPU conveniently like other"},{"line_number":32,"context_line":"accelerator devices such as pGPU, FPGA and so on."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Proposed change"},{"line_number":35,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":36,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_c6cf07b1","line":33,"updated":"2020-10-07 16:48:30.000000000","message":"I\u0027d recommend you to use the formal phrasing we generally use in our specs :\n\"As a user, I\u0027d like to...\" or \"as an operator, I\u0027d appreciate to have...\" (for instance)","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":35,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"One GPU can be virtualized as multiple vGPUs, operator can use every vGPU to"},{"line_number":38,"context_line":"create guests."},{"line_number":39,"context_line":"Currently, operator find vGPU types, config vGPU types in nova-cpu.conf,"},{"line_number":40,"context_line":"then set flavor metadata with \"resources:VGPU\u003d1\", and finally use the flavor"},{"line_number":41,"context_line":"to create guest with vGPU devices."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_e6ca0b9e","line":38,"updated":"2020-10-07 16:48:30.000000000","message":"this is not the proposed change, this is current behaviour.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":42,"context_line":"This spec simple the process. By cyborg, vGPU types will be discover period"},{"line_number":43,"context_line":"by cyborg agent period task, then reported to Placement automatic. The"},{"line_number":44,"context_line":"operator just create device profile with vGPU types and set the device"},{"line_number":45,"context_line":"profile to flavor metadata like \"accel:device_profile\u003dvgpu-dp\", and finally"},{"line_number":46,"context_line":"use the flavor to create guest with vGPU devices."},{"line_number":47,"context_line":"The operator can choose any programs according to actual situation. Reduce"},{"line_number":48,"context_line":"the nova-cpu.conf is suggestion."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_26452316","line":45,"range":{"start_line":45,"start_character":11,"end_line":45,"end_character":62},"updated":"2020-10-07 16:48:30.000000000","message":"then you assume that this metadata will translate into both quantitative request (I want N vGPUs) and a qualitative request (from this specific type). Am I correct ?\n\nIn this case, you\u0027re proposing one profile per combination of \"nb of vGPUs\" multiplied by \"type of GPU\", correct ?","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"308a8a62e02673c4cbb8cd330b162dc51bee4a85","unresolved":false,"context_lines":[{"line_number":42,"context_line":"This spec simple the process. By cyborg, vGPU types will be discover period"},{"line_number":43,"context_line":"by cyborg agent period task, then reported to Placement automatic. The"},{"line_number":44,"context_line":"operator just create device profile with vGPU types and set the device"},{"line_number":45,"context_line":"profile to flavor metadata like \"accel:device_profile\u003dvgpu-dp\", and finally"},{"line_number":46,"context_line":"use the flavor to create guest with vGPU devices."},{"line_number":47,"context_line":"The operator can choose any programs according to actual situation. Reduce"},{"line_number":48,"context_line":"the nova-cpu.conf is suggestion."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f65232a_3a7819f7","line":45,"range":{"start_line":45,"start_character":11,"end_line":45,"end_character":62},"in_reply_to":"3f65232a_7782aaca","updated":"2020-10-21 07:04:41.000000000","message":"Please also mention the Cyborg driver side change so that we can fully support this vGPU management :https://review.opendev.org/#/c/758925/.(just mention that and add this link will make sense to me)","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"5dfab90f2cf4d6ce6911bdaf0ebfa7f1726486ec","unresolved":false,"context_lines":[{"line_number":42,"context_line":"This spec simple the process. By cyborg, vGPU types will be discover period"},{"line_number":43,"context_line":"by cyborg agent period task, then reported to Placement automatic. The"},{"line_number":44,"context_line":"operator just create device profile with vGPU types and set the device"},{"line_number":45,"context_line":"profile to flavor metadata like \"accel:device_profile\u003dvgpu-dp\", and finally"},{"line_number":46,"context_line":"use the flavor to create guest with vGPU devices."},{"line_number":47,"context_line":"The operator can choose any programs according to actual situation. Reduce"},{"line_number":48,"context_line":"the nova-cpu.conf is suggestion."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_e04acf21","line":45,"range":{"start_line":45,"start_character":11,"end_line":45,"end_character":62},"in_reply_to":"9f560f44_26452316","updated":"2020-10-15 12:27:45.000000000","message":"My point of view: If we use cyborg manage vGPU, we can create a device_profile with vgpu, that we dont need to maintain a profile in seperate. We are already support creating device_profile by cyborg, and support boot server with accelerator if the *accel:device_profile* in flavor, the completed BP in Ussuri, you can refer https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"c6028c50080563d8b1ea26c04e80371a11776d96","unresolved":false,"context_lines":[{"line_number":42,"context_line":"This spec simple the process. By cyborg, vGPU types will be discover period"},{"line_number":43,"context_line":"by cyborg agent period task, then reported to Placement automatic. The"},{"line_number":44,"context_line":"operator just create device profile with vGPU types and set the device"},{"line_number":45,"context_line":"profile to flavor metadata like \"accel:device_profile\u003dvgpu-dp\", and finally"},{"line_number":46,"context_line":"use the flavor to create guest with vGPU devices."},{"line_number":47,"context_line":"The operator can choose any programs according to actual situation. Reduce"},{"line_number":48,"context_line":"the nova-cpu.conf is suggestion."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3f65232a_7782aaca","line":45,"range":{"start_line":45,"start_character":11,"end_line":45,"end_character":62},"in_reply_to":"9f560f44_e04acf21","updated":"2020-10-21 05:03:08.000000000","message":"IMO, virt driver change should be the main point we need to cover in this spec. nova already supports writing mdev UUID into an instance XML, but if the mdev device is managed by  Cyborg, that will be a little different. So that\u0027s the change we need to propose.And we also need to make sure proposed change won\u0027t conflict with the current mdev process in virt driver.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":85,"context_line":"Other deployer impact"},{"line_number":86,"context_line":"---------------------"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"* vGPU types in nova-cpu.conf will be no need."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"Developer impact"},{"line_number":91,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_46f657da","line":88,"updated":"2020-10-07 16:48:30.000000000","message":"Only if operators want to deploy Cyborg for this.\nIn other words, this spec can\u0027t say it will deprecate this option.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a76417c1c706bc2c903343a57faef63079ba91b6","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"* Create vGPU mdev when spawn guest"},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"* Unit test and function test to be added"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Dependencies"},{"line_number":125,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_461f7708","line":122,"updated":"2020-10-07 16:48:30.000000000","message":"what are the exact required changes in Nova you\u0027re planning to ?\nFrom what I can see, there is no Nova code change, right?","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":26458,"name":"Brin Zhang","email":"zhangbailin@inspur.com","username":"zhangbailin"},"change_message_id":"5dfab90f2cf4d6ce6911bdaf0ebfa7f1726486ec","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"* Create vGPU mdev when spawn guest"},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"* Unit test and function test to be added"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Dependencies"},{"line_number":125,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f560f44_25da191c","line":122,"in_reply_to":"9f560f44_461f7708","updated":"2020-10-15 12:27:45.000000000","message":"No, we should change the code logical when the server use Cyborg manage the vgpu.","commit_id":"29af38e0c0f9b3f95f6619d37db007cc7cee64b0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"db6b9a23d2cf7c34c7330faaa40ad46fa113839f","unresolved":false,"context_lines":[{"line_number":21,"context_line":"As the long-term goal, the cyborg project aims to provide a general-purpose"},{"line_number":22,"context_line":"management framework for acceleration resources and GPU is one of the"},{"line_number":23,"context_line":"important accelerators that cyborg needs to support if cyborg wanna be that"},{"line_number":24,"context_line":"kind of framework. In vGPU case, Cyborg provides management for vGPU(discover,"},{"line_number":25,"context_line":"report, binding, etc), nova does the GPU resource scheduler and we also need"},{"line_number":26,"context_line":"to integrate with nova to do the vGPU allocation in virt driver level (eg."},{"line_number":27,"context_line":"writing a vGPU managed by cyborg into an instance\u0027s XML). So this spec aims"},{"line_number":28,"context_line":"to support vGPU management by Cyborg."}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_00d2d55c","line":25,"range":{"start_line":24,"start_character":33,"end_line":25,"end_character":22},"updated":"2020-10-26 11:07:02.000000000","message":"well, fwiw, nova also does the report and binding for vGPUs, so what would be the difference ? Please explain better why and how Cyborg would automatically discover the GPUs, know the vGPU types, etc.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"4f3f371dbbd146d9cff3d33e50fe59b94e7a8fd5","unresolved":false,"context_lines":[{"line_number":23,"context_line":"important accelerators that cyborg needs to support if cyborg wanna be that"},{"line_number":24,"context_line":"kind of framework. In vGPU case, Cyborg provides management for vGPU(discover,"},{"line_number":25,"context_line":"report, binding, etc), nova does the GPU resource scheduler and we also need"},{"line_number":26,"context_line":"to integrate with nova to do the vGPU allocation in virt driver level (eg."},{"line_number":27,"context_line":"writing a vGPU managed by cyborg into an instance\u0027s XML). So this spec aims"},{"line_number":28,"context_line":"to support vGPU management by Cyborg."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_14f162ce","line":27,"range":{"start_line":26,"start_character":70,"end_line":27,"end_character":57},"updated":"2020-10-27 19:36:37.000000000","message":"well cyborg wont be involed with ath it will just tell nova what mdev to use or what pci device to use depening on if its a vgpu or not.\n\nthe xml generation will be done entirely by the nova virt driver. cyborg should ideally not be aware that an xml exists and shoudl be abstracted from the virt dirver details.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"db6b9a23d2cf7c34c7330faaa40ad46fa113839f","unresolved":false,"context_lines":[{"line_number":30,"context_line":"Use Cases"},{"line_number":31,"context_line":"---------"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"As an operator, I want to use general-purpose management software to manage"},{"line_number":34,"context_line":"heterogeneous accelerators such as GPUs in the platform."},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_20f8b1e3","line":33,"range":{"start_line":33,"start_character":30,"end_line":33,"end_character":65},"updated":"2020-10-26 11:07:02.000000000","message":"well, this would only work with Cyborg and not with any other inventory software, right?\n\nIf so, please tell it, like by :\n\"As an operator, I want to use Cyborg to manage [...]\"","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"4f3f371dbbd146d9cff3d33e50fe59b94e7a8fd5","unresolved":false,"context_lines":[{"line_number":30,"context_line":"Use Cases"},{"line_number":31,"context_line":"---------"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"As an operator, I want to use general-purpose management software to manage"},{"line_number":34,"context_line":"heterogeneous accelerators such as GPUs in the platform."},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_94b612fa","line":33,"range":{"start_line":33,"start_character":30,"end_line":33,"end_character":65},"in_reply_to":"3f65232a_20f8b1e3","updated":"2020-10-27 19:36:37.000000000","message":"openstack as a whole is general purpose infrastucture manamnet software.\nnova and cyborg are both specific purpose managment layeers so this term is not correct to use in general.\n\naslo sylvain is wright this wont be plugable so this is specific to cyborg integration.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"db6b9a23d2cf7c34c7330faaa40ad46fa113839f","unresolved":false,"context_lines":[{"line_number":48,"context_line":"So that\u0027s the change we need to propose. And we also need to make sure proposed"},{"line_number":49,"context_line":"change won\u0027t conflict with the current mdev process in virt driver. Please see"},{"line_number":50,"context_line":"the Cyborg driver side change[3]_ so that we can fully support this vGPU"},{"line_number":51,"context_line":"management."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"Alternatives"},{"line_number":54,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_20839155","line":51,"updated":"2020-10-26 11:07:02.000000000","message":"how can you please explain what would be the design for the libvirt driver to have the difference between the already done methods in there for vGPUs and what Cyborg would do in it ?\n\nWhat would be the change, for example which inventories would use Cyborg for this, or how Cyborg would be able to create new mdevs without trampling the existing vGPU methods ?","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"4f3f371dbbd146d9cff3d33e50fe59b94e7a8fd5","unresolved":false,"context_lines":[{"line_number":48,"context_line":"So that\u0027s the change we need to propose. And we also need to make sure proposed"},{"line_number":49,"context_line":"change won\u0027t conflict with the current mdev process in virt driver. Please see"},{"line_number":50,"context_line":"the Cyborg driver side change[3]_ so that we can fully support this vGPU"},{"line_number":51,"context_line":"management."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"Alternatives"},{"line_number":54,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_548aba1a","line":51,"in_reply_to":"3f65232a_20839155","updated":"2020-10-27 19:36:37.000000000","message":"the only thing that should change is that the mdev woudl be provided by nova.\n\nin anycase the nova spec need to contain the detail fo any nova changes so you need to move the relevent info to this spec and have the cyborg one reference it if needed.\n\n@sylvain i think this is an either/or  propsal e.g. either you mange vgpus with cyborg or with nova on a given host.\nwe might suppor t resize between flavor that first had nova vgpus to flavor with cyborg vgpus but i dont think we shoudl support a mix of both on any given host. this also means tha t cyborg cannot assume it can manage all gpus on the system by default.\n\ne.g. the driver need to be configurable so that by defualt cybrog will not report inventories for all gpus on the host either via  a device filer or other config options.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"4f3f371dbbd146d9cff3d33e50fe59b94e7a8fd5","unresolved":false,"context_lines":[{"line_number":83,"context_line":"Performance Impact"},{"line_number":84,"context_line":"------------------"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"None"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"Other deployer impact"},{"line_number":89,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_74b536cd","line":86,"range":{"start_line":86,"start_character":0,"end_line":86,"end_character":4},"updated":"2020-10-27 19:36:37.000000000","message":"there will be a minor negitive performace hit to schduling and spawning as we will need to interact with the cycborg service.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"4f3f371dbbd146d9cff3d33e50fe59b94e7a8fd5","unresolved":false,"context_lines":[{"line_number":88,"context_line":"Other deployer impact"},{"line_number":89,"context_line":"---------------------"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"* vGPU types in nova-cpu.conf will be no need."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Developer impact"},{"line_number":94,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_54b8fac6","line":91,"range":{"start_line":91,"start_character":0,"end_line":91,"end_character":46},"updated":"2020-10-27 19:36:37.000000000","message":"for this to be true you would need an upgrade path form nova ot cybporg vgpu tracking#\nthe simpler approch is to do resizes but i dont see any way to do it inplace.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"db6b9a23d2cf7c34c7330faaa40ad46fa113839f","unresolved":false,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":".._[1]:https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html"},{"line_number":147,"context_line":".._[2]:https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html"},{"line_number":148,"context_line":".._[3]:https://review.opendev.org/#/c/758925/"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"History"},{"line_number":151,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_6094698c","line":148,"updated":"2020-10-26 11:07:02.000000000","message":"honestly, I looked at this change and I haven\u0027t seen how this would even work with the Placement RPs.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"4f3f371dbbd146d9cff3d33e50fe59b94e7a8fd5","unresolved":false,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":".._[1]:https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html"},{"line_number":147,"context_line":".._[2]:https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html"},{"line_number":148,"context_line":".._[3]:https://review.opendev.org/#/c/758925/"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"History"},{"line_number":151,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3f65232a_b49d0e59","line":148,"in_reply_to":"3f65232a_6094698c","updated":"2020-10-27 19:36:37.000000000","message":"looking at the cyborg spec it really does not explain how the device filtering work or how the placment resouce will be reported. so i think both specs need significant work.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"a5a666339990ed6d56e196cd77b21c63bc229bd7","unresolved":false,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":".._[1]:https://specs.openstack.org/openstack/nova-specs/specs/ussuri/implemented/nova-cyborg-interaction.html"},{"line_number":147,"context_line":".._[2]:https://docs.openstack.org/nova/ussuri/admin/virtual-gpu.html"},{"line_number":148,"context_line":".._[3]:https://review.opendev.org/#/c/758925/"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"History"},{"line_number":151,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1f621f24_5db9c112","line":148,"in_reply_to":"3f65232a_b49d0e59","updated":"2020-11-13 06:27:07.000000000","message":"Hi @Sylvain and @Sean, please see the update on https://review.opendev.org/#/c/758925/. I have updated according to AGREEMENT on the PTG discussion.","commit_id":"005019167fb64764828f413d39e254e91b2c380f"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"21ccb393aad91a02d35753b789f65c3da03d0fdf","unresolved":false,"context_lines":[{"line_number":92,"context_line":"           # add to guest XML"},{"line_number":93,"context_line":"           guest.add_device(dev)"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"3. Who creates mdev device in the sys path?"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"   In theory, if nova and cyborg both can support vGPU management, nova or"},{"line_number":98,"context_line":"   cyborg should support create mdev in its own respective. However both"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1f621f24_fea088f6","line":95,"range":{"start_line":95,"start_character":3,"end_line":95,"end_character":43},"updated":"2020-11-13 01:52:33.000000000","message":"I prefer Solution 1. It\u0027s easy and reasonable. In Solution 2, we have to support create mdev in cyborg and take more time between nova and cyborg interaction, which decrease the server spawn success rate.","commit_id":"f39ebff31098207136949ebec205cf833f2655f0"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"21ccb393aad91a02d35753b789f65c3da03d0fdf","unresolved":false,"context_lines":[{"line_number":103,"context_line":""},{"line_number":104,"context_line":"   * Solution 1: nova creates a new mdev with the given \u0027attach_handle_uuid\u0027."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"     Changes requrired: No new change will happen in nova libvirt, just invoke"},{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"},{"line_number":108,"context_line":"     create mdev, and related change in cyborg-api."},{"line_number":109,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"1f621f24_5e59bcfb","line":106,"range":{"start_line":106,"start_character":13,"end_line":106,"end_character":22},"updated":"2020-11-13 01:52:33.000000000","message":"required","commit_id":"f39ebff31098207136949ebec205cf833f2655f0"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"21ccb393aad91a02d35753b789f65c3da03d0fdf","unresolved":false,"context_lines":[{"line_number":219,"context_line":"Other deployer impact"},{"line_number":220,"context_line":"---------------------"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Deployer should maake sure the device in one pci address is not configured in"},{"line_number":223,"context_line":"both Nova and Cyborg. If it is configured in bot, cyborg and nova should be"},{"line_number":224,"context_line":"able to raise this as an exception."},{"line_number":225,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"1f621f24_3e432015","line":222,"range":{"start_line":222,"start_character":16,"end_line":222,"end_character":21},"updated":"2020-11-13 01:52:33.000000000","message":"make","commit_id":"f39ebff31098207136949ebec205cf833f2655f0"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"21ccb393aad91a02d35753b789f65c3da03d0fdf","unresolved":false,"context_lines":[{"line_number":220,"context_line":"---------------------"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Deployer should maake sure the device in one pci address is not configured in"},{"line_number":223,"context_line":"both Nova and Cyborg. If it is configured in bot, cyborg and nova should be"},{"line_number":224,"context_line":"able to raise this as an exception."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Developer impact"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1f621f24_9e4d1440","line":223,"range":{"start_line":223,"start_character":45,"end_line":223,"end_character":48},"updated":"2020-11-13 01:52:33.000000000","message":"both","commit_id":"f39ebff31098207136949ebec205cf833f2655f0"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"21ccb393aad91a02d35753b789f65c3da03d0fdf","unresolved":false,"context_lines":[{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Deployer should maake sure the device in one pci address is not configured in"},{"line_number":223,"context_line":"both Nova and Cyborg. If it is configured in bot, cyborg and nova should be"},{"line_number":224,"context_line":"able to raise this as an exception."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Developer impact"},{"line_number":227,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1f621f24_fe6528b6","line":224,"range":{"start_line":224,"start_character":14,"end_line":224,"end_character":18},"updated":"2020-11-13 01:52:33.000000000","message":"conflict","commit_id":"f39ebff31098207136949ebec205cf833f2655f0"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_3b1949d6","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"updated":"2020-11-17 11:40:17.000000000","message":"as I said in the commit message, this is not true. The current nova driver for libvirt supports creating mdevs and attaching them to an instance domain since Queens.\n\nThis spec should explain why you wouldn\u0027t want to use the existing support for Cyborg too and why you should do other ways to do it.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_3694c036","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"in_reply_to":"1f621f24_3b1949d6","updated":"2020-11-17 12:47:57.000000000","message":"+1\n\nWhat will be the benefits of attaching an mdev (vGPU) via Cyborg compared to attaching the same vGPU today via nova + libvirt? For the end user? For the deployer? For the maintainer of the OpenStack codebase?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_76ebd876","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"in_reply_to":"1f621f24_3b1949d6","updated":"2020-11-17 14:49:11.000000000","message":"sylvain is correct nova has supporte mdev based vGPU for several release now. it is know to work with both intels and nvidias implemation of vGPU. amds mx_gpu uses sriov instead of vfio-mdev and uses the stanard pci alias approch instead.\n\nto enabel a vgpu in nova you simply add resouces:VGPU\u003d1 to your flavor after you have configured your host.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"8813ddb1_3b9003ec","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"in_reply_to":"1f621f24_76ebd876","updated":"2020-11-23 06:15:12.000000000","message":"Yes sylvain is correct, please see my reply and update in the commit message and here.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"932372c8345cedf9894f980ec77bc881fbf2191e","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"edb0ec6e_14cae2a5","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"in_reply_to":"8813ddb1_3b9003ec","updated":"2020-11-27 12:18:22.000000000","message":"@Yumeng: I still looking for the answers to my questions from above:\n\n\u003e What will be the benefits of attaching an mdev (vGPU) via Cyborg compared to attaching the \n\u003e same vGPU today via nova + libvirt? For the end user? For the deployer? For the maintainer \n\u003e of the OpenStack codebase?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3aed2c218cb88d8b966603d8a59a42e8c66d3c74","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"efd2692e_9ff95f56","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"in_reply_to":"edb0ec6e_14cae2a5","updated":"2020-12-07 11:47:10.000000000","message":"@Balazs Sorry for missing this question. The prominent benefit I can see is for the cloud maintainer(or the deployer), which also reflects the goal of Cyborg. For the maintainer, using Cyborg, it will have an unified interface to manage the devices.\ne.g. System admin can list all devices by one command \"openstack accelerator device list\", and see accelerator resource usage for instances(if applicable) by \"openstack accelerator arq list\", or calling program usage API by \"openstack accelerator deployable \u003cdeployable_uuid\u003e \u003cimage_uuid\u003e\" etc.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"b856973d_c9439cab","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"in_reply_to":"edb0ec6e_14cae2a5","updated":"2020-12-07 09:34:16.000000000","message":"IMHO, Cyborg is the accelerator device management project, include GPU/FPGA and vGPU and so on. support vGPU in Cyborg is neccessary for cloud platform end user.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e0e70b69ab115660e0858489a9d146071455f982","unresolved":false,"context_lines":[{"line_number":26,"context_line":"lifecycle management details can be found in [3]_. The whole cyborg-nova"},{"line_number":27,"context_line":"interaction can be found in [1]_."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"Currently, nova libvirt driver already supported allocating pass-through GPU"},{"line_number":30,"context_line":"accelerator to an instance as a PCI device, but not yet support that for"},{"line_number":31,"context_line":"a virtual GPU accelerator. This spec proposes to support composing vGPU"},{"line_number":32,"context_line":"accelerator into domain XML in nova libvirt driver."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":9,"id":"0e8cddf7_5f805b21","line":31,"range":{"start_line":29,"start_character":0,"end_line":31,"end_character":27},"in_reply_to":"efd2692e_9ff95f56","updated":"2020-12-17 13:30:55.000000000","message":"OK. So the goal here is to unify accelerator the device handling. Does it mean that you have the intention to deprecate Nova\u0027s GPU handling code in favor of cyborg GPU handling in the future? If so then please state it in the spec. Also if so then please detail the plan of moving users currently using nova\u0027s GPU support to use cyborg\u0027s GPU support. See my discussion with Sean at the upgrade section of this spec","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":34,"context_line":"Use Cases"},{"line_number":35,"context_line":"---------"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"As an operator, when I use Cyborg to manage lifecycle of mediated"},{"line_number":38,"context_line":"devices(eg. vGPU), I need a new way in nova virt driver to write the mdev"},{"line_number":39,"context_line":"accelerator info into an instance XML."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_f6f6c85c","line":38,"range":{"start_line":37,"start_character":57,"end_line":38,"end_character":7},"updated":"2020-11-17 14:49:11.000000000","message":"please dont pharse it this way.\n\nmediated device are not the same thing as vGPUs\n\nthis is needlessly generic and unhelpful.\n\njsut say \nAs an operator, want to use Cyborg to manage lifecycle of vGPUs.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":35,"context_line":"---------"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"As an operator, when I use Cyborg to manage lifecycle of mediated"},{"line_number":38,"context_line":"devices(eg. vGPU), I need a new way in nova virt driver to write the mdev"},{"line_number":39,"context_line":"accelerator info into an instance XML."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_7b0f4187","line":38,"range":{"start_line":38,"start_character":19,"end_line":38,"end_character":35},"updated":"2020-11-17 11:40:17.000000000","message":"why do you need a new way of doing this in Nova ? Operators don\u0027t care about the implementation, they only care about their user experience (ie. the config options and the projects they use, that\u0027s it)","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":35,"context_line":"---------"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"As an operator, when I use Cyborg to manage lifecycle of mediated"},{"line_number":38,"context_line":"devices(eg. vGPU), I need a new way in nova virt driver to write the mdev"},{"line_number":39,"context_line":"accelerator info into an instance XML."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_0a9d1865","line":38,"range":{"start_line":38,"start_character":19,"end_line":38,"end_character":35},"in_reply_to":"1f621f24_7b0f4187","updated":"2020-11-23 06:15:12.000000000","message":"Yes, as sean suggested. The use case is: As an operator, want to use Cyborg to manage lifecycle of vGPUs.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":35,"context_line":"---------"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"As an operator, when I use Cyborg to manage lifecycle of mediated"},{"line_number":38,"context_line":"devices(eg. vGPU), I need a new way in nova virt driver to write the mdev"},{"line_number":39,"context_line":"accelerator info into an instance XML."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_d68aacbc","line":38,"range":{"start_line":38,"start_character":19,"end_line":38,"end_character":35},"in_reply_to":"1f621f24_7b0f4187","updated":"2020-11-17 14:49:11.000000000","message":"the second half of this sentance can be removed.\n\nthe usecase is just\n\nAs an operator, want to use Cyborg to manage lifecycle of vGPUs.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":35,"context_line":"---------"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"As an operator, when I use Cyborg to manage lifecycle of mediated"},{"line_number":38,"context_line":"devices(eg. vGPU), I need a new way in nova virt driver to write the mdev"},{"line_number":39,"context_line":"accelerator info into an instance XML."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_595ff4e4","line":38,"range":{"start_line":38,"start_character":19,"end_line":38,"end_character":35},"in_reply_to":"1f621f24_d68aacbc","updated":"2020-11-23 06:15:12.000000000","message":"Done","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":41,"context_line":"Proposed change"},{"line_number":42,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"1. Define the data model in arq to track a MDEV accelerator."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"   This data model should provide attach_handle_type to distinguish from a"},{"line_number":47,"context_line":"   PCI device accelerator, and attach_handle_uuid as the mdev UUID which is"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_1b20cdf6","line":44,"range":{"start_line":44,"start_character":3,"end_line":44,"end_character":60},"updated":"2020-11-17 11:40:17.000000000","message":"that\u0027s not related to Nova, right?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":41,"context_line":"Proposed change"},{"line_number":42,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"1. Define the data model in arq to track a MDEV accelerator."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"   This data model should provide attach_handle_type to distinguish from a"},{"line_number":47,"context_line":"   PCI device accelerator, and attach_handle_uuid as the mdev UUID which is"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_76b3d8e9","line":44,"range":{"start_line":44,"start_character":3,"end_line":44,"end_character":60},"in_reply_to":"1f621f24_1b20cdf6","updated":"2020-11-17 12:47:57.000000000","message":"I think the below struct is relevant for nova to know what kind of XML piece needs to be generated. Isn\u0027t it?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":41,"context_line":"Proposed change"},{"line_number":42,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"1. Define the data model in arq to track a MDEV accelerator."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"   This data model should provide attach_handle_type to distinguish from a"},{"line_number":47,"context_line":"   PCI device accelerator, and attach_handle_uuid as the mdev UUID which is"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_0a5358e3","line":44,"range":{"start_line":44,"start_character":3,"end_line":44,"end_character":60},"in_reply_to":"1f621f24_76b3d8e9","updated":"2020-11-23 06:15:12.000000000","message":"yes, exactly. It is relevant for nova to know the UUID of mdev to merge into XML.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":54,"context_line":"           \u0027attach_handle_type\u0027: \u0027MDEV\u0027,"},{"line_number":55,"context_line":"           \u0027attach_handle_uuid\u0027: \u002791ac1606-427e-44bb-8233-f4ff4bf3d241\u0027,"},{"line_number":56,"context_line":"           \u0027attach_handle_info\u0027: {\u0027asked_type\u0027: \u0027nvidia-224\u0027,"},{"line_number":57,"context_line":"                                  \u0027domain\u0027: \u00270000\u0027,"},{"line_number":58,"context_line":"                                  \u0027bus\u0027: \u0027af\u0027,"},{"line_number":59,"context_line":"                                  \u0027device\u0027: \u002700\u0027,"},{"line_number":60,"context_line":"                                  \u0027function\u0027: \u00270\u0027}"},{"line_number":61,"context_line":"       }"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_f68b68bc","line":60,"range":{"start_line":57,"start_character":34,"end_line":60,"end_character":48},"updated":"2020-11-17 14:49:11.000000000","message":"this should not be present.\n\nyou should not be passing any pci info to nova in this case just the mdev uuid\n\ncyborg is not allowed to specify the address witin the guest wehre the device will be attached and nova does not need to know parent device.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":54,"context_line":"           \u0027attach_handle_type\u0027: \u0027MDEV\u0027,"},{"line_number":55,"context_line":"           \u0027attach_handle_uuid\u0027: \u002791ac1606-427e-44bb-8233-f4ff4bf3d241\u0027,"},{"line_number":56,"context_line":"           \u0027attach_handle_info\u0027: {\u0027asked_type\u0027: \u0027nvidia-224\u0027,"},{"line_number":57,"context_line":"                                  \u0027domain\u0027: \u00270000\u0027,"},{"line_number":58,"context_line":"                                  \u0027bus\u0027: \u0027af\u0027,"},{"line_number":59,"context_line":"                                  \u0027device\u0027: \u002700\u0027,"},{"line_number":60,"context_line":"                                  \u0027function\u0027: \u00270\u0027}"},{"line_number":61,"context_line":"       }"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_3c993e1f","line":60,"range":{"start_line":57,"start_character":34,"end_line":60,"end_character":48},"in_reply_to":"1f621f24_f68b68bc","updated":"2020-11-23 06:15:12.000000000","message":"ok, removed. once the mdev is pre-created by cyborg, this is not necessary here.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"   ::"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"       def _guest_add_accel_mdev_devices(self, guest, arq):"},{"line_number":72,"context_line":"           \"\"\"Adding a mdev accelerator from ARQ list."},{"line_number":73,"context_line":"           \"\"\""},{"line_number":74,"context_line":"           dev \u003d vconfig.LibvirtConfigGuestHostdevMDEV()"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_db6e552d","line":71,"range":{"start_line":71,"start_character":0,"end_line":71,"end_character":59},"updated":"2020-11-17 11:40:17.000000000","message":"you\u0027re litterally duplicating the existing libvirt methods we have in our tree. Why such thing ?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"   ::"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"       def _guest_add_accel_mdev_devices(self, guest, arq):"},{"line_number":72,"context_line":"           \"\"\"Adding a mdev accelerator from ARQ list."},{"line_number":73,"context_line":"           \"\"\""},{"line_number":74,"context_line":"           dev \u003d vconfig.LibvirtConfigGuestHostdevMDEV()"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_2fabb31d","line":71,"range":{"start_line":71,"start_character":0,"end_line":71,"end_character":59},"in_reply_to":"1f621f24_db6e552d","updated":"2020-11-23 06:15:12.000000000","message":"This method intends to take out the mdev UUID from acc_info and add to guest, which is similar to method _guest_add_accel_pci_devices(guest,acc_info)","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":81,"context_line":"           # nova.privsep.libvirt.create_mdev() or cyborg-api rpc call"},{"line_number":82,"context_line":"           # cyborg.agent.create_mdev() to do this during arq binding."},{"line_number":83,"context_line":"           mdev_info \u003d arq[\u0027attach_handle_info\u0027]"},{"line_number":84,"context_line":"           pci_addr \u003d \"{}:{}:{}.{}\".format("},{"line_number":85,"context_line":"                mdev_info[\u0027domain\u0027], mdev_info[\u0027bus\u0027],"},{"line_number":86,"context_line":"                mdev_info[\u0027device\u0027], mdev_info[\u0027function\u0027])"},{"line_number":87,"context_line":"           asked_type \u003d mdev_info[\u0027asked_type\u0027]"},{"line_number":88,"context_line":"           if not self._is_existing_mdev(dev.uuid):"},{"line_number":89,"context_line":"                nova.privsep.libvirt.create_mdev(pci_addr, asked_type,"},{"line_number":90,"context_line":"                                                 uuid\u003ddev.uuid)"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"           # add to guest XML"},{"line_number":93,"context_line":"           guest.add_device(dev)"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_2a7bb48b","line":90,"range":{"start_line":84,"start_character":9,"end_line":90,"end_character":63},"updated":"2020-11-17 14:49:11.000000000","message":"this i think woudl make sense to be done in cyborg not nova.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":81,"context_line":"           # nova.privsep.libvirt.create_mdev() or cyborg-api rpc call"},{"line_number":82,"context_line":"           # cyborg.agent.create_mdev() to do this during arq binding."},{"line_number":83,"context_line":"           mdev_info \u003d arq[\u0027attach_handle_info\u0027]"},{"line_number":84,"context_line":"           pci_addr \u003d \"{}:{}:{}.{}\".format("},{"line_number":85,"context_line":"                mdev_info[\u0027domain\u0027], mdev_info[\u0027bus\u0027],"},{"line_number":86,"context_line":"                mdev_info[\u0027device\u0027], mdev_info[\u0027function\u0027])"},{"line_number":87,"context_line":"           asked_type \u003d mdev_info[\u0027asked_type\u0027]"},{"line_number":88,"context_line":"           if not self._is_existing_mdev(dev.uuid):"},{"line_number":89,"context_line":"                nova.privsep.libvirt.create_mdev(pci_addr, asked_type,"},{"line_number":90,"context_line":"                                                 uuid\u003ddev.uuid)"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"           # add to guest XML"},{"line_number":93,"context_line":"           guest.add_device(dev)"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_9cb78a87","line":90,"range":{"start_line":84,"start_character":9,"end_line":90,"end_character":63},"in_reply_to":"1f621f24_2a7bb48b","updated":"2020-11-23 06:15:12.000000000","message":"ok, removed.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":101,"context_line":"   less change in cyborg, we would prefer this. But would like to see how"},{"line_number":102,"context_line":"   others think."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"   * Solution 1: nova creates a new mdev with the given \u0027attach_handle_uuid\u0027."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"     Changes required: No new change will happen in nova libvirt, just invoke"},{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_fb4871b3","line":104,"updated":"2020-11-17 11:40:17.000000000","message":"well, this could work, but after having done this since a while and saw the fact that mdevs aren\u0027t persisted, I personnally think it\u0027s better to have some other software (here, Cyborg) which would manage the mdevs beforehand.\n\nI personally have the idea to stop creating mdevs by Nova as it\u0027s not an orchestration software (exactly like we do for SR-IOV VFs) and pretend those mdevs already exist.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":101,"context_line":"   less change in cyborg, we would prefer this. But would like to see how"},{"line_number":102,"context_line":"   others think."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"   * Solution 1: nova creates a new mdev with the given \u0027attach_handle_uuid\u0027."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"     Changes required: No new change will happen in nova libvirt, just invoke"},{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_d2ad79eb","line":104,"in_reply_to":"1f621f24_56dfbc0b","updated":"2020-11-23 06:15:12.000000000","message":"Cyborg is fine with creating mdevs by itself.\nBut for deleting an instance with cyborg-owned vGPU, cyborg will delete this mdev UUID from sys path instead of just doing the unbound. This is different from the way that nova deletes the instance with nova-owned vGPU.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":101,"context_line":"   less change in cyborg, we would prefer this. But would like to see how"},{"line_number":102,"context_line":"   others think."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"   * Solution 1: nova creates a new mdev with the given \u0027attach_handle_uuid\u0027."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"     Changes required: No new change will happen in nova libvirt, just invoke"},{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_927db170","line":104,"in_reply_to":"1f621f24_eae9bce2","updated":"2020-11-23 06:15:12.000000000","message":"ok. then here we\u0027ve achieved the agreement: cyborg create mdevs","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":101,"context_line":"   less change in cyborg, we would prefer this. But would like to see how"},{"line_number":102,"context_line":"   others think."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"   * Solution 1: nova creates a new mdev with the given \u0027attach_handle_uuid\u0027."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"     Changes required: No new change will happen in nova libvirt, just invoke"},{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_0f07da1e","line":104,"in_reply_to":"1f621f24_fb4871b3","updated":"2020-11-23 06:15:12.000000000","message":"@Sylvain. yes, sure, agreed. Cyborg can create mdev beforehand.\nPS,I\u0027m a bit confused by the \"mdevs aren\u0027t persisted\" here.\nCan you please give a specific example of such case?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":101,"context_line":"   less change in cyborg, we would prefer this. But would like to see how"},{"line_number":102,"context_line":"   others think."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"   * Solution 1: nova creates a new mdev with the given \u0027attach_handle_uuid\u0027."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"     Changes required: No new change will happen in nova libvirt, just invoke"},{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_56dfbc0b","line":104,"in_reply_to":"1f621f24_fb4871b3","updated":"2020-11-17 12:47:57.000000000","message":"I support the idea to treat mdevs and SRIOV VF devices similarly from nova perspective. Which means both are pre-created devices on the hypervisor from nova viewpoint. If this means that cyborg pre-creates them then that is OK to me. Also if it means that the deployer makes sure that they exists that also works for me.\n\nI think it is good that Sylvain mentioned the problem of mdev persistency. If agree that Cyborg will create the mdevs then they need to know about this problem as they need to solve it.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":101,"context_line":"   less change in cyborg, we would prefer this. But would like to see how"},{"line_number":102,"context_line":"   others think."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"   * Solution 1: nova creates a new mdev with the given \u0027attach_handle_uuid\u0027."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"     Changes required: No new change will happen in nova libvirt, just invoke"},{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_eae9bce2","line":104,"in_reply_to":"1f621f24_fb4871b3","updated":"2020-11-17 14:49:11.000000000","message":"i dont agree with your \"nova is not an orchestration software argument\" the crfeation of mdevs and vfs are not equivelent.\n\nits tempting to say they are but when mdevs were intoduced they were intended to be created dynmically by higher level systems.\n\nthat said i think cyborg should create them","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"},{"line_number":108,"context_line":"     create mdev, and related change in cyborg-api."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"   * Solution 2: cyborg create a new mdev by itself."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_fb6fd124","line":110,"updated":"2020-11-17 11:40:17.000000000","message":"I\u0027m quite opposed to this solution, this would make nova calling an external agent, while Cyborg could rather just create the mdevs *before* directly by calling its agent.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"},{"line_number":108,"context_line":"     create mdev, and related change in cyborg-api."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"   * Solution 2: cyborg create a new mdev by itself."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_7213ed0e","line":110,"in_reply_to":"1f621f24_fb6fd124","updated":"2020-11-23 06:15:12.000000000","message":"Just to clarify, this will not cause extra calling on the nova side. nova-conductor just needs to call cyborg PATCH arq API as it used to do. I\u0027ve added the detailed interaction process below. Please check my update in the next patch.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":107,"context_line":"     nova.privsep.libvirt.create_mdev. No new change in cyborg-agent to"},{"line_number":108,"context_line":"     create mdev, and related change in cyborg-api."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"   * Solution 2: cyborg create a new mdev by itself."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_cad16025","line":110,"in_reply_to":"1f621f24_fb6fd124","updated":"2020-11-17 14:49:11.000000000","message":"it depends on when its done.\nif cyborg did it when we bound the ARQ in teh conductor then sure. if the suggestion is to do it from the compute node then no.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":110,"context_line":"   * Solution 2: cyborg create a new mdev by itself."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"     - reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":116,"context_line":"       API, cyborg-api firstly need to check if the target mdev already"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_d6fbcc75","line":113,"range":{"start_line":113,"start_character":41,"end_line":113,"end_character":50},"updated":"2020-11-17 12:47:57.000000000","message":"Is this the same arq binding we do already today for fpga accelerators?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":110,"context_line":"   * Solution 2: cyborg create a new mdev by itself."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"     - reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":116,"context_line":"       API, cyborg-api firstly need to check if the target mdev already"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_076693ee","line":113,"range":{"start_line":113,"start_character":41,"end_line":113,"end_character":50},"in_reply_to":"1f621f24_0a4eb8bf","updated":"2020-11-23 06:15:12.000000000","message":"yes, it is the same.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":110,"context_line":"   * Solution 2: cyborg create a new mdev by itself."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"     - reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":116,"context_line":"       API, cyborg-api firstly need to check if the target mdev already"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_0a4eb8bf","line":113,"range":{"start_line":113,"start_character":41,"end_line":113,"end_character":50},"in_reply_to":"1f621f24_d6fbcc75","updated":"2020-11-17 14:49:11.000000000","message":"that is what i was assuming.\nif we are reuing that im ok with this.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"     - reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":116,"context_line":"       API, cyborg-api firstly need to check if the target mdev already"},{"line_number":117,"context_line":"       exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"},{"line_number":118,"context_line":"       to create a new mdev, otherwise just return an expected arq."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"     Changes required: cyborg-agent add create mdev function, and cyborg-api"},{"line_number":121,"context_line":"     need to change the GET arq and PATCH arq APIs."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_8da30200","line":118,"range":{"start_line":115,"start_character":4,"end_line":118,"end_character":67},"updated":"2020-11-17 14:49:11.000000000","message":"just to clarify, for fpgas today do we currently call get arq on vm reboot. if so this is ok but if we currently dont call cyborg durign hard reboot im not sure if we should add a new call.\n\nmaking \"get\" do a rpc call to the agent feels kind fo heavy wait.\n\ni would prefer if you had a perodic that checked if the mdev exsited and reflected that in the binding state of the arq or something.. e.g. if its not there set it to \"provisioning\" or something then leave the get as a simple db lookup and have nova loop waiting for the bind event like we do with the inital binding step in spawn.\n\ne.g. on a host reboot cyborg would check which arqs correspond to current host and create the mdevs required at startup setting there state to provisioning until done. When nova resumes guest on host start or via api call then it will wait for the arq to get to the bound state or whatever state it waits for on spawn. when cybrog finishes setting creating the mdevs it will set them to that state allowing nova to proceed.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"     - reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":116,"context_line":"       API, cyborg-api firstly need to check if the target mdev already"},{"line_number":117,"context_line":"       exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"},{"line_number":118,"context_line":"       to create a new mdev, otherwise just return an expected arq."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"     Changes required: cyborg-agent add create mdev function, and cyborg-api"},{"line_number":121,"context_line":"     need to change the GET arq and PATCH arq APIs."}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_67915f9a","line":118,"range":{"start_line":115,"start_character":4,"end_line":118,"end_character":67},"in_reply_to":"1f621f24_8da30200","updated":"2020-11-23 06:15:12.000000000","message":"yes, currently we do call get arq on VM reboot[0].\nI personally think this periodic check is quite a good idea(I will discuss with the cyborg team to see how others think). Assuming that cyborg does the periodic check if mdev existed and follows up necessary operations. e.g. On a host reboot, mdev will be missing from the sys path. In this situation cyborg may need to 1) set arq status to \"UNBOUND\", 2) re-create mdev in sys path using the UUID recorded in the arq table 3) update the status to \"BOUND\" and notify nova. And on nova compute side, I suppose, in the method _resume_guests_state, we might need to add a new call to get acc_info, am I right?\n[0]:https://github.com/openstack/nova/blob/master/nova/compute/manager.py#L3755","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3eb32695a8524eb1fddebf4cb361d1f25d4af9ba","unresolved":false,"context_lines":[{"line_number":112,"context_line":"     - spawn instance interaction: cyborg creates a new mdev by rpc.call"},{"line_number":113,"context_line":"       cyborg-agent when it receives the PATCH arq request from nova-compute."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"     - reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":116,"context_line":"       API, cyborg-api firstly need to check if the target mdev already"},{"line_number":117,"context_line":"       exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"},{"line_number":118,"context_line":"       to create a new mdev, otherwise just return an expected arq."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"     Changes required: cyborg-agent add create mdev function, and cyborg-api"},{"line_number":121,"context_line":"     need to change the GET arq and PATCH arq APIs."}],"source_content_type":"text/x-rst","patch_set":9,"id":"e6f0b2e6_ee73b3ca","line":118,"range":{"start_line":115,"start_character":4,"end_line":118,"end_character":67},"in_reply_to":"fffc6b78_67915f9a","updated":"2020-11-27 09:07:32.000000000","message":"@Sean, I reconsidered about the periodic check. in the periodic check, if mdev does not exsit, cyborg re-create it and update the arq to \"BOUND\" state. From nova\u0027s perspective, when nova resumes guest on host restart, it will get_arq. if at that time the state is not \"BOUND\", nova needs to wait. but what should nova do about the wait? should it come back later or wait for notification and resumes the instance? Sounds like it will increase the burden in nova side. \n\nSo, I was thinking another possible solution: Cyborg does the check only when a get_arq request comes instead of checking it periodically. e.g. when nova resumes guest on host start, nova calls cyborg\u0027s get_arq to get mdev UUID. on cyborg API side, when it receives this get_arq(if attach_type\u003dMDEV) request, it checks if mdev exists, if yes then return UUID directly, if not then create the mdev and return its UUID.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":118,"context_line":"       to create a new mdev, otherwise just return an expected arq."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"     Changes required: cyborg-agent add create mdev function, and cyborg-api"},{"line_number":121,"context_line":"     need to change the GET arq and PATCH arq APIs."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"4. Avoid conflicts when cyborg vGPU management co-exists with nova"},{"line_number":124,"context_line":"   vGPU management."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_1be3ada5","line":121,"updated":"2020-11-17 11:40:17.000000000","message":"to be clear, I see a solution 3 : Cyborg creates the mdevs before those are allocated to instances, based on what the operator defined.\n\nIn this scenario, operator defines the mdev type for each PCI device, and then Cyborg calls cyborg-agent for generating the mdev.\n\nIn this case, the only thing missing would be the mdev binding to the guest domain, but we already have this in nova.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":118,"context_line":"       to create a new mdev, otherwise just return an expected arq."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"     Changes required: cyborg-agent add create mdev function, and cyborg-api"},{"line_number":121,"context_line":"     need to change the GET arq and PATCH arq APIs."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"4. Avoid conflicts when cyborg vGPU management co-exists with nova"},{"line_number":124,"context_line":"   vGPU management."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_4debaa11","line":121,"in_reply_to":"1f621f24_1be3ada5","updated":"2020-11-17 14:49:11.000000000","message":"am that would still need some kind of syconisiation so that we can ensure that cybrog has finshed setting up the mdevs before the guest boots after a host reboot. but precreating them would be vald for devices that only support one type but it not really vaild without more complicated config options for devics that support multiple.\n\ni think this shoudl be left ot cybrog to decided internally but there are advantages to precreating them too.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":125,"context_line":""},{"line_number":126,"context_line":"   * Use owner (nova, cyborg) trait in Placement when inventory is reported."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"     - This will need to add a new namepace in os-traits: OWNER_NOVA,"},{"line_number":129,"context_line":"       OWNER_CYBORG. Then cyborg/nova report owner trait as one of the trait"},{"line_number":130,"context_line":"       when inventory is reported."},{"line_number":131,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_2dd74e4d","line":128,"range":{"start_line":128,"start_character":35,"end_line":128,"end_character":43},"updated":"2020-11-17 14:49:11.000000000","message":"nit the namespace is OWNER \n\nOWNER_NOVA is a trait not a namespace.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":125,"context_line":""},{"line_number":126,"context_line":"   * Use owner (nova, cyborg) trait in Placement when inventory is reported."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"     - This will need to add a new namepace in os-traits: OWNER_NOVA,"},{"line_number":129,"context_line":"       OWNER_CYBORG. Then cyborg/nova report owner trait as one of the trait"},{"line_number":130,"context_line":"       when inventory is reported."},{"line_number":131,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_a7306709","line":128,"range":{"start_line":128,"start_character":35,"end_line":128,"end_character":43},"in_reply_to":"1f621f24_2dd74e4d","updated":"2020-11-23 06:15:12.000000000","message":"Done","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":136,"context_line":"         ::"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"           trait1: OWNER_CYBORG."},{"line_number":139,"context_line":"           trait2: CUSTOM_\u003cVENDOR_NAME\u003e_\u003cPRODUCT_ID\u003e_\u003cVirtual_GPU_Type\u003e."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":142,"context_line":"         for vGPU resources during inventory report."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_8d3f8285","line":139,"range":{"start_line":139,"start_character":40,"end_line":139,"end_character":52},"updated":"2020-11-17 14:49:11.000000000","message":"this should not be needed.\n\nif you have the same mdev type from two different products then they shoudl be equivelent and interchangeable otherwise it should not be the same mdev type.\n\nso this could just be CUSTOM_\u003cmdev_type\u003e\n\nthis is out of scope of nova however since we will just pass these traits to placement.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3eb32695a8524eb1fddebf4cb361d1f25d4af9ba","unresolved":false,"context_lines":[{"line_number":136,"context_line":"         ::"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"           trait1: OWNER_CYBORG."},{"line_number":139,"context_line":"           trait2: CUSTOM_\u003cVENDOR_NAME\u003e_\u003cPRODUCT_ID\u003e_\u003cVirtual_GPU_Type\u003e."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":142,"context_line":"         for vGPU resources during inventory report."}],"source_content_type":"text/x-rst","patch_set":9,"id":"beca6e63_4b4ed043","line":139,"range":{"start_line":139,"start_character":40,"end_line":139,"end_character":52},"in_reply_to":"1f621f24_8d3f8285","updated":"2020-11-27 09:07:32.000000000","message":"makes sense from a user\u0027s perspective.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":138,"context_line":"           trait1: OWNER_CYBORG."},{"line_number":139,"context_line":"           trait2: CUSTOM_\u003cVENDOR_NAME\u003e_\u003cPRODUCT_ID\u003e_\u003cVirtual_GPU_Type\u003e."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":142,"context_line":"         for vGPU resources during inventory report."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"     - When the end user makes a vGPU request, one can request a cyborg owned"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_8d5822fa","line":141,"range":{"start_line":141,"start_character":64,"end_line":141,"end_character":74},"updated":"2020-11-17 14:49:11.000000000","message":"i would actully suggest we add OWNER_NOVA to all RPs nova creates and other service should do likewise.\n\ne.g. cyborg will add OWNER_CYBORG to all its RPs and neutron will add OWNER_NEUTRON exctra. right now this is not required but i think if we have owner triats we shoudl do this by defualt for all rps.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":138,"context_line":"           trait1: OWNER_CYBORG."},{"line_number":139,"context_line":"           trait2: CUSTOM_\u003cVENDOR_NAME\u003e_\u003cPRODUCT_ID\u003e_\u003cVirtual_GPU_Type\u003e."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":142,"context_line":"         for vGPU resources during inventory report."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"     - When the end user makes a vGPU request, one can request a cyborg owned"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_ca0db238","line":141,"range":{"start_line":141,"start_character":64,"end_line":141,"end_character":74},"in_reply_to":"1f621f24_8d5822fa","updated":"2020-11-23 06:15:12.000000000","message":"that would be cool :)","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":141,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":142,"context_line":"         for vGPU resources during inventory report."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"     - When the end user makes a vGPU request, one can request a cyborg owned"},{"line_number":145,"context_line":"       vGPU by pre-defined device profile, and request a nova owned vGPU by"},{"line_number":146,"context_line":"       pre-defined flavor."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"},{"line_number":149,"context_line":"       by the traits. No new change is required."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_8ddb024d","line":146,"range":{"start_line":144,"start_character":4,"end_line":146,"end_character":26},"updated":"2020-11-17 14:49:11.000000000","message":"nit currently device-profiles are requested via the flavor\n\nwhat you ment was in the nova case you can use resouces:VGPU\u003d1\nand we can transform that to \nresouces##:VGPU\u003d1, trait##:OWNER_NOVA\u003drequired\nvia a prefilter.\n\nand in the cyborg case you will add device_profile\u003d\u003cvgpu profile\u003e","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":141,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":142,"context_line":"         for vGPU resources during inventory report."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"     - When the end user makes a vGPU request, one can request a cyborg owned"},{"line_number":145,"context_line":"       vGPU by pre-defined device profile, and request a nova owned vGPU by"},{"line_number":146,"context_line":"       pre-defined flavor."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"},{"line_number":149,"context_line":"       by the traits. No new change is required."}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_4ae38234","line":146,"range":{"start_line":144,"start_character":4,"end_line":146,"end_character":26},"in_reply_to":"1f621f24_8ddb024d","updated":"2020-11-23 06:15:12.000000000","message":"Done","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_bb0a7922","line":183,"updated":"2020-11-17 11:40:17.000000000","message":"Before discussing about how to segregate within cyborg and nova mdevs, I\u0027d prefer we discuss about the big picture.\n\nI\u0027d prefer solution 3 honestly and the workflow I could see would be :\n\n- Cyborg precreates the mediated devices on computes by calling cyborg-agent directly, after the operator defined the mdev types for each GPU\n- Cyborg reports the VGPU inventories accordingly once those mdevs are created\n- When a new instance comes by, Nova asks Cyborg whether it can allocate a mdev, and then Cyborg provides an existing mdev uuid\n- at spawn() time, nova uses the existing binding methods we have for attaching the mdev uuid to the guest definition.\n\nEt voila, maximum reuse with minimum overlap.\nThis would leave Cyborg managing the fleet of mdevs exactly like those were SR-IOV VFs, and Cyborg could decide to persist mdev information on computes since those aren\u0027t persisted (something which nova doesn\u0027t do atm). In the case of a compute reboot, cyborg-agent could recreate the mdevs without involving nova at all, which is the best IMHO.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecd19edf0908846a7120e71505ac378a68d97040","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"93993c3f_8a586fcd","line":183,"in_reply_to":"14a31b48_05f9e85a","updated":"2020-12-17 13:38:40.000000000","message":"Yumeng: Your last suggestion is OK to me.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_8af44a5d","line":183,"in_reply_to":"1f621f24_969dd40b","updated":"2020-11-23 06:15:12.000000000","message":"Good catch. reboot is a case that I also wanna discuss.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_00e5f937","line":183,"in_reply_to":"1f621f24_969dd40b","updated":"2020-11-17 14:49:11.000000000","message":"yes this is why most deployment using systemd required nova to start after the netwrok target to ensure that the vfs are allocated before nova starts.\n\nthe same is not typically true for mdevs since the expectation is for the orchestrator to crate them as required.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_969dd40b","line":183,"in_reply_to":"1f621f24_bb0a7922","updated":"2020-11-17 12:47:57.000000000","message":"\u003e Before discussing about how to segregate within cyborg and nova\n \u003e mdevs, I\u0027d prefer we discuss about the big picture.\n \u003e \n \u003e I\u0027d prefer solution 3 honestly and the workflow I could see would\n \u003e be :\n \u003e \n \u003e - Cyborg precreates the mediated devices on computes by calling\n \u003e cyborg-agent directly, after the operator defined the mdev types\n \u003e for each GPU\n \u003e - Cyborg reports the VGPU inventories accordingly once those mdevs\n \u003e are created\n \u003e - When a new instance comes by, Nova asks Cyborg whether it can\n \u003e allocate a mdev, and then Cyborg provides an existing mdev uuid\n\nFrom nova perspective I think it does not matter if the mdev it gets from Cyborg at this point was pre-created or created on the fly. \n\n \u003e - at spawn() time, nova uses the existing binding methods we have\n \u003e for attaching the mdev uuid to the guest definition.\n \u003e \n \u003e Et voila, maximum reuse with minimum overlap.\n \u003e This would leave Cyborg managing the fleet of mdevs exactly like\n \u003e those were SR-IOV VFs, and Cyborg could decide to persist mdev\n \u003e information on computes since those aren\u0027t persisted (something\n \u003e which nova doesn\u0027t do atm). In the case of a compute reboot,\n \u003e cyborg-agent could recreate the mdevs without involving nova at\n \u003e all, which is the best IMHO.\n\nThis reboot case makes me wonder about ordering. Assume we have a VM on the host using a cyborg managed mdev, so the mdev is referred in the VM\u0027s domain xml. Then the hypervisor reboot happens so the mdev is gone. The nova-compute and the cyborg-agent restarts along with the hypervisor reboot. How could we make sure that the nova-compute does not come up earlier than the cyborg-agent? Will the nova-compute delays startup until cyborg-agent is started up and re-created the mdevs? If not then resume_guests_state_on_host_boot\u003dTrue will cause or VM to go into ERROR state. \n\nIn the SRIOV case the solution (I know of) for the disappearing VF problem is to configure the _hypervisor_ to re-create the VFs at PF driver loading. And nova assumes that such driver loading happens early in the hypervisor boot process so when nova-compute starts up the VFs already exists and therefore the VMs can be started up.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_aaef8e85","line":183,"in_reply_to":"1f621f24_bb0a7922","updated":"2020-11-23 06:15:12.000000000","message":"From cyborg\u0027s perspective, it is a problem of choosing \"creating as necessary\" or \"create all and allocate as necessary\". might need to figure out more about the details. I will come back later and update.\nHowever, IMO, cyborg-agent can recreate the mdev without involving nova in either way.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"932372c8345cedf9894f980ec77bc881fbf2191e","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"e150997d_f0bf535c","line":183,"in_reply_to":"bd886485_fbf1545d","updated":"2020-11-27 12:18:22.000000000","message":"Do I understand correctly that the  solution of host reboot is that nova-compute will call cyborg during the hard_reboot of the VM (triggered by resume_guests_state_on_host_boot\u003dTrue config) and nova waits until cyborg can respond with the re-created mdev. So if there are VMs on a host using cyborg managed mdevs, and if resume_guests_state_on_host_boot\u003dTrue then the nova-compute startup sequence will be synchronized with the cyborg startup sequence.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3aed2c218cb88d8b966603d8a59a42e8c66d3c74","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"14a31b48_05f9e85a","line":183,"in_reply_to":"e150997d_f0bf535c","updated":"2020-12-07 11:47:10.000000000","message":"@Balazs I was thinking this as a possible solution before, but this might need to modify init_host when host starts, which is not a general solution. So I think we can just follow how we process so far, if it is an unexpected host reboot, the VM STATUS will be shut down after host reboot, users need to handle it on their own. What would be your suggestions?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"816a9db7_b2e58905","line":183,"in_reply_to":"e150997d_f0bf535c","updated":"2020-12-07 09:34:16.000000000","message":"IMHO, it\u0027s not necessary to synchronize. if the cyborg service start late, we cannot start vm, just like accelerator process so far.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3eb32695a8524eb1fddebf4cb361d1f25d4af9ba","unresolved":false,"context_lines":[{"line_number":180,"context_line":""},{"line_number":181,"context_line":"     - And accordingly, we will also need new change in mdev_tag\u0027s consumer"},{"line_number":182,"context_line":"       side, the nova operations side, such as hard_reboot and spawn."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Alternatives"},{"line_number":185,"context_line":"------------"},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"bd886485_fbf1545d","line":183,"in_reply_to":"fffc6b78_aaef8e85","updated":"2020-11-27 09:07:32.000000000","message":"Hi @Sylvain,@Balazs and @Sean, I\u0027m back again to sync that cyborg AGRRED that \"creating mdev at arq binding time\" is more efficient(if there is no obvious performance gap) because this saves the maintainance consumption compared to that of \"pre-create all the allocates even it is not requested\". So from cyborg\u0027s internal perspective, we would prefer \"creating mdev at arq binding time\".","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":214,"context_line":"Performance Impact"},{"line_number":215,"context_line":"------------------"},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"Nova and Cyborg interaction will be a minor impact in scheduling and spawning."},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"Other deployer impact"},{"line_number":220,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_bbaef92a","line":217,"updated":"2020-11-17 11:40:17.000000000","message":"Please just tell that you will have new HTTP roundtrips with an external service","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":214,"context_line":"Performance Impact"},{"line_number":215,"context_line":"------------------"},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"Nova and Cyborg interaction will be a minor impact in scheduling and spawning."},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"Other deployer impact"},{"line_number":220,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_4afa6242","line":217,"in_reply_to":"1f621f24_a0c24583","updated":"2020-11-23 06:15:12.000000000","message":"yes. So seems I should just leave a None here, right?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":214,"context_line":"Performance Impact"},{"line_number":215,"context_line":"------------------"},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"Nova and Cyborg interaction will be a minor impact in scheduling and spawning."},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"Other deployer impact"},{"line_number":220,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_a0c24583","line":217,"in_reply_to":"1f621f24_bbaef92a","updated":"2020-11-17 14:49:11.000000000","message":"we should not need any new request to cyborg for this.\nthe existing ones for fpgas can be resued","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"a6ac434e9286c70eb4beedb0726739cf4066e599","unresolved":false,"context_lines":[{"line_number":220,"context_line":"---------------------"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":223,"context_line":"both Nova and Cyborg. If it is configured in both cyborg and nova should be"},{"line_number":224,"context_line":"able to raise this as an conflict exception."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Developer impact"},{"line_number":227,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_9b6b3def","line":224,"range":{"start_line":223,"start_character":22,"end_line":224,"end_character":44},"updated":"2020-11-17 11:40:17.000000000","message":"which project would be responsible for raising this exception ?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":220,"context_line":"---------------------"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":223,"context_line":"both Nova and Cyborg. If it is configured in both cyborg and nova should be"},{"line_number":224,"context_line":"able to raise this as an conflict exception."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Developer impact"},{"line_number":227,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_d6262cce","line":224,"range":{"start_line":223,"start_character":22,"end_line":224,"end_character":44},"in_reply_to":"1f621f24_9b6b3def","updated":"2020-11-17 12:47:57.000000000","message":"This is interesting. I think if we want to detect the conflict then the detection would be needed on both sides. I.e.:\n\n* If nova sees a PCI device in the whitelist that has an RP already in placement with OWNER_CYBORG then nova-compute can simply fail to start\n\n* if cyborg wants to create a RP for a PCI device but such RP already exists in placement with an OWNER_NOVA trait, then cyborg should fail.\n\nHowever the second check cannot be implemented today as nova does not report PCI devices in placement today. The PCI tracker is internal and in memory in nova-compute service.\n\nAssuming that in the future nova will store PCI inventory in placement, we need to make sure that both Nova and Cyborg will use the same RP name for the same PCI device. So that placement can reject the second RP creation for the same device. Even after the agreed naming schema we can have race condition between Nova and Cyborg trying to create the RP and then adding the OWNER_ trait to the RP, as these are two separate operations in placement. E.g. There could be a case when the device RP exists but no OWNER_ trait is added yet to it.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"9b3dcba07d47d7d67293dffd5cf6da1e666eacbb","unresolved":false,"context_lines":[{"line_number":220,"context_line":"---------------------"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":223,"context_line":"both Nova and Cyborg. If it is configured in both cyborg and nova should be"},{"line_number":224,"context_line":"able to raise this as an conflict exception."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Developer impact"},{"line_number":227,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"fffc6b78_ea9eb605","line":224,"range":{"start_line":223,"start_character":22,"end_line":224,"end_character":44},"in_reply_to":"1f621f24_d6262cce","updated":"2020-11-23 06:15:12.000000000","message":"Agree, we should make sure Cyborg and Nova use the same RP name for the same PCI device, so the placement will reject the second RP creation, then both sides can aware this.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":220,"context_line":"---------------------"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":223,"context_line":"both Nova and Cyborg. If it is configured in both cyborg and nova should be"},{"line_number":224,"context_line":"able to raise this as an conflict exception."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Developer impact"},{"line_number":227,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_e066fdba","line":224,"range":{"start_line":223,"start_character":22,"end_line":224,"end_character":44},"in_reply_to":"1f621f24_d6262cce","updated":"2020-11-17 14:49:11.000000000","message":"the pci tracker is stored in the db and all updates to it are atomicly commited to the db.\n\nthe in memory part you are refering to is that we load the list of devices we own/manage at start up and then operate on that in memroy without looking at the devices again. so if the device goes away nova will not see that.\n\nwe could use the same rp name yes but it basically would have to be the pci address of the pf.\n\nhonestly i think this is out fo scope for a runtime check and the better way to solve this is to move to a common config file like the provider.yaml which both nova and cyborg or even neutron could read form to contol this. or provide a tool to check this externally and in nova and cyborg jsut docuemnt it.\n\nno need for an excption.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":231,"context_line":"Upgrade impact"},{"line_number":232,"context_line":"--------------"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"None"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"Implementation"},{"line_number":237,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_aa0fc42b","line":234,"updated":"2020-11-17 12:47:57.000000000","message":"I imagine that whatever nova supports today regarding mdevs (vGPU) Cyborg at least will support the same and will also support some extra (see the question at L31 regarding this extra). \n\nAt that point a deployer that started using nova\u0027s mdev support might want to migrate to the cyborg\u0027s mdev support. We should be able to support that migration somehow. Do you have ideas how to make it work? \n\nWe can always document the harsh process of:\n1) delete the VMs that using the nova\u0027s mdev support\n2) deconfigure mdevs in the nova-compute\n3) configure mdevs in the cyborg side\n4) recreate the VMs with cyborg provided mdevs\n\nBut if there is a better way then deployers would be glad.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":231,"context_line":"Upgrade impact"},{"line_number":232,"context_line":"--------------"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"None"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"Implementation"},{"line_number":237,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_e0fb9dba","line":234,"in_reply_to":"1f621f24_aa0fc42b","updated":"2020-11-17 14:49:11.000000000","message":"we dont need to do any delets\n\nto migrate a vm you jsut resize the flavor.\n\nif you to migrate a host live/cold migrate everything off then update the config.\n\nthere shoudl be no need to delete and recreate vms","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"932372c8345cedf9894f980ec77bc881fbf2191e","unresolved":false,"context_lines":[{"line_number":231,"context_line":"Upgrade impact"},{"line_number":232,"context_line":"--------------"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"None"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"Implementation"},{"line_number":237,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f8b4e12_e2e5a03f","line":234,"in_reply_to":"1f621f24_e0fb9dba","updated":"2020-11-27 12:18:22.000000000","message":"OK So with a resize to a flavor without VGPU resources, would remove the nova managed mdev from the VM. But then how could the user add a cyborg managed mdev to the VM as a replacement?\n\nIn my upgrade scenario the deployer wants to make sure that nova managed mdevs are replaced with cyborg managed mdevs. So far I don\u0027t see how this can be solved without delete + recrete.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"26aade8418d1d961926b44487abd3255eeb4b204","unresolved":false,"context_lines":[{"line_number":231,"context_line":"Upgrade impact"},{"line_number":232,"context_line":"--------------"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"None"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"Implementation"},{"line_number":237,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bb572732_60e5dc26","line":234,"in_reply_to":"1f8b4e12_e2e5a03f","updated":"2020-11-27 12:40:04.000000000","message":"you resize directly from a flavor with a nova managed gpu (resouces:vgpu\u003d1 in the flavor)\nto a flaovr with a cyborg managed vgpu(accel:device-profile\u003dcyborg-vgpu-device-profile-name)\n\ncyborg devices currently can only be requested via the flavor.\n\nthat will change with the smart nic spec where they can be requested via neutron prot but todya\nthe only way to request one is form the flavor.\n\nfor upgrades you will just do a rolling upgrade with some hosts using cyborg and other using nova and move host one by one as you resize guests from one to the other.\n\nat least in the next 1-2 year i dont really expect that people using novas vgpus support will move to cyborg for it.\n\nthey likely will continue to use novas support if it works for them\n\nin the long term they might switch but i think resize is a valid way to do that.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecd19edf0908846a7120e71505ac378a68d97040","unresolved":false,"context_lines":[{"line_number":231,"context_line":"Upgrade impact"},{"line_number":232,"context_line":"--------------"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"None"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"Implementation"},{"line_number":237,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"5e0462a6_2f627698","line":234,"in_reply_to":"bb572732_60e5dc26","updated":"2020-12-17 13:38:40.000000000","message":"OK so a single resize could move the user from nova to cyborg regarding GPU support. I think that is good option that should be documented.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"ecb34aab02080d86aea56b8d0ea27222244cfcea","unresolved":false,"context_lines":[{"line_number":268,"context_line":"Testing"},{"line_number":269,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"* New unit test should be added"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Documentation Impact"},{"line_number":274,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_4a2a707e","line":271,"updated":"2020-11-17 12:47:57.000000000","message":"would there be a way to test this in the gate with the cyborg fake driver?","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e2c72fc3aaeeebe2595efe7a0b1a403deccd051c","unresolved":false,"context_lines":[{"line_number":268,"context_line":"Testing"},{"line_number":269,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"* New unit test should be added"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Documentation Impact"},{"line_number":274,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1f621f24_e0c27d52","line":271,"in_reply_to":"1f621f24_4a2a707e","updated":"2020-11-17 14:49:11.000000000","message":"there is a sample kernel vgpu mdev module called mdpy\n\nthat can be used to test this fully in the gate.\n\nthe mdpy moduel presents a very simple frambuffer to the guest but its enouch for very basic graphics supprot so you can boot vms with it. we cant use that with nova today unless i remove one assumation sylvain made that all mdevs are owned by pci devices.\n\nthis is something that cybrog should also not assuem becase it simple is not true.\nmdevs can be used with devices on other busses like usb too.\n\nso i would suggest using the mdpy module to test in the gate on both the nova and cyborg side. we have held of doing it before just because i have not had time to get back to this but i started writing a devstack plugin for this a while ago then got busy.\n\neven without that however we need more then unit tests we should have functional test for this.\n\nextending the fake driver to use the mdpy moduel would also be a good thing to do on the cyborg side in paralle.","commit_id":"a3a8f0b6ff261c0d5f6aa5f2f2bab536fcc4958b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"},{"line_number":27,"context_line":"composing cyborg owned vGPU into domain XML in nova libvirt driver."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For more information about cyborg side lifecycle management (discover, data"}],"source_content_type":"text/x-rst","patch_set":12,"id":"db90e034_59b13ec9","line":26,"range":{"start_line":25,"start_character":53,"end_line":26,"end_character":45},"updated":"2020-12-01 09:44:17.000000000","message":"well, I don\u0027t call it a gap, just a requested feature enhancement 😊","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":true,"context_lines":[{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"},{"line_number":27,"context_line":"composing cyborg owned vGPU into domain XML in nova libvirt driver."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For more information about cyborg side lifecycle management (discover, data"}],"source_content_type":"text/x-rst","patch_set":12,"id":"7c51cfb1_4687be2e","line":26,"range":{"start_line":25,"start_character":53,"end_line":26,"end_character":45},"in_reply_to":"db90e034_59b13ec9","updated":"2020-12-07 09:34:16.000000000","message":"cool.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3aed2c218cb88d8b966603d8a59a42e8c66d3c74","unresolved":true,"context_lines":[{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"},{"line_number":27,"context_line":"composing cyborg owned vGPU into domain XML in nova libvirt driver."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For more information about cyborg side lifecycle management (discover, data"}],"source_content_type":"text/x-rst","patch_set":12,"id":"403ac76f_8d73cc79","line":26,"range":{"start_line":25,"start_character":53,"end_line":26,"end_character":45},"in_reply_to":"db90e034_59b13ec9","updated":"2020-12-07 11:47:10.000000000","message":"sure. we can change it to a requested feature enhancement. This sounds more suitable from nova\u0027s perspective.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":49,"context_line":"       {"},{"line_number":50,"context_line":"           \u0027attach_handle_type\u0027: \u0027MDEV\u0027,"},{"line_number":51,"context_line":"           \u0027attach_handle_uuid\u0027: \u002791ac1606-427e-44bb-8233-f4ff4bf3d241\u0027,"},{"line_number":52,"context_line":"       }"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bd833f13_a55a9a66","line":52,"updated":"2020-12-01 09:44:17.000000000","message":"Ok, so Cyborg would precreate the mdev and then passing it to the nova-libvirt driver? LGTM if so.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":true,"context_lines":[{"line_number":49,"context_line":"       {"},{"line_number":50,"context_line":"           \u0027attach_handle_type\u0027: \u0027MDEV\u0027,"},{"line_number":51,"context_line":"           \u0027attach_handle_uuid\u0027: \u002791ac1606-427e-44bb-8233-f4ff4bf3d241\u0027,"},{"line_number":52,"context_line":"       }"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"f6a45437_0c2052b5","line":52,"in_reply_to":"bd833f13_a55a9a66","updated":"2020-12-07 09:34:16.000000000","message":"yep","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3aed2c218cb88d8b966603d8a59a42e8c66d3c74","unresolved":true,"context_lines":[{"line_number":49,"context_line":"       {"},{"line_number":50,"context_line":"           \u0027attach_handle_type\u0027: \u0027MDEV\u0027,"},{"line_number":51,"context_line":"           \u0027attach_handle_uuid\u0027: \u002791ac1606-427e-44bb-8233-f4ff4bf3d241\u0027,"},{"line_number":52,"context_line":"       }"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"8e870258_d97ceba1","line":52,"in_reply_to":"bd833f13_a55a9a66","updated":"2020-12-07 11:47:10.000000000","message":"yes, exactly.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":68,"context_line":"               dev.uuid \u003d arq[\u0027attach_handle_uuid\u0027]"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"               # add to guest XML"},{"line_number":71,"context_line":"               guest.add_device(dev)"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"3. Cyborg creates mdev device in the sys path"},{"line_number":74,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"8f05c4c0_d2e505ed","line":71,"updated":"2020-12-01 09:44:17.000000000","message":"IMHO, you don\u0027t need to add a new method, just using the same method that we have for vGPU support in libvirt.\nFirst, we have _guest_add_mdevs() here https://github.com/openstack/nova/blob/5ce25fe/nova/virt/libvirt/driver.py#L6258\n\nBut, surely, instead of calling it, why couldn\u0027t you just do like https://github.com/openstack/nova/blob/5ce25fe/nova/virt/libvirt/driver.py#L3658 and look at the ARQs there for passing the mdevs to https://github.com/openstack/nova/blob/5ce25fe/nova/virt/libvirt/driver.py#L3675 ?\n\nThe same logic would apply for all the other move operations, like for rescue in https://github.com/openstack/nova/blob/5ce25fe/nova/virt/libvirt/driver.py#L3598 or for cold_migrate/resize https://github.com/openstack/nova/blob/5ce25fe/nova/virt/libvirt/driver.py#L10485 ?","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3aed2c218cb88d8b966603d8a59a42e8c66d3c74","unresolved":true,"context_lines":[{"line_number":68,"context_line":"               dev.uuid \u003d arq[\u0027attach_handle_uuid\u0027]"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"               # add to guest XML"},{"line_number":71,"context_line":"               guest.add_device(dev)"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"3. Cyborg creates mdev device in the sys path"},{"line_number":74,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"740152d7_ac64178e","line":71,"in_reply_to":"8f05c4c0_d2e505ed","updated":"2020-12-07 11:47:10.000000000","message":"Thanks Sylvain for the good question.\n\nWhy didn\u0027t look at ARQs in L3658 and pass mdev to L3675? \nmy initial idea is to keep accel_info handling in one place https://github.com/openstack/nova/blob/5ce25fe/nova/virt/libvirt/driver.py#L6112, we can handle according to different arq[\u0027attach_handle_type\u0027](MDEV or PCI) there. this can have consistency consistency .\n\nWhy didn\u0027t reuse _guest_add_mdevs?\n_guest_add_mdevs(self, guest, chosen_mdevs) now only handles UUID(chosen_mdevs). acc_info would be a dict, if we plan to reuse it, we also need to add all the same change here to _guest_add_mdevs, which I actually didn\u0027t see any difference. What I can also see is a better readability, clarity and consistency benefit just like we already have in _guest_add_pci_devices vs _guest_add_accel_pci_devices.\n\nAnd I actually fine with either reuse _guest_add_mdevs or not. What do you think? All these changes will take place in libvirt driver, so your ideas really count.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":true,"context_lines":[{"line_number":68,"context_line":"               dev.uuid \u003d arq[\u0027attach_handle_uuid\u0027]"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"               # add to guest XML"},{"line_number":71,"context_line":"               guest.add_device(dev)"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"3. Cyborg creates mdev device in the sys path"},{"line_number":74,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"9a014d41_3b356861","line":71,"in_reply_to":"8f05c4c0_d2e505ed","updated":"2020-12-07 09:34:16.000000000","message":"good suggestion.\nwe can get mdevs from accel_info, and then pass mdevs to self._get_guest_xml.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":93,"context_line":"     - nova-compute receives bound notification and GET resolved arq"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"     - nova-compute calls virt driver to spawn an instance"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"   * reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":98,"context_line":"     API, cyborg-api firstly need to check if the target mdev already"},{"line_number":99,"context_line":"     exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"}],"source_content_type":"text/x-rst","patch_set":12,"id":"4ccc2694_d3f7a4d4","line":96,"updated":"2020-12-01 09:44:17.000000000","message":"see my comment above in L71","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":false,"context_lines":[{"line_number":93,"context_line":"     - nova-compute receives bound notification and GET resolved arq"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"     - nova-compute calls virt driver to spawn an instance"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"   * reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":98,"context_line":"     API, cyborg-api firstly need to check if the target mdev already"},{"line_number":99,"context_line":"     exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"}],"source_content_type":"text/x-rst","patch_set":12,"id":"2eac9715_43f7e88a","line":96,"in_reply_to":"4ccc2694_d3f7a4d4","updated":"2020-12-07 09:34:16.000000000","message":"Done","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":106,"context_line":""},{"line_number":107,"context_line":"     - This will need to add a new namepace in os-traits: OWNER_*."},{"line_number":108,"context_line":"       Then nova/cyborg report owner trait OWNER_NOVA/OWNER_CYBORG"},{"line_number":109,"context_line":"       as one of the trait when inventory is reported."},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"       + In the cyborg driver, it will report two traits for vGPU accelerator"},{"line_number":112,"context_line":"         using the format below. For more details, pls refer to the driver"}],"source_content_type":"text/x-rst","patch_set":12,"id":"1f6366da_4cebb081","line":109,"updated":"2020-12-01 09:44:17.000000000","message":"agreed, as we discussed at the PTG.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":118,"context_line":"           trait2: CUSTOM_\u003cVENDOR_NAME\u003e_\u003cPRODUCT_ID\u003e_\u003cVirtual_GPU_Type\u003e."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":121,"context_line":"         for vGPU resources during inventory report."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"     - When the end user makes a vGPU request, for a nova owned vGPU request,"},{"line_number":124,"context_line":"       one can specify ``resources##:VGPU\u003d1, trait##:OWNER_NOVA\u003drequired``"}],"source_content_type":"text/x-rst","patch_set":12,"id":"6ae843c8_860a0ea9","line":121,"updated":"2020-12-01 09:44:17.000000000","message":"this is fairly simple to do : https://github.com/openstack/nova/blob/5ce25fe/nova/virt/libvirt/driver.py#L7890\n\nWould you do it ?","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3aed2c218cb88d8b966603d8a59a42e8c66d3c74","unresolved":true,"context_lines":[{"line_number":118,"context_line":"           trait2: CUSTOM_\u003cVENDOR_NAME\u003e_\u003cPRODUCT_ID\u003e_\u003cVirtual_GPU_Type\u003e."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":121,"context_line":"         for vGPU resources during inventory report."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"     - When the end user makes a vGPU request, for a nova owned vGPU request,"},{"line_number":124,"context_line":"       one can specify ``resources##:VGPU\u003d1, trait##:OWNER_NOVA\u003drequired``"}],"source_content_type":"text/x-rst","patch_set":12,"id":"22f24c83_0ad2b05b","line":121,"in_reply_to":"6ae843c8_860a0ea9","updated":"2020-12-07 11:47:10.000000000","message":"we would like to do it if no others already take it.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":true,"context_lines":[{"line_number":118,"context_line":"           trait2: CUSTOM_\u003cVENDOR_NAME\u003e_\u003cPRODUCT_ID\u003e_\u003cVirtual_GPU_Type\u003e."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":121,"context_line":"         for vGPU resources during inventory report."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"     - When the end user makes a vGPU request, for a nova owned vGPU request,"},{"line_number":124,"context_line":"       one can specify ``resources##:VGPU\u003d1, trait##:OWNER_NOVA\u003drequired``"}],"source_content_type":"text/x-rst","patch_set":12,"id":"227bbfa0_86c14a61","line":121,"in_reply_to":"6ae843c8_860a0ea9","updated":"2020-12-07 09:34:16.000000000","message":"yes. we will do.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":120,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":121,"context_line":"         for vGPU resources during inventory report."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"     - When the end user makes a vGPU request, for a nova owned vGPU request,"},{"line_number":124,"context_line":"       one can specify ``resources##:VGPU\u003d1, trait##:OWNER_NOVA\u003drequired``"},{"line_number":125,"context_line":"       in flavor. For a cyborg owned vGPU, one need specify in pre-defined"},{"line_number":126,"context_line":"       device profile, then add it to flavor."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"}],"source_content_type":"text/x-rst","patch_set":12,"id":"1f7d6c21_681daafe","line":125,"range":{"start_line":123,"start_character":47,"end_line":125,"end_character":16},"updated":"2020-12-01 09:44:17.000000000","message":"I\u0027m absolutely against this, which will break existing deployments.\nPlease don\u0027t ask operators to modify their flavors.\n\nInstead, you should make sure that by default Nova will call Placement by adding a new trait request if we have an asked VGPU resource.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":true,"context_lines":[{"line_number":120,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":121,"context_line":"         for vGPU resources during inventory report."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"     - When the end user makes a vGPU request, for a nova owned vGPU request,"},{"line_number":124,"context_line":"       one can specify ``resources##:VGPU\u003d1, trait##:OWNER_NOVA\u003drequired``"},{"line_number":125,"context_line":"       in flavor. For a cyborg owned vGPU, one need specify in pre-defined"},{"line_number":126,"context_line":"       device profile, then add it to flavor."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"}],"source_content_type":"text/x-rst","patch_set":12,"id":"50ce4b42_10f2e329","line":125,"range":{"start_line":123,"start_character":47,"end_line":125,"end_character":16},"in_reply_to":"1f7d6c21_681daafe","updated":"2020-12-07 09:34:16.000000000","message":"IMHO, the trait of \u0027OWNER_NOVA\u0027 is necessary to query Placement, we can add it in the code by default.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"3aed2c218cb88d8b966603d8a59a42e8c66d3c74","unresolved":true,"context_lines":[{"line_number":120,"context_line":"       + In the nova side, it will need to report a new trait **OWNER_NOVA**"},{"line_number":121,"context_line":"         for vGPU resources during inventory report."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"     - When the end user makes a vGPU request, for a nova owned vGPU request,"},{"line_number":124,"context_line":"       one can specify ``resources##:VGPU\u003d1, trait##:OWNER_NOVA\u003drequired``"},{"line_number":125,"context_line":"       in flavor. For a cyborg owned vGPU, one need specify in pre-defined"},{"line_number":126,"context_line":"       device profile, then add it to flavor."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"}],"source_content_type":"text/x-rst","patch_set":12,"id":"19bcaca3_7d4b71c0","line":125,"range":{"start_line":123,"start_character":47,"end_line":125,"end_character":16},"in_reply_to":"1f7d6c21_681daafe","updated":"2020-12-07 11:47:10.000000000","message":"So in this way, the OWNER_* trait would be just an internal tag for scheduler, the operator will not feel it, and will not break existing deployment. \n\nSo there will be a small change in api part.if VGPU request is from flavor with ``resources:VGPU\u003d1``, then we need to add the trait:OWNER_NOVA to request_spec, otherwise if is from flavor with \"accel:device_profile \u003d device_profile_name\", then add trait:OWNER_CYBORG to request_spec. Do I understand it correctly?","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"04ad2b08aa83592bdc177a46f207d5f8af247304","unresolved":true,"context_lines":[{"line_number":201,"context_line":""},{"line_number":202,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":203,"context_line":"both Nova and Cyborg. If it is configured in both cyborg and nova should be"},{"line_number":204,"context_line":"able to raise this as an conflict exception."},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"Developer impact"},{"line_number":207,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":12,"id":"90fa6c86_003a3e1c","line":204,"updated":"2020-12-01 09:44:17.000000000","message":"How ? Please detail this.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":31412,"name":"Wenping Song","email":"songwenping@inspur.com","username":"songwenping"},"change_message_id":"da9169bdaba0338b63a63816a0a79a4485fa7ed0","unresolved":true,"context_lines":[{"line_number":201,"context_line":""},{"line_number":202,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":203,"context_line":"both Nova and Cyborg. If it is configured in both cyborg and nova should be"},{"line_number":204,"context_line":"able to raise this as an conflict exception."},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"Developer impact"},{"line_number":207,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":12,"id":"69022b67_f3146017","line":204,"in_reply_to":"90fa6c86_003a3e1c","updated":"2020-12-07 09:34:16.000000000","message":"If the deployer configure same device in both Cyborg and Nova, they may report same data to Placement at the same time, we can raise conflict exception at Placement side, and return to Cyborg or Nova, and warn the deployer about the conflict device.","commit_id":"8cbc87c67b178bb80f4bb2b2675766b3dd35e777"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"efac8d5faf4dc7e52630738fdfb84cbf80599924","unresolved":true,"context_lines":[{"line_number":96,"context_line":"     exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"},{"line_number":97,"context_line":"     to create a new mdev, otherwise just return an expected arq."},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"   * host reboot: cyborg-owned vGPU will be mssing from the sys path after"},{"line_number":100,"context_line":"     host reboot. From cyborg\u0027s side, cyborg-agent automaticaly creates the"},{"line_number":101,"context_line":"     mdevs on start up. From nova\u0027s perspective, when the compute host starts"},{"line_number":102,"context_line":"     up it will try to get accel_info and reboot all the instances. But if"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7bfbad55_57afa275","line":99,"range":{"start_line":99,"start_character":44,"end_line":99,"end_character":50},"updated":"2021-01-07 10:05:40.000000000","message":"nit: missing","commit_id":"d8ba9d866e65e11eaaf6c73997fc0791f68acc65"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"efac8d5faf4dc7e52630738fdfb84cbf80599924","unresolved":true,"context_lines":[{"line_number":99,"context_line":"   * host reboot: cyborg-owned vGPU will be mssing from the sys path after"},{"line_number":100,"context_line":"     host reboot. From cyborg\u0027s side, cyborg-agent automaticaly creates the"},{"line_number":101,"context_line":"     mdevs on start up. From nova\u0027s perspective, when the compute host starts"},{"line_number":102,"context_line":"     up it will try to get accel_info and reboot all the instances. But if"},{"line_number":103,"context_line":"     cyborg is not available at that time when the log an error and skip the"},{"line_number":104,"context_line":"     instance that would need cyborg."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"4. Avoid conflicts when cyborg vGPU management co-exists with nova"},{"line_number":107,"context_line":"   vGPU management."}],"source_content_type":"text/x-rst","patch_set":14,"id":"fe0671cd_6792bf3c","line":104,"range":{"start_line":102,"start_character":69,"end_line":104,"end_character":37},"updated":"2021-01-07 10:05:40.000000000","message":"+1 looks good to me","commit_id":"d8ba9d866e65e11eaaf6c73997fc0791f68acc65"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"28c4bcbef0111561ad4674628908f84671d1e841","unresolved":true,"context_lines":[{"line_number":222,"context_line":""},{"line_number":223,"context_line":"Nova management vGPU:"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"Advantage: without the need to hack a new component."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":228,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"}],"source_content_type":"text/x-rst","patch_set":14,"id":"e3396a86_5a152bc4","line":225,"range":{"start_line":225,"start_character":11,"end_line":225,"end_character":52},"updated":"2020-12-17 15:22:46.000000000","message":"this is not only \"hacking\" a new component. This literrally means *supporting* a whole new set of RPC and HTTP services with their own versions, compatibilities and dependencies. For distributions offering downstream products, that also means testing and providing deployment tooling with resiliency mechanisms into their products with a lifecycle of around 5 years for *every* release they\u0027re going to provide.\n\nThis isn\u0027t just a matter of splitting code for the sake of microservices love, and debating over the possibility to remove Nova code seems VERY premature to me, compared to the state of Cyborg at the moment.","commit_id":"d8ba9d866e65e11eaaf6c73997fc0791f68acc65"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"d04999daa9f4f9452481cc8d47bf5524b2db5e47","unresolved":true,"context_lines":[{"line_number":222,"context_line":""},{"line_number":223,"context_line":"Nova management vGPU:"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"Advantage: without the need to hack a new component."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":228,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"}],"source_content_type":"text/x-rst","patch_set":14,"id":"f6f285fc_9f77657a","line":225,"range":{"start_line":225,"start_character":11,"end_line":225,"end_character":52},"in_reply_to":"4e9ae3b5_e7994cf7","updated":"2020-12-18 11:10:58.000000000","message":"ACK. Thanks gibi for the mention in meeting.\nI also want to check, can we continue review and merge the feature implementation of this spec if we cannot reach an agreement before spec freeze or need more discussion in next PTG on the deprecation problem? As far as I see, these two things are indenpendent, because users who use cyborg need this spec. **eventually** deprecation should be another thing. what do you think?","commit_id":"d8ba9d866e65e11eaaf6c73997fc0791f68acc65"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e38c8db42749d822ee7b648cdd41b3625f5acbaa","unresolved":true,"context_lines":[{"line_number":222,"context_line":""},{"line_number":223,"context_line":"Nova management vGPU:"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"Advantage: without the need to hack a new component."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":228,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"}],"source_content_type":"text/x-rst","patch_set":14,"id":"4e9ae3b5_e7994cf7","line":225,"range":{"start_line":225,"start_character":11,"end_line":225,"end_character":52},"in_reply_to":"e3396a86_5a152bc4","updated":"2020-12-17 16:39:53.000000000","message":"We discussed parts of this on the weekly meeting http://eavesdrop.openstack.org/meetings/nova/2020/nova.2020-12-17-16.00.log.html#l-75","commit_id":"d8ba9d866e65e11eaaf6c73997fc0791f68acc65"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"efac8d5faf4dc7e52630738fdfb84cbf80599924","unresolved":true,"context_lines":[{"line_number":222,"context_line":""},{"line_number":223,"context_line":"Nova management vGPU:"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"Advantage: without the need to hack a new component."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":228,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"}],"source_content_type":"text/x-rst","patch_set":14,"id":"da9a3bd4_f58a8614","line":225,"range":{"start_line":225,"start_character":11,"end_line":225,"end_character":52},"in_reply_to":"f6f285fc_9f77657a","updated":"2021-01-07 10:05:40.000000000","message":"My viewpoint here is that deprecation is something we have to think about and discuss it as without that we will keep two competing implementation potentially for ever. But it should not block the progress on the implementation of this.\n\nAbout the wording, I agree with Sylvain to be a bit more specific. Something like\n\n  Advantage: No need to integrate and support a new OpenStack projec","commit_id":"d8ba9d866e65e11eaaf6c73997fc0791f68acc65"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"efac8d5faf4dc7e52630738fdfb84cbf80599924","unresolved":true,"context_lines":[{"line_number":243,"context_line":"For those who want to upgrade from nova-owned vGPU to cyborg-owned vGPU,"},{"line_number":244,"context_line":"one can resize directly from a flavor with a nova managed gpu"},{"line_number":245,"context_line":"(resouces:vgpu\u003d1 in the flavor) to a flaovr with a cyborg managed vgpu"},{"line_number":246,"context_line":"(accel:device-profile\u003dcyborg-vgpu-device-profile-name)."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"Implementation"},{"line_number":249,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":14,"id":"b5351434_45530dfe","line":246,"updated":"2021-01-07 10:05:40.000000000","message":"+1 cool","commit_id":"d8ba9d866e65e11eaaf6c73997fc0791f68acc65"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"d04999daa9f4f9452481cc8d47bf5524b2db5e47","unresolved":true,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"   This will need to get mdevs form arq list and pass it to generate guest xml"},{"line_number":57,"context_line":"   in nova/virt/libvirt/driver.py. Please be aware that the following is just"},{"line_number":58,"context_line":"   the pseudocode to show the whole function and process."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"   ::"}],"source_content_type":"text/x-rst","patch_set":15,"id":"f7c13a70_14e0bc18","line":57,"range":{"start_line":56,"start_character":2,"end_line":57,"end_character":34},"updated":"2020-12-18 11:10:58.000000000","message":"can you mention that code will be added around https://github.com/openstack/nova/blob/master/nova/virt/libvirt/driver.py#L6139?\nthis might be faster for others to check.","commit_id":"169e674a64ec9542b124632ef5ae2e75a39b344c"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"e6e6bb8ed645a3dd20343a18083bf265a684fffd","unresolved":true,"context_lines":[{"line_number":228,"context_line":""},{"line_number":229,"context_line":"Nova management vGPU:"},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"Advantage: without the need to hack a new component."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":234,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"}],"source_content_type":"text/x-rst","patch_set":15,"id":"ac388b54_2f5585b9","line":231,"updated":"2020-12-18 09:24:43.000000000","message":"Restating my main concerns from PS14  : \nthis is not only \"hacking\" a new component. This literrally means *supporting* a whole new set of RPC and HTTP services with their own versions, compatibilities and dependencies. For distributions offering downstream products, that also means testing and providing deployment tooling with resiliency mechanisms into their products with a lifecycle of around 5 years for *every* release they\u0027re going to provide.\n\nThis isn\u0027t just a matter of splitting code for the sake of microservices love, and debating over the possibility to remove Nova code seems VERY premature to me, compared to the state of Cyborg at the moment.","commit_id":"169e674a64ec9542b124632ef5ae2e75a39b344c"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"555a6741e72bd9cdfb221eddf0e3566710180525","unresolved":true,"context_lines":[{"line_number":71,"context_line":"               # get attach_handle_uuid"},{"line_number":72,"context_line":"               dev.uuid \u003d arq[\u0027attach_handle_uuid\u0027]"},{"line_number":73,"context_line":"               # add to guest XML"},{"line_number":74,"context_line":"               guest.add_device(dev)"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3. Cyborg creates mdev device in the sys path"},{"line_number":77,"context_line":""}],"source_content_type":"text/x-rst","patch_set":16,"id":"73ea67f4_a8678d5e","line":74,"updated":"2021-01-08 16:23:34.000000000","message":"to be honest, we already attach mdevs to the guest by https://github.com/openstack/nova/blob/46899968619e4ea0ff2ab380977619bb29578d43/nova/virt/libvirt/driver.py#L6281 so you should just reuse it by passing arq[\u0027attach_handle_uuid\u0027] value as a parameter.","commit_id":"9e61d89dab13750d4d77e998823323b58368f401"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":71,"context_line":"               # get attach_handle_uuid"},{"line_number":72,"context_line":"               dev.uuid \u003d arq[\u0027attach_handle_uuid\u0027]"},{"line_number":73,"context_line":"               # add to guest XML"},{"line_number":74,"context_line":"               guest.add_device(dev)"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3. Cyborg creates mdev device in the sys path"},{"line_number":77,"context_line":""}],"source_content_type":"text/x-rst","patch_set":16,"id":"fe38d5ae_5004fba1","line":74,"in_reply_to":"73ea67f4_a8678d5e","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"9e61d89dab13750d4d77e998823323b58368f401"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"555a6741e72bd9cdfb221eddf0e3566710180525","unresolved":true,"context_lines":[{"line_number":209,"context_line":"Other deployer impact"},{"line_number":210,"context_line":"---------------------"},{"line_number":211,"context_line":""},{"line_number":212,"context_line":"Before discussing deployment, we might need to think do we need to keep both"},{"line_number":213,"context_line":"or deprecate Nova\u0027s GPU handling code in favor of cyborg GPU handling in the"},{"line_number":214,"context_line":"future? Here I will only list simple advantages for two sides and leave for"},{"line_number":215,"context_line":"cores to review and comments."},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"Cyborg management vGPU:"},{"line_number":218,"context_line":""}],"source_content_type":"text/x-rst","patch_set":16,"id":"9bd1dd00_3b43f0db","line":215,"range":{"start_line":212,"start_character":0,"end_line":215,"end_character":29},"updated":"2021-01-08 16:23:34.000000000","message":"the whole section about nova vGPU support deprecation shouldn\u0027t be done in thi spec, and honestly waiting for more than 3 or 4 cycles before we discuss it. It\u0027s not about a competition question, it\u0027s more about the fact that Cyborg is not producted for a lot of operators at the moment, so asking them to move is difficult.","commit_id":"9e61d89dab13750d4d77e998823323b58368f401"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":true,"context_lines":[{"line_number":209,"context_line":"Other deployer impact"},{"line_number":210,"context_line":"---------------------"},{"line_number":211,"context_line":""},{"line_number":212,"context_line":"Before discussing deployment, we might need to think do we need to keep both"},{"line_number":213,"context_line":"or deprecate Nova\u0027s GPU handling code in favor of cyborg GPU handling in the"},{"line_number":214,"context_line":"future? Here I will only list simple advantages for two sides and leave for"},{"line_number":215,"context_line":"cores to review and comments."},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"Cyborg management vGPU:"},{"line_number":218,"context_line":""}],"source_content_type":"text/x-rst","patch_set":16,"id":"d5d2e46d_58abeb8a","line":215,"range":{"start_line":212,"start_character":0,"end_line":215,"end_character":29},"in_reply_to":"9bd1dd00_3b43f0db","updated":"2021-01-18 11:07:48.000000000","message":"ok. Agree that we\u0027d better talk several cycles later. This is also good for Cyborg to get more feedbacks from operators who use it and become more mature.","commit_id":"9e61d89dab13750d4d77e998823323b58368f401"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c2692bf64e0f70ded89c96acede20cfc46ad046a","unresolved":true,"context_lines":[{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"In order to allow operators to use cyborg to manage the lifecycle of vGPU,"},{"line_number":20,"context_line":"cyborg needs to discover the vGPUs, reports them to placement, and nova virt"},{"line_number":21,"context_line":"driver helps to allocate this cyborg-owned vGPU to an instance. Please be"},{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"a8c3e2db_c14cadce","line":21,"range":{"start_line":20,"start_character":63,"end_line":21,"end_character":62},"updated":"2021-01-14 22:25:19.000000000","message":"and instruct nova to allocate a specific vgpu mdev to the instance.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"In order to allow operators to use cyborg to manage the lifecycle of vGPU,"},{"line_number":20,"context_line":"cyborg needs to discover the vGPUs, reports them to placement, and nova virt"},{"line_number":21,"context_line":"driver helps to allocate this cyborg-owned vGPU to an instance. Please be"},{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"2326ffbf_788a19a5","line":21,"range":{"start_line":20,"start_character":63,"end_line":21,"end_character":62},"in_reply_to":"a8c3e2db_c14cadce","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"d3b66d429894c13f7b7a8ddce4e06a85d74b40a0","unresolved":true,"context_lines":[{"line_number":19,"context_line":"In order to allow operators to use cyborg to manage the lifecycle of vGPU,"},{"line_number":20,"context_line":"cyborg needs to discover the vGPUs, reports them to placement, and nova virt"},{"line_number":21,"context_line":"driver helps to allocate this cyborg-owned vGPU to an instance. Please be"},{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"}],"source_content_type":"text/x-rst","patch_set":17,"id":"477e16c6_fbaaec6d","line":22,"range":{"start_line":22,"start_character":62,"end_line":22,"end_character":67},"updated":"2021-01-18 07:48:01.000000000","message":"this is misodered. it should be [1] here.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":19,"context_line":"In order to allow operators to use cyborg to manage the lifecycle of vGPU,"},{"line_number":20,"context_line":"cyborg needs to discover the vGPUs, reports them to placement, and nova virt"},{"line_number":21,"context_line":"driver helps to allocate this cyborg-owned vGPU to an instance. Please be"},{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"}],"source_content_type":"text/x-rst","patch_set":17,"id":"7578bf37_a1833870","line":22,"range":{"start_line":22,"start_character":62,"end_line":22,"end_character":67},"in_reply_to":"477e16c6_fbaaec6d","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c2692bf64e0f70ded89c96acede20cfc46ad046a","unresolved":true,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"In order to allow operators to use cyborg to manage the lifecycle of vGPU,"},{"line_number":20,"context_line":"cyborg needs to discover the vGPUs, reports them to placement, and nova virt"},{"line_number":21,"context_line":"driver helps to allocate this cyborg-owned vGPU to an instance. Please be"},{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"}],"source_content_type":"text/x-rst","patch_set":17,"id":"2a1745f0_a662e5bc","line":23,"range":{"start_line":21,"start_character":64,"end_line":23,"end_character":21},"updated":"2021-01-14 22:25:19.000000000","message":"Cyborg managed vGPUs will not replace nova\u0027s native vGPU capabilities [2]_ and will\nprovide an alternative management mechanism in parallel to the existing nova feature.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"In order to allow operators to use cyborg to manage the lifecycle of vGPU,"},{"line_number":20,"context_line":"cyborg needs to discover the vGPUs, reports them to placement, and nova virt"},{"line_number":21,"context_line":"driver helps to allocate this cyborg-owned vGPU to an instance. Please be"},{"line_number":22,"context_line":"informed that this is another way co-exists with the nova way [2]_ of vGPU"},{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"}],"source_content_type":"text/x-rst","patch_set":17,"id":"b2654b07_5a9bd9fa","line":23,"range":{"start_line":21,"start_character":64,"end_line":23,"end_character":21},"in_reply_to":"2a1745f0_a662e5bc","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"d3b66d429894c13f7b7a8ddce4e06a85d74b40a0","unresolved":true,"context_lines":[{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"},{"line_number":27,"context_line":"composing cyborg owned vGPU into domain XML in nova libvirt driver."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For more information about cyborg side lifecycle management (discover, data"},{"line_number":30,"context_line":"modeling etc.) of vGPU please refer to [3]_."}],"source_content_type":"text/x-rst","patch_set":17,"id":"7ca2a506_59bfdecd","line":27,"range":{"start_line":26,"start_character":46,"end_line":27,"end_character":67},"updated":"2021-01-18 07:48:01.000000000","message":"In my understanding, this spec will cover 2 parts:\n1.claim and allocate vgpu resource for vm.\n2. generate vgpu xmls section in libvirt driver.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":true,"context_lines":[{"line_number":23,"context_line":"lifecycle management."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [1]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"},{"line_number":27,"context_line":"composing cyborg owned vGPU into domain XML in nova libvirt driver."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For more information about cyborg side lifecycle management (discover, data"},{"line_number":30,"context_line":"modeling etc.) of vGPU please refer to [3]_."}],"source_content_type":"text/x-rst","patch_set":17,"id":"dc8d1d10_02ef06b7","line":27,"range":{"start_line":26,"start_character":46,"end_line":27,"end_character":67},"in_reply_to":"7ca2a506_59bfdecd","updated":"2021-01-18 11:07:48.000000000","message":"emmm in my understanding, the first part is already implemented in current cyborg-nova interaction [1]_, this spec is going to point out the gap and proposed changes. do you think we still need to repeat first part again here?","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"d3b66d429894c13f7b7a8ddce4e06a85d74b40a0","unresolved":true,"context_lines":[{"line_number":32,"context_line":"Use Cases"},{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As an operator, I want to use Cyborg to manage lifecycle of vGPUs."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"Proposed change"},{"line_number":38,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":17,"id":"6f8e2bdb_69b68f18","line":35,"range":{"start_line":35,"start_character":47,"end_line":35,"end_character":57},"updated":"2021-01-18 07:48:01.000000000","message":"the lifecycle","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":32,"context_line":"Use Cases"},{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As an operator, I want to use Cyborg to manage lifecycle of vGPUs."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"Proposed change"},{"line_number":38,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":17,"id":"9f350ed3_0e7e1b22","line":35,"range":{"start_line":35,"start_character":47,"end_line":35,"end_character":57},"in_reply_to":"6f8e2bdb_69b68f18","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"d3b66d429894c13f7b7a8ddce4e06a85d74b40a0","unresolved":true,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"       {"},{"line_number":50,"context_line":"           \u0027attach_handle_type\u0027: \u0027MDEV\u0027,"},{"line_number":51,"context_line":"           \u0027attach_handle_uuid\u0027: \u002791ac1606-427e-44bb-8233-f4ff4bf3d241\u0027,"},{"line_number":52,"context_line":"       }"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."}],"source_content_type":"text/x-rst","patch_set":17,"id":"9c4e6a24_e25cb84c","line":51,"range":{"start_line":51,"start_character":34,"end_line":51,"end_character":70},"updated":"2021-01-18 07:48:01.000000000","message":"who generate this uuid.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":true,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"       {"},{"line_number":50,"context_line":"           \u0027attach_handle_type\u0027: \u0027MDEV\u0027,"},{"line_number":51,"context_line":"           \u0027attach_handle_uuid\u0027: \u002791ac1606-427e-44bb-8233-f4ff4bf3d241\u0027,"},{"line_number":52,"context_line":"       }"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"2. Nova virt driver merge the mdev info from arq into the XML of an instance."}],"source_content_type":"text/x-rst","patch_set":17,"id":"b54d59e5_cf39721f","line":51,"range":{"start_line":51,"start_character":34,"end_line":51,"end_character":70},"in_reply_to":"9c4e6a24_e25cb84c","updated":"2021-01-18 11:07:48.000000000","message":"this uuid is generated by the cyborg\u0027s gpu driver when driver objects are reported.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"d3b66d429894c13f7b7a8ddce4e06a85d74b40a0","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"     - cyborg-api rpc.call the cyborg-agent to create a new mdev"},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"     - cyborg-agent succefully created mdev and returns result"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"     - cyborg start bind arq"},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"611f72d7_807ff81b","line":83,"range":{"start_line":83,"start_character":20,"end_line":83,"end_character":31},"updated":"2021-01-18 07:48:01.000000000","message":"sucessfully","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"     - cyborg-api rpc.call the cyborg-agent to create a new mdev"},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"     - cyborg-agent succefully created mdev and returns result"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"     - cyborg start bind arq"},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"77bbbc3e_60c44151","line":83,"range":{"start_line":83,"start_character":20,"end_line":83,"end_character":31},"in_reply_to":"611f72d7_807ff81b","updated":"2021-01-18 11:07:48.000000000","message":"good catch.:)","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"d3b66d429894c13f7b7a8ddce4e06a85d74b40a0","unresolved":true,"context_lines":[{"line_number":77,"context_line":"   * spawn instance interaction(arq interaction part):"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"     - nova-conductor requests arq create and bind"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"     - cyborg-api rpc.call the cyborg-agent to create a new mdev"},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"     - cyborg-agent succefully created mdev and returns result"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"     - cyborg start bind arq"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"     - cyborg arq successfully bound and notify nova that ARQ bindings"},{"line_number":88,"context_line":"       are resolved for a given instance"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"     - nova-compute receives bound notification and GET resolved arq"},{"line_number":91,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"a35ebc39_fc010a93","line":88,"range":{"start_line":80,"start_character":0,"end_line":88,"end_character":40},"updated":"2021-01-18 07:48:01.000000000","message":"This four steps is the sub-step of the first one. please re-struct the format to clarify that.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":77,"context_line":"   * spawn instance interaction(arq interaction part):"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"     - nova-conductor requests arq create and bind"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"     - cyborg-api rpc.call the cyborg-agent to create a new mdev"},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"     - cyborg-agent succefully created mdev and returns result"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"     - cyborg start bind arq"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"     - cyborg arq successfully bound and notify nova that ARQ bindings"},{"line_number":88,"context_line":"       are resolved for a given instance"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"     - nova-compute receives bound notification and GET resolved arq"},{"line_number":91,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"161ffe2d_42527357","line":88,"range":{"start_line":80,"start_character":0,"end_line":88,"end_character":40},"in_reply_to":"a35ebc39_fc010a93","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c2692bf64e0f70ded89c96acede20cfc46ad046a","unresolved":true,"context_lines":[{"line_number":91,"context_line":""},{"line_number":92,"context_line":"     - nova-compute calls virt driver to spawn an instance"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"   * reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":95,"context_line":"     API, cyborg-api firstly need to check if the target mdev already"},{"line_number":96,"context_line":"     exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"},{"line_number":97,"context_line":"     to create a new mdev, otherwise just return an expected arq."},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"   * host reboot: cyborg-owned vGPU will be missing from the sys path after"},{"line_number":100,"context_line":"     host reboot. From cyborg\u0027s side, cyborg-agent automatically creates the"}],"source_content_type":"text/x-rst","patch_set":17,"id":"ec21473d_b86fcd95","line":97,"range":{"start_line":94,"start_character":2,"end_line":97,"end_character":65},"updated":"2021-01-14 22:25:19.000000000","message":"i still think that making a GET arq API request have a side effect of \nprovisiouning a device is a mistake.\n\nthe get should just be a db lookup with no rpc ot the agent.\nif the agenet has not heartbeat back in a given interval the arq state\nshoudl be set to unknow. on start up the cybrog agent should recate the mdev\nif we need to force it to set it up again we shoudl call bind.\n\nas i noted in older versions fo this spec is we make GET have this sideffect then \nteh cyborg api can never support the readonly stardard roles that keystone has\nbeen trying to enable for some time. the reason for this is a get to the arq endpoint\nwould actully be a write action in some cases.\n\nalso form an api design this is not a good apporch since get to the arq api shoudl be short\nblocking call but if you have to do an rpc to the agent and wait that short operation becomes\na poteially long operation which means it really should be async like the bind arq api call.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":true,"context_lines":[{"line_number":91,"context_line":""},{"line_number":92,"context_line":"     - nova-compute calls virt driver to spawn an instance"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"   * reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":95,"context_line":"     API, cyborg-api firstly need to check if the target mdev already"},{"line_number":96,"context_line":"     exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"},{"line_number":97,"context_line":"     to create a new mdev, otherwise just return an expected arq."},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"   * host reboot: cyborg-owned vGPU will be missing from the sys path after"},{"line_number":100,"context_line":"     host reboot. From cyborg\u0027s side, cyborg-agent automatically creates the"}],"source_content_type":"text/x-rst","patch_set":17,"id":"f145ba3b_290753e7","line":97,"range":{"start_line":94,"start_character":2,"end_line":97,"end_character":65},"in_reply_to":"838ec0fe_03c66b53","updated":"2021-01-18 11:07:48.000000000","message":"Thanks sean for your suggestion and the discussion!\nAgree and updated.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"6f36c6ab5c9fca6287b81a98db96a337747850a7","unresolved":true,"context_lines":[{"line_number":91,"context_line":""},{"line_number":92,"context_line":"     - nova-compute calls virt driver to spawn an instance"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"   * reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":95,"context_line":"     API, cyborg-api firstly need to check if the target mdev already"},{"line_number":96,"context_line":"     exists, if not exists, cyborg-api need to rpc.call the cyborg-agent"},{"line_number":97,"context_line":"     to create a new mdev, otherwise just return an expected arq."},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"   * host reboot: cyborg-owned vGPU will be missing from the sys path after"},{"line_number":100,"context_line":"     host reboot. From cyborg\u0027s side, cyborg-agent automatically creates the"}],"source_content_type":"text/x-rst","patch_set":17,"id":"838ec0fe_03c66b53","line":97,"range":{"start_line":94,"start_character":2,"end_line":97,"end_character":65},"in_reply_to":"ec21473d_b86fcd95","updated":"2021-01-15 13:01:20.000000000","message":"i spoke to gibi about this breifly on irce starting at\nhttp://eavesdrop.openstack.org/irclogs/%23openstack-nova/latest.log.html#t2021-01-15T12:36:00\nand ending at 12:57\n\nthe logs are not quite up to date as i post this but the remaining few minutes should be avaiable soon.\n\nhopefully that is more clear.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c2692bf64e0f70ded89c96acede20cfc46ad046a","unresolved":true,"context_lines":[{"line_number":129,"context_line":"       vGPU, one need specify in pre-defined device profile, then add it to"},{"line_number":130,"context_line":"       flavor."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"},{"line_number":133,"context_line":"       by the traits. No new change is required."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"   * In libvirt driver, use a mdev_tag to identify which vGPU is the current"},{"line_number":136,"context_line":"     operation request(eg. spawn, reboot) going to land. If it is cyborg vGPU,"}],"source_content_type":"text/x-rst","patch_set":17,"id":"eb940b87_b00b84a6","line":133,"range":{"start_line":132,"start_character":4,"end_line":133,"end_character":48},"updated":"2021-01-14 22:25:19.000000000","message":"remove this or just say placement will use the generated traits to select a host\nwith a gpu provided by cyborg or nova based on the request.\n\nthe filtering on traits is done by placement not by the nova scheduler.\nby the time the request get to the scheduler we have a set of allocation candidates\ncontain allocations based on the resource and traits requests.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":129,"context_line":"       vGPU, one need specify in pre-defined device profile, then add it to"},{"line_number":130,"context_line":"       flavor."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"},{"line_number":133,"context_line":"       by the traits. No new change is required."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"   * In libvirt driver, use a mdev_tag to identify which vGPU is the current"},{"line_number":136,"context_line":"     operation request(eg. spawn, reboot) going to land. If it is cyborg vGPU,"}],"source_content_type":"text/x-rst","patch_set":17,"id":"195a716c_89cc3a55","line":133,"range":{"start_line":132,"start_character":4,"end_line":133,"end_character":48},"in_reply_to":"eb940b87_b00b84a6","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"d3b66d429894c13f7b7a8ddce4e06a85d74b40a0","unresolved":true,"context_lines":[{"line_number":132,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"},{"line_number":133,"context_line":"       by the traits. No new change is required."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"   * In libvirt driver, use a mdev_tag to identify which vGPU is the current"},{"line_number":136,"context_line":"     operation request(eg. spawn, reboot) going to land. If it is cyborg vGPU,"},{"line_number":137,"context_line":"     tag will be \u0027ACCELERATOR\u0027, otherwise it is \u0027COMPUTE\u0027."},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"0d505872_9dda1051","line":135,"range":{"start_line":135,"start_character":5,"end_line":135,"end_character":62},"updated":"2021-01-18 07:48:01.000000000","message":"why we need this tag. In libvirt driver, no matter the mdev comes from cyborg or nova, it will be treat the same.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":true,"context_lines":[{"line_number":132,"context_line":"     - When nova-scheduler receives a vGPU request, it just need to filter"},{"line_number":133,"context_line":"       by the traits. No new change is required."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"   * In libvirt driver, use a mdev_tag to identify which vGPU is the current"},{"line_number":136,"context_line":"     operation request(eg. spawn, reboot) going to land. If it is cyborg vGPU,"},{"line_number":137,"context_line":"     tag will be \u0027ACCELERATOR\u0027, otherwise it is \u0027COMPUTE\u0027."},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"973bc247_ff4f140a","line":135,"range":{"start_line":135,"start_character":5,"end_line":135,"end_character":62},"in_reply_to":"0d505872_9dda1051","updated":"2021-01-18 11:07:48.000000000","message":"no, mdevs comes from different owner will be treated differently.\nfor example, in the spawn() process, nova-owned vGPU will be handled at https://github.com/openstack/nova/blob/master/nova/virt/libvirt/driver.py#L3664, while cyborg-owned vGPU cannot reuse this function because: use this func cannot help us find a cyborg-owned _allocate_mdevs. instead cyborg will pass the _allocate_mdevs in accel_info.\nSo we should handle cyborg-owned vGPU at here https://github.com/openstack/nova/blob/5b65812ef2c6ac46bea30e2e603284eac232339e/nova/virt/libvirt/driver.py#L6135.\nThat\u0027s why we need this tag to identy the mdev source.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c2692bf64e0f70ded89c96acede20cfc46ad046a","unresolved":true,"context_lines":[{"line_number":203,"context_line":"Other deployer impact"},{"line_number":204,"context_line":"---------------------"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"Before discussing deployment, I will list simple advantages for two sides."},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"Cyborg management vGPU:"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"Advantages: The unified interface for accelerators."},{"line_number":211,"context_line":""},{"line_number":212,"context_line":"The prominent benefit I can see is for the cloud maintainer(or the deployer),"},{"line_number":213,"context_line":"which also reflects the goal of Cyborg. For the maintainer, using Cyborg, it"},{"line_number":214,"context_line":"will have an unified interface to manage the devices."},{"line_number":215,"context_line":"e.g. System admin can list all devices by one command \"openstack accelerator"},{"line_number":216,"context_line":"device list\", and see accelerator resource usage for instances(if applicable)"},{"line_number":217,"context_line":"by \"openstack accelerator arq list\", or calling program usage API by"},{"line_number":218,"context_line":"\"openstack accelerator deployable \u003cdeployable_uuid\u003e \u003cimage_uuid\u003e\" etc."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"Nova management vGPU:"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Advantage: No need to integrate and support a new OpenStack project."},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":225,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"}],"source_content_type":"text/x-rst","patch_set":17,"id":"0fe797b6_97a52d35","line":222,"range":{"start_line":206,"start_character":2,"end_line":222,"end_character":68},"updated":"2021-01-14 22:25:19.000000000","message":"i dont actully think cyborg prvodes and advantage over what nova support today and i dont think this\nreally is useful for the deployer to read so i would remove this and jsut leave the note below about ensuring device are not configured in both.","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":203,"context_line":"Other deployer impact"},{"line_number":204,"context_line":"---------------------"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"Before discussing deployment, I will list simple advantages for two sides."},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"Cyborg management vGPU:"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"Advantages: The unified interface for accelerators."},{"line_number":211,"context_line":""},{"line_number":212,"context_line":"The prominent benefit I can see is for the cloud maintainer(or the deployer),"},{"line_number":213,"context_line":"which also reflects the goal of Cyborg. For the maintainer, using Cyborg, it"},{"line_number":214,"context_line":"will have an unified interface to manage the devices."},{"line_number":215,"context_line":"e.g. System admin can list all devices by one command \"openstack accelerator"},{"line_number":216,"context_line":"device list\", and see accelerator resource usage for instances(if applicable)"},{"line_number":217,"context_line":"by \"openstack accelerator arq list\", or calling program usage API by"},{"line_number":218,"context_line":"\"openstack accelerator deployable \u003cdeployable_uuid\u003e \u003cimage_uuid\u003e\" etc."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"Nova management vGPU:"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"Advantage: No need to integrate and support a new OpenStack project."},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":225,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"}],"source_content_type":"text/x-rst","patch_set":17,"id":"78bb862e_7b2c0914","line":222,"range":{"start_line":206,"start_character":2,"end_line":222,"end_character":68},"in_reply_to":"0fe797b6_97a52d35","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"c2692bf64e0f70ded89c96acede20cfc46ad046a","unresolved":true,"context_lines":[{"line_number":239,"context_line":""},{"line_number":240,"context_line":"For those who want to upgrade from nova-owned vGPU to cyborg-owned vGPU,"},{"line_number":241,"context_line":"one can resize directly from a flavor with a nova managed gpu"},{"line_number":242,"context_line":"(resouces:vgpu\u003d1 in the flavor) to a flaovr with a cyborg managed vgpu"},{"line_number":243,"context_line":"(accel:device-profile\u003dcyborg-vgpu-device-profile-name)."},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":17,"id":"63907bf8_5fc5c1c0","line":242,"range":{"start_line":242,"start_character":37,"end_line":242,"end_character":43},"updated":"2021-01-14 22:25:19.000000000","message":"flavor","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"92a99aae5467aa3846ed5c81fccdcc17a681e0aa","unresolved":false,"context_lines":[{"line_number":239,"context_line":""},{"line_number":240,"context_line":"For those who want to upgrade from nova-owned vGPU to cyborg-owned vGPU,"},{"line_number":241,"context_line":"one can resize directly from a flavor with a nova managed gpu"},{"line_number":242,"context_line":"(resouces:vgpu\u003d1 in the flavor) to a flaovr with a cyborg managed vgpu"},{"line_number":243,"context_line":"(accel:device-profile\u003dcyborg-vgpu-device-profile-name)."},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":17,"id":"cff82a94_c389320d","line":242,"range":{"start_line":242,"start_character":37,"end_line":242,"end_character":43},"in_reply_to":"63907bf8_5fc5c1c0","updated":"2021-01-18 11:07:48.000000000","message":"Done","commit_id":"f3d0331ad56d81d68f07da33e14b4699b548672a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":20,"context_line":"cyborg needs to discover the vGPUs, reports them to placement, and instruct"},{"line_number":21,"context_line":"nova to allocate a specific vgpu mdev to the instance. Cyborg managed vGPUs"},{"line_number":22,"context_line":"will not replace nova\u0027s native vGPU capabilities [1]_ and will provide an"},{"line_number":23,"context_line":"alternative management mechanism in parallel to the existing nova feature."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"In the current cyborg-nova interaction [2]_, there is still a small gap in"},{"line_number":26,"context_line":"allocating a cyborg owned vGPU to an instance. This spec proposes to support"}],"source_content_type":"text/x-rst","patch_set":18,"id":"dc305a52_6f53f185","line":23,"updated":"2021-01-19 09:17:17.000000000","message":"++, thanks for it.","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":74,"context_line":"           \"\"\"Adding mdev accelerators from ARQ list."},{"line_number":75,"context_line":"           \"\"\""},{"line_number":76,"context_line":"           for arq in acc_info:"},{"line_number":77,"context_line":"               self._guest_add_mdevs(guest, arq[\u0027attach_handle_uuid\u0027])"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"3. Cyborg creates mdev device in the sys path"},{"line_number":80,"context_line":""}],"source_content_type":"text/x-rst","patch_set":18,"id":"1feb39ba_f9a0e4e4","line":77,"updated":"2021-01-19 09:17:17.000000000","message":"let\u0027s just review this by the implementation, as I still have questions for the above but I\u0027d prefer to discuss this within the implementation changes","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"   In theory, if nova and cyborg both can support vGPU management, nova or"},{"line_number":82,"context_line":"   cyborg should support create mdev in its own respective. Therefore, in the"},{"line_number":83,"context_line":"   cyborg lifecycle management of vGPU, cyborg should creates mdev beforehead,"},{"line_number":84,"context_line":"   nova virt driver uses it as an existed resousrce."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"   * spawn instance interaction(arq interaction part):"}],"source_content_type":"text/x-rst","patch_set":18,"id":"efc5d3e9_c09e8031","line":83,"range":{"start_line":83,"start_character":67,"end_line":83,"end_character":77},"updated":"2021-01-19 09:17:17.000000000","message":"nit: beforehand","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"   In theory, if nova and cyborg both can support vGPU management, nova or"},{"line_number":82,"context_line":"   cyborg should support create mdev in its own respective. Therefore, in the"},{"line_number":83,"context_line":"   cyborg lifecycle management of vGPU, cyborg should creates mdev beforehead,"},{"line_number":84,"context_line":"   nova virt driver uses it as an existed resousrce."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"   * spawn instance interaction(arq interaction part):"}],"source_content_type":"text/x-rst","patch_set":18,"id":"2559b234_a9a4d75c","line":83,"range":{"start_line":83,"start_character":62,"end_line":83,"end_character":66},"updated":"2021-01-19 09:17:17.000000000","message":"nit: mediated devices","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":89,"context_line":""},{"line_number":90,"context_line":"       + cyborg-api rpc.call the cyborg-agent to create a new mdev"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"       + cyborg-agent sucessfully created mdev and returns result"},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"       + cyborg start bind arq"},{"line_number":95,"context_line":""}],"source_content_type":"text/x-rst","patch_set":18,"id":"f424aca7_1a24bf3c","line":92,"range":{"start_line":92,"start_character":34,"end_line":92,"end_character":41},"updated":"2021-01-19 09:17:17.000000000","message":"nit: you use the present tense for all the other verbs, so it should be \"creates\"","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":98,"context_line":""},{"line_number":99,"context_line":"     - nova-compute receives bound notification and GET resolved arq"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"     - nova-compute calls virt driver to spawn an instance"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"   * reboot instance interaction: when nova-compute calls cyborg GET arq"},{"line_number":104,"context_line":"     API, cyborg does the db lookup and returns the expected arq."}],"source_content_type":"text/x-rst","patch_set":18,"id":"76d8110a_18caeebe","line":101,"updated":"2021-01-19 09:17:17.000000000","message":"I still have concerns about the above and how this would work, but let\u0027s discuss this by the implementation.","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"3fe19fbc5c792365b63802b2b52daa6c99a1523f","unresolved":true,"context_lines":[{"line_number":106,"context_line":"   * host reboot: cyborg-owned vGPU will be missing from the sys path after"},{"line_number":107,"context_line":"     host reboot. The cyborg agent should create the mdevs for all bound arqs"},{"line_number":108,"context_line":"     on start up. Ideally it would set the binding state to \"provisioning\" on"},{"line_number":109,"context_line":"     start up also. And we would have anohter state \"unknown\" that cyborg-api"},{"line_number":110,"context_line":"     would report if the cyborg agent misses its heart beat."},{"line_number":111,"context_line":"     So on start up, when nova compute tries to get accel_info and reboot all"},{"line_number":112,"context_line":"     the instances, it would see 1 of 3 states, \"bound\" if the cyborg agent"}],"source_content_type":"text/x-rst","patch_set":18,"id":"40a72f06_60a8cf5d","line":109,"range":{"start_line":109,"start_character":38,"end_line":109,"end_character":45},"updated":"2021-01-18 14:05:47.000000000","message":"nit: another","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":109,"context_line":"     start up also. And we would have anohter state \"unknown\" that cyborg-api"},{"line_number":110,"context_line":"     would report if the cyborg agent misses its heart beat."},{"line_number":111,"context_line":"     So on start up, when nova compute tries to get accel_info and reboot all"},{"line_number":112,"context_line":"     the instances, it would see 1 of 3 states, \"bound\" if the cyborg agent"},{"line_number":113,"context_line":"     started first and already completed bininding, \"unknown\" if the cyborg"},{"line_number":114,"context_line":"     agent has not heartbeat to the cyborg conductor yet, and \"provisioning\""},{"line_number":115,"context_line":"     if the agent is in the process of creating the mdev. Cyborg will send the"}],"source_content_type":"text/x-rst","patch_set":18,"id":"4741c254_898e1a05","line":112,"range":{"start_line":112,"start_character":20,"end_line":112,"end_character":32},"updated":"2021-01-19 09:17:17.000000000","message":"you mean that the compute service would call the Cyborg API and get one of the three states, right ?","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"94195f4330d9c3cc5d6fc38ef581205970006a67","unresolved":true,"context_lines":[{"line_number":109,"context_line":"     start up also. And we would have anohter state \"unknown\" that cyborg-api"},{"line_number":110,"context_line":"     would report if the cyborg agent misses its heart beat."},{"line_number":111,"context_line":"     So on start up, when nova compute tries to get accel_info and reboot all"},{"line_number":112,"context_line":"     the instances, it would see 1 of 3 states, \"bound\" if the cyborg agent"},{"line_number":113,"context_line":"     started first and already completed bininding, \"unknown\" if the cyborg"},{"line_number":114,"context_line":"     agent has not heartbeat to the cyborg conductor yet, and \"provisioning\""},{"line_number":115,"context_line":"     if the agent is in the process of creating the mdev. Cyborg will send the"}],"source_content_type":"text/x-rst","patch_set":18,"id":"d98575b8_8374f172","line":112,"range":{"start_line":112,"start_character":20,"end_line":112,"end_character":32},"in_reply_to":"4741c254_898e1a05","updated":"2021-01-28 02:44:01.000000000","message":"yes,exactly","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":114,"context_line":"     agent has not heartbeat to the cyborg conductor yet, and \"provisioning\""},{"line_number":115,"context_line":"     if the agent is in the process of creating the mdev. Cyborg will send the"},{"line_number":116,"context_line":"     same binding complete event notification when it change the status form"},{"line_number":117,"context_line":"     \"provisioning\" to \"bound\" as it does during normal arq binding."},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"4. Avoid conflicts when cyborg vGPU management co-exists with nova"},{"line_number":120,"context_line":"   vGPU management."}],"source_content_type":"text/x-rst","patch_set":18,"id":"5d1b2d6f_e0f7d09f","line":117,"updated":"2021-01-19 09:17:17.000000000","message":"I have some concerns about the differences between the 3 state supports and what the libvirt driver should do for each of them, but let\u0027s discuss this during the implementation.","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":24872,"name":"YumengBao","email":"yumeng_bao@yahoo.com","username":"Yumeng_Bao"},"change_message_id":"94195f4330d9c3cc5d6fc38ef581205970006a67","unresolved":true,"context_lines":[{"line_number":114,"context_line":"     agent has not heartbeat to the cyborg conductor yet, and \"provisioning\""},{"line_number":115,"context_line":"     if the agent is in the process of creating the mdev. Cyborg will send the"},{"line_number":116,"context_line":"     same binding complete event notification when it change the status form"},{"line_number":117,"context_line":"     \"provisioning\" to \"bound\" as it does during normal arq binding."},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"4. Avoid conflicts when cyborg vGPU management co-exists with nova"},{"line_number":120,"context_line":"   vGPU management."}],"source_content_type":"text/x-rst","patch_set":18,"id":"f92649c6_8d1bed03","line":117,"in_reply_to":"5d1b2d6f_e0f7d09f","updated":"2021-01-28 02:44:01.000000000","message":"ok","commit_id":"48e51689709262e2a60d572381ac18262365317a"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7ee8036661354d55503b1302bc8ade87fcbc90f4","unresolved":true,"context_lines":[{"line_number":212,"context_line":""},{"line_number":213,"context_line":"Other deployer impact"},{"line_number":214,"context_line":"---------------------"},{"line_number":215,"context_line":""},{"line_number":216,"context_line":"Deployer should make sure the device in one pci address is not configured in"},{"line_number":217,"context_line":"both Nova and Cyborg. If it is configured in both Cyborg and Nova should be"},{"line_number":218,"context_line":"able to raise this as an conflict exception."}],"source_content_type":"text/x-rst","patch_set":18,"id":"e8843bf2_a4daf631","line":215,"updated":"2021-01-19 09:17:17.000000000","message":"thanks for removing the section I was having huge concerns.","commit_id":"48e51689709262e2a60d572381ac18262365317a"}]}