)]}'
{"doc/source/specs/rocky/approved/compute-node.rst":[{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are left attached."},{"line_number":134,"context_line":"  However, when it is shelved-offloaded, the accelerar resources are"},{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."},{"line_number":137,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_f369c2f6","line":134,"range":{"start_line":134,"start_character":45,"end_line":134,"end_character":54},"updated":"2018-07-27 21:36:42.000000000","message":"accelerator","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are left attached."},{"line_number":134,"context_line":"  However, when it is shelved-offloaded, the accelerar resources are"},{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."},{"line_number":137,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_e2920f57","line":134,"range":{"start_line":134,"start_character":45,"end_line":134,"end_character":54},"in_reply_to":"5f7c97a3_f369c2f6","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":17813,"name":"wangzhh","email":"wzh_1993@126.com","username":"wangzhh"},"change_message_id":"4cf72cae2c9037d29788faf3352bfc058e4be011","unresolved":false,"context_lines":[{"line_number":165,"context_line":"potential to cause problems. The operator may use RBAC to limit such"},{"line_number":166,"context_line":"operations."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"* Reboot: The accelerator resources are left intact. However, the DMA"},{"line_number":169,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":170,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":171,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_9520c5b3","line":168,"updated":"2018-06-29 17:21:53.000000000","message":"Nova redefine the domain when an instance reboot, and the devices info are gotten from nova db, maybe we should record acceletators info in nova?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":165,"context_line":"potential to cause problems. The operator may use RBAC to limit such"},{"line_number":166,"context_line":"operations."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"* Reboot: The accelerator resources are left intact. However, the DMA"},{"line_number":169,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":170,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":171,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_da97d7ea","line":168,"in_reply_to":"5f7c97a3_3c699025","updated":"2018-07-06 02:23:20.000000000","message":"Not sure I follow this. When you run \u0027openstack server reboot\u0027 CLI, the instance UUID does not change AFAIK and the accelerators are not detached. Why does anything need to change in any db? \n\nWhen you say, \u0027Nova redefines the domain\u0027, do you mean that the domain XML is regenerated (for libvirt)?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"d56dea2295b66c7c1e325335f37a1cabbb823125","unresolved":false,"context_lines":[{"line_number":165,"context_line":"potential to cause problems. The operator may use RBAC to limit such"},{"line_number":166,"context_line":"operations."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"* Reboot: The accelerator resources are left intact. However, the DMA"},{"line_number":169,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":170,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":171,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c6e52b40","line":168,"in_reply_to":"5f7c97a3_9520c5b3","updated":"2018-06-30 02:18:24.000000000","message":"I don\u0027t think recording the acc in Nova is a good idea tho. We can just track them in Cyborg db.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":17813,"name":"wangzhh","email":"wzh_1993@126.com","username":"wangzhh"},"change_message_id":"b7e72c2283cea49fc1ca54e28265dc2b97674700","unresolved":false,"context_lines":[{"line_number":165,"context_line":"potential to cause problems. The operator may use RBAC to limit such"},{"line_number":166,"context_line":"operations."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"* Reboot: The accelerator resources are left intact. However, the DMA"},{"line_number":169,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":170,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":171,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_3c699025","line":168,"in_reply_to":"5f7c97a3_c6e52b40","updated":"2018-06-30 10:45:17.000000000","message":"It\u0027s an option. But do we need query cyborg if the instance didn\u0027t attach accelerator?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":165,"context_line":"potential to cause problems. The operator may use RBAC to limit such"},{"line_number":166,"context_line":"operations."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"* Reboot: The accelerator resources are left intact. However, the DMA"},{"line_number":169,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":170,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":171,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_428d0330","line":168,"in_reply_to":"5f7c97a3_cafb78bc","updated":"2018-08-14 17:14:37.000000000","message":"It is proposed to store instance-VAN associations in Nova, while the VANs themselves are stored in Cyborg. So, Nova can retrieve all needed VANs during any instance operation.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":17813,"name":"wangzhh","email":"wzh_1993@126.com","username":"wangzhh"},"change_message_id":"c6e3a33e0690cbbc967e442029ebfeb52a8a611b","unresolved":false,"context_lines":[{"line_number":165,"context_line":"potential to cause problems. The operator may use RBAC to limit such"},{"line_number":166,"context_line":"operations."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"* Reboot: The accelerator resources are left intact. However, the DMA"},{"line_number":169,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":170,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":171,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_cafb78bc","line":168,"in_reply_to":"5f7c97a3_da97d7ea","updated":"2018-07-06 15:20:19.000000000","message":"Please run:\n1.openstack server stop instance_uuid\n2.openstack server start instance_uuid\nor run:\nopenstack server reboot instance_uuid --hard\nHere is the code about this issue:\nhttps://github.com/openstack/nova/blob/0e090f31e845a3fc683260f529b9ac3fddbcb6f8/nova/virt/libvirt/driver.py#L2738","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"d56dea2295b66c7c1e325335f37a1cabbb823125","unresolved":false,"context_lines":[{"line_number":176,"context_line":"  in a particular state. When booting off a previous snapshot, the"},{"line_number":177,"context_line":"  current configuration and state of accelerators may not match the"},{"line_number":178,"context_line":"  snapshot. The instance can reset the accelerator(s) to a clean state"},{"line_number":179,"context_line":"  after the boot, but that requires trusting the instance."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"The following instance operations are not supported in this release:"},{"line_number":182,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_46d91b82","line":179,"range":{"start_line":179,"start_character":49,"end_line":179,"end_character":57},"updated":"2018-06-30 02:18:24.000000000","message":"I believe we can let Cyborg to the resetting instead let instance do it.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":176,"context_line":"  in a particular state. When booting off a previous snapshot, the"},{"line_number":177,"context_line":"  current configuration and state of accelerators may not match the"},{"line_number":178,"context_line":"  snapshot. The instance can reset the accelerator(s) to a clean state"},{"line_number":179,"context_line":"  after the boot, but that requires trusting the instance."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"The following instance operations are not supported in this release:"},{"line_number":182,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_ba60db0e","line":179,"range":{"start_line":179,"start_character":49,"end_line":179,"end_character":57},"in_reply_to":"5f7c97a3_46d91b82","updated":"2018-07-06 02:23:20.000000000","message":"This is probably implementation-dependent. Once you have attached a VF to VM and mapped its MMIO registers, accessing those same registers from the PF (for Cyborg to do reset) may be prevented by the implementation for safety reasons. For example, if Cyborg does a reset and the VM happens to be accessing the registers at that time, bad things can happen.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":176,"context_line":"  in a particular state. When booting off a previous snapshot, the"},{"line_number":177,"context_line":"  current configuration and state of accelerators may not match the"},{"line_number":178,"context_line":"  snapshot. The instance can reset the accelerator(s) to a clean state"},{"line_number":179,"context_line":"  after the boot, but that requires trusting the instance."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"The following instance operations are not supported in this release:"},{"line_number":182,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_a2a657aa","line":179,"range":{"start_line":179,"start_character":49,"end_line":179,"end_character":57},"in_reply_to":"5f7c97a3_b306ea20","updated":"2018-08-14 17:14:37.000000000","message":"This is an issue even without OpenStack (or any orchestration like K8s). It should be handled by the hypervisor and driver, as explained in patchset 9.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":176,"context_line":"  in a particular state. When booting off a previous snapshot, the"},{"line_number":177,"context_line":"  current configuration and state of accelerators may not match the"},{"line_number":178,"context_line":"  snapshot. The instance can reset the accelerator(s) to a clean state"},{"line_number":179,"context_line":"  after the boot, but that requires trusting the instance."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"The following instance operations are not supported in this release:"},{"line_number":182,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_b306ea20","line":179,"range":{"start_line":179,"start_character":49,"end_line":179,"end_character":57},"in_reply_to":"5f7c97a3_ba60db0e","updated":"2018-07-27 21:36:42.000000000","message":"This is really the same issue we have with e.g. RAM state not being saved in a \"normal\" snapshot: You can\u0027t (and shouldn\u0027t) try to preserve the accelerator state.\n\nHowever, you must make sure that any running state of an accelerator is cleared when you attach, so you don\u0027t wind up with one instance being able to see private data from a previous instance that was using that accelerator.\n\nThis should be done by cyborg or nova, not left up to the instance.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"d56dea2295b66c7c1e325335f37a1cabbb823125","unresolved":false,"context_lines":[{"line_number":190,"context_line":"--------"},{"line_number":191,"context_line":"Nova Compute (n-cpu) on the compute node shall interact with Cyborg and"},{"line_number":192,"context_line":"otherwise handle accelerators through a new library named os-acc. This"},{"line_number":193,"context_line":"library handles the attach and detach of accelerators to instances,"},{"line_number":194,"context_line":"somewhat similar to the way that os-vif handles Neutron ports."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"The accelerator - instance attachment has two parts:"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_26a3bf45","line":193,"range":{"start_line":193,"start_character":54,"end_line":193,"end_character":56},"updated":"2018-06-30 02:18:24.000000000","message":"to/from","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":190,"context_line":"--------"},{"line_number":191,"context_line":"Nova Compute (n-cpu) on the compute node shall interact with Cyborg and"},{"line_number":192,"context_line":"otherwise handle accelerators through a new library named os-acc. This"},{"line_number":193,"context_line":"library handles the attach and detach of accelerators to instances,"},{"line_number":194,"context_line":"somewhat similar to the way that os-vif handles Neutron ports."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"The accelerator - instance attachment has two parts:"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_62beffb4","line":193,"range":{"start_line":193,"start_character":54,"end_line":193,"end_character":56},"in_reply_to":"5f7c97a3_1aa22f40","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":190,"context_line":"--------"},{"line_number":191,"context_line":"Nova Compute (n-cpu) on the compute node shall interact with Cyborg and"},{"line_number":192,"context_line":"otherwise handle accelerators through a new library named os-acc. This"},{"line_number":193,"context_line":"library handles the attach and detach of accelerators to instances,"},{"line_number":194,"context_line":"somewhat similar to the way that os-vif handles Neutron ports."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"The accelerator - instance attachment has two parts:"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_1aa22f40","line":193,"range":{"start_line":193,"start_character":54,"end_line":193,"end_character":56},"in_reply_to":"5f7c97a3_26a3bf45","updated":"2018-07-06 02:23:20.000000000","message":"ok","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":205,"context_line":"accelerators, and that translates into differences in the workflow"},{"line_number":206,"context_line":"and in the design of os-acc vs. os-vif:"},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":209,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":210,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":211,"context_line":"* To create the Device Half of the attach, the device may need to be"},{"line_number":212,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":213,"context_line":"  function in the hardware, or reprogramming the FPGA. Whereas there is"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_d2a68555","line":210,"range":{"start_line":208,"start_character":2,"end_line":210,"end_character":55},"updated":"2018-06-28 00:46:16.000000000","message":"When I first read this, I was ready to start objecting, because it sounds like you\u0027re saying \"accelerators don\u0027t need to be configured\".  But then you address that (sort of) in the next bullet.\n\nAccelerators *do* need to be configured, but *don\u0027t* need configuration that\u0027s *unique to each accelerator instance* like a MAC/IP is for a port.  At least not today.\n\nI\u0027m still not convinced that that means there\u0027s no analog to the port binding step.  I would think this would be the stage at which, for example, we carve off a VGPU of the appropriate size, or configure an SR-IOV VF with appropriate QoS parameters.  Was your plan for all of that to happen during the attach phase?\n\n[Later] Looks like getVAN() may in fact be the `bind` analog.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":205,"context_line":"accelerators, and that translates into differences in the workflow"},{"line_number":206,"context_line":"and in the design of os-acc vs. os-vif:"},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":209,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":210,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":211,"context_line":"* To create the Device Half of the attach, the device may need to be"},{"line_number":212,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":213,"context_line":"  function in the hardware, or reprogramming the FPGA. Whereas there is"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_0697da47","line":210,"range":{"start_line":208,"start_character":2,"end_line":210,"end_character":55},"in_reply_to":"5f7c97a3_d2a68555","updated":"2018-06-28 06:12:44.000000000","message":"\u003e Accelerators *do* need to be configured, but *don\u0027t* need\n \u003e configuration that\u0027s *unique to each accelerator instance* like a\n \u003e MAC/IP is for a port.  At least not today.\n\nExactly.\n\n \u003e Was your plan for all of that to happen during the attach phase?\n\nYes, as part of getVAN().\n\n \u003e [Later] Looks like getVAN() may in fact be the `bind` analog.\n\nYup.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":211,"context_line":"* To create the Device Half of the attach, the device may need to be"},{"line_number":212,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":213,"context_line":"  function in the hardware, or reprogramming the FPGA. Whereas there is"},{"line_number":214,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":215,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":216,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":217,"context_line":"  any, for the Instance Half of the attach."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_f653f013","line":214,"range":{"start_line":214,"start_character":2,"end_line":214,"end_character":40},"updated":"2018-07-27 21:36:42.000000000","message":"even for things like QoS?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":211,"context_line":"* To create the Device Half of the attach, the device may need to be"},{"line_number":212,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":213,"context_line":"  function in the hardware, or reprogramming the FPGA. Whereas there is"},{"line_number":214,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":215,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":216,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":217,"context_line":"  any, for the Instance Half of the attach."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_82c47b41","line":214,"range":{"start_line":214,"start_character":2,"end_line":214,"end_character":40},"in_reply_to":"5f7c97a3_f653f013","updated":"2018-08-14 17:14:37.000000000","message":"Not sure how QoS works in Neutron today. Even if it involves configuring the NIC, does it work the same way as we have proposed here?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":213,"context_line":"  function in the hardware, or reprogramming the FPGA. Whereas there is"},{"line_number":214,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":215,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":216,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":217,"context_line":"  any, for the Instance Half of the attach."},{"line_number":218,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_2d90b896","line":215,"range":{"start_line":215,"start_character":7,"end_line":215,"end_character":18},"updated":"2018-06-28 00:46:16.000000000","message":"what does this mean?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":213,"context_line":"  function in the hardware, or reprogramming the FPGA. Whereas there is"},{"line_number":214,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":215,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":216,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":217,"context_line":"  any, for the Instance Half of the attach."},{"line_number":218,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c68fa220","line":215,"range":{"start_line":215,"start_character":7,"end_line":215,"end_character":18},"in_reply_to":"5f7c97a3_2d90b896","updated":"2018-06-28 06:12:44.000000000","message":"PCI whitelists etc. are in Nova. Inserting PCI devices into a VM\u0027s definition -- e.g. domain XML in libvirt -- is done by the Nova virt driver.\n\nI am not saying that PCI is the only way to attach accelerators; only that, if PCI is involved, Nova (and its virt drivers) do most of the work of handling PCI.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":213,"context_line":"  function in the hardware, or reprogramming the FPGA. Whereas there is"},{"line_number":214,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":215,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":216,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":217,"context_line":"  any, for the Instance Half of the attach."},{"line_number":218,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_ff3253c0","line":215,"range":{"start_line":215,"start_character":7,"end_line":215,"end_character":18},"in_reply_to":"5f7c97a3_c68fa220","updated":"2018-06-28 18:33:16.000000000","message":"One of my hopes for this whole effort is that we can get rid of the existing nova PCI subsystem entirely, as opposed to trying to reroute those code paths through cyborg flows.\n\nSaying \"Cyborg handles it\" is a step in the right direction there.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":221,"context_line":"* Create a Virtual Accelerator Nexus (VAN), analogous to a VIF, which is"},{"line_number":222,"context_line":"  a handle object for attaching an accelerator to an instance (see"},{"line_number":223,"context_line":"  `VANs and plugins`_)."},{"line_number":224,"context_line":"* Create the Device Half of the attach:"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"  * Choose an accelerator in the chosen device RP (resource provider) if"},{"line_number":227,"context_line":"    there is more than one."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_cde1042e","line":224,"range":{"start_line":224,"start_character":1,"end_line":224,"end_character":39},"updated":"2018-06-28 00:46:16.000000000","message":"right, so this would be the `bind` part...","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":221,"context_line":"* Create a Virtual Accelerator Nexus (VAN), analogous to a VIF, which is"},{"line_number":222,"context_line":"  a handle object for attaching an accelerator to an instance (see"},{"line_number":223,"context_line":"  `VANs and plugins`_)."},{"line_number":224,"context_line":"* Create the Device Half of the attach:"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"  * Choose an accelerator in the chosen device RP (resource provider) if"},{"line_number":227,"context_line":"    there is more than one."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_8699aa6e","line":224,"range":{"start_line":224,"start_character":1,"end_line":224,"end_character":39},"in_reply_to":"5f7c97a3_cde1042e","updated":"2018-06-28 06:12:44.000000000","message":"Yes","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":223,"context_line":"  `VANs and plugins`_)."},{"line_number":224,"context_line":"* Create the Device Half of the attach:"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"  * Choose an accelerator in the chosen device RP (resource provider) if"},{"line_number":227,"context_line":"    there is more than one."},{"line_number":228,"context_line":"  * If needed, create a host-visible attach handle, such as a mediated"},{"line_number":229,"context_line":"    device UUID."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_8de46c44","line":226,"range":{"start_line":226,"start_character":4,"end_line":226,"end_character":10},"updated":"2018-06-28 00:46:16.000000000","message":"Or in fact carve off.  They may not exist (or be able to exist) before we get here, because we wouldn\u0027t know how to configure them.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":223,"context_line":"  `VANs and plugins`_)."},{"line_number":224,"context_line":"* Create the Device Half of the attach:"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"  * Choose an accelerator in the chosen device RP (resource provider) if"},{"line_number":227,"context_line":"    there is more than one."},{"line_number":228,"context_line":"  * If needed, create a host-visible attach handle, such as a mediated"},{"line_number":229,"context_line":"    device UUID."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c6b802c9","line":226,"range":{"start_line":226,"start_character":4,"end_line":226,"end_character":10},"in_reply_to":"5f7c97a3_8de46c44","updated":"2018-06-28 06:12:44.000000000","message":"Sure. I\u0027ll say \u0027choose or create\u0027.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":223,"context_line":"  `VANs and plugins`_)."},{"line_number":224,"context_line":"* Create the Device Half of the attach:"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"  * Choose an accelerator in the chosen device RP (resource provider) if"},{"line_number":227,"context_line":"    there is more than one."},{"line_number":228,"context_line":"  * If needed, create a host-visible attach handle, such as a mediated"},{"line_number":229,"context_line":"    device UUID."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_22e9e7b5","line":226,"range":{"start_line":226,"start_character":4,"end_line":226,"end_character":10},"in_reply_to":"5f7c97a3_c6b802c9","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"d56dea2295b66c7c1e325335f37a1cabbb823125","unresolved":false,"context_lines":[{"line_number":227,"context_line":"    there is more than one."},{"line_number":228,"context_line":"  * If needed, create a host-visible attach handle, such as a mediated"},{"line_number":229,"context_line":"    device UUID."},{"line_number":230,"context_line":"  * If needed, configure the device hardware to associate the chosen"},{"line_number":231,"context_line":"    accelerator with a host-visible attach handle (PCI function, mediated"},{"line_number":232,"context_line":"    device, etc.)"},{"line_number":233,"context_line":"  * Update the VAN to record the accelerator and attach handle association."},{"line_number":234,"context_line":"  * Persist the VAN object and the instance-VAN association."},{"line_number":235,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_e64b8783","line":232,"range":{"start_line":230,"start_character":4,"end_line":232,"end_character":17},"updated":"2018-06-30 02:18:24.000000000","message":"May this involve FPGA reprogramming if needed?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":227,"context_line":"    there is more than one."},{"line_number":228,"context_line":"  * If needed, create a host-visible attach handle, such as a mediated"},{"line_number":229,"context_line":"    device UUID."},{"line_number":230,"context_line":"  * If needed, configure the device hardware to associate the chosen"},{"line_number":231,"context_line":"    accelerator with a host-visible attach handle (PCI function, mediated"},{"line_number":232,"context_line":"    device, etc.)"},{"line_number":233,"context_line":"  * Update the VAN to record the accelerator and attach handle association."},{"line_number":234,"context_line":"  * Persist the VAN object and the instance-VAN association."},{"line_number":235,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_da65971e","line":232,"range":{"start_line":230,"start_character":4,"end_line":232,"end_character":17},"in_reply_to":"5f7c97a3_e64b8783","updated":"2018-07-06 02:23:20.000000000","message":"Yes. The detailed workflow that comes later gets into that.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":233,"context_line":"  * Update the VAN to record the accelerator and attach handle association."},{"line_number":234,"context_line":"  * Persist the VAN object and the instance-VAN association."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"* Create the Instance Half of the attach:"},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"  * Use the VAN information in a hypervisor-specific way to attach the"},{"line_number":239,"context_line":"    accelerator to the instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_edde4069","line":236,"range":{"start_line":236,"start_character":2,"end_line":236,"end_character":41},"updated":"2018-06-28 00:46:16.000000000","message":"...and this would be the `plug` part.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":233,"context_line":"  * Update the VAN to record the accelerator and attach handle association."},{"line_number":234,"context_line":"  * Persist the VAN object and the instance-VAN association."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"* Create the Instance Half of the attach:"},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"  * Use the VAN information in a hypervisor-specific way to attach the"},{"line_number":239,"context_line":"    accelerator to the instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_06d35a03","line":236,"range":{"start_line":236,"start_character":2,"end_line":236,"end_character":41},"in_reply_to":"5f7c97a3_edde4069","updated":"2018-06-28 06:12:44.000000000","message":"Yes","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":259,"context_line":"However, the os-vif plugins deal with a small set of entities to which"},{"line_number":260,"context_line":"Neutron ports connect: Linux bridge, OVS, SR-IOV etc. Whereas os-acc"},{"line_number":261,"context_line":"plugins are expected to handle a much wider range of variation among"},{"line_number":262,"context_line":"devices. So, instead of having the Nova virt driver choose a plugin as"},{"line_number":263,"context_line":"with os-vif, os-acc will let the operator specify a set of plugins in a"},{"line_number":264,"context_line":"configuration file (``setup.cfg``) and will load them through stevedore"},{"line_number":265,"context_line":"(similar to Neutron controller plugins). Out-of-tree plugins can be"},{"line_number":266,"context_line":"enabled in this way."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"Workflow"},{"line_number":269,"context_line":"--------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_cd72644b","line":266,"range":{"start_line":262,"start_character":9,"end_line":266,"end_character":20},"updated":"2018-06-28 00:46:16.000000000","message":"I\u0027m not sure how you\u0027re expecting the os-acc setup to be different.  I\u0027m no expert, but my understanding is that:\n\n- the operator installs/enables whatever set of plugin packages they desire\n- those plugins self-register on initialization so they\u0027re discoverable\n- when we need to do a thing XYZ that requires a plugin, we walk through the list of registered plugins asking, \"Can you do XYZ?\" and pick the first one that says \"yes\".  In VIF-land, I believe that question looks like, \"Can you handle VIF type such-and-such?\"  In acc-land, I would expect the question to be \"Can you handle mechanism X on accelerator of type Y from vendor Z?\"  (I wouldn\u0027t think we would need the question to encompass arch/hypervisor, because the op should only be installing plugins that work where they\u0027re being installed.)\n\nDoes that gel with your expectation of the os-acc flow?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":259,"context_line":"However, the os-vif plugins deal with a small set of entities to which"},{"line_number":260,"context_line":"Neutron ports connect: Linux bridge, OVS, SR-IOV etc. Whereas os-acc"},{"line_number":261,"context_line":"plugins are expected to handle a much wider range of variation among"},{"line_number":262,"context_line":"devices. So, instead of having the Nova virt driver choose a plugin as"},{"line_number":263,"context_line":"with os-vif, os-acc will let the operator specify a set of plugins in a"},{"line_number":264,"context_line":"configuration file (``setup.cfg``) and will load them through stevedore"},{"line_number":265,"context_line":"(similar to Neutron controller plugins). Out-of-tree plugins can be"},{"line_number":266,"context_line":"enabled in this way."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"Workflow"},{"line_number":269,"context_line":"--------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_981d7064","line":266,"range":{"start_line":262,"start_character":9,"end_line":266,"end_character":20},"in_reply_to":"5f7c97a3_5fa5879e","updated":"2018-06-29 02:23:10.000000000","message":"os-vif gets away with this approach because the #plugins are few and Nova virt drivers know about them.. For os-acc, there could possibly be a profusion of plugins, and virt drivers will have no clue about them, so we don\u0027t want virt drivers to pick plugins.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":259,"context_line":"However, the os-vif plugins deal with a small set of entities to which"},{"line_number":260,"context_line":"Neutron ports connect: Linux bridge, OVS, SR-IOV etc. Whereas os-acc"},{"line_number":261,"context_line":"plugins are expected to handle a much wider range of variation among"},{"line_number":262,"context_line":"devices. So, instead of having the Nova virt driver choose a plugin as"},{"line_number":263,"context_line":"with os-vif, os-acc will let the operator specify a set of plugins in a"},{"line_number":264,"context_line":"configuration file (``setup.cfg``) and will load them through stevedore"},{"line_number":265,"context_line":"(similar to Neutron controller plugins). Out-of-tree plugins can be"},{"line_number":266,"context_line":"enabled in this way."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"Workflow"},{"line_number":269,"context_line":"--------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_96aabcd0","line":266,"range":{"start_line":262,"start_character":9,"end_line":266,"end_character":20},"in_reply_to":"5f7c97a3_981d7064","updated":"2018-07-27 21:36:42.000000000","message":"This makes sense, thank you for the explanation.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":259,"context_line":"However, the os-vif plugins deal with a small set of entities to which"},{"line_number":260,"context_line":"Neutron ports connect: Linux bridge, OVS, SR-IOV etc. Whereas os-acc"},{"line_number":261,"context_line":"plugins are expected to handle a much wider range of variation among"},{"line_number":262,"context_line":"devices. So, instead of having the Nova virt driver choose a plugin as"},{"line_number":263,"context_line":"with os-vif, os-acc will let the operator specify a set of plugins in a"},{"line_number":264,"context_line":"configuration file (``setup.cfg``) and will load them through stevedore"},{"line_number":265,"context_line":"(similar to Neutron controller plugins). Out-of-tree plugins can be"},{"line_number":266,"context_line":"enabled in this way."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"Workflow"},{"line_number":269,"context_line":"--------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_e6297e9c","line":266,"range":{"start_line":262,"start_character":9,"end_line":266,"end_character":20},"in_reply_to":"5f7c97a3_cd72644b","updated":"2018-06-28 06:12:44.000000000","message":"\u003e - the operator installs/enables whatever set of plugin packages they desire\n\nYes\n\n \u003e - those plugins self-register on initialization so they\u0027re\n \u003e discoverable\n\nOk\n\n \u003e - when we need to do a thing XYZ that requires a plugin, we walk\n \u003e through the list of registered plugins asking, \"Can you do XYZ?\"\n \u003e and pick the first one that says \"yes\".  In VIF-land, I believe\n \u003e that question looks like, \"Can you handle VIF type such-and-such?\" \n\nIIUC, that is not how os-vif works. Apparently, the Nova virt driver picks a plugin: https://github.com/openstack/nova/blob/master/nova/network/os_vif_util.py#L272 \nwhich is called by:\nhttps://github.com/openstack/nova/blob/master/nova/network/os_vif_util.py#L499\nwhich is called, among others, by:\nhttps://github.com/openstack/nova/blob/master/nova/virt/libvirt/vif.py#L618 \n\n \u003e In acc-land, I would expect the question to be \"Can you handle\n \u003e mechanism X on accelerator of type Y from vendor Z?\"  (I wouldn\u0027t\n \u003e think we would need the question to encompass arch/hypervisor,\n \u003e because the op should only be installing plugins that work where\n \u003e they\u0027re being installed.)\n\n \u003e Does that gel with your expectation of the os-acc flow?\n\nBroadly yes. The plugin chain approach in L625 below is along these lines.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":259,"context_line":"However, the os-vif plugins deal with a small set of entities to which"},{"line_number":260,"context_line":"Neutron ports connect: Linux bridge, OVS, SR-IOV etc. Whereas os-acc"},{"line_number":261,"context_line":"plugins are expected to handle a much wider range of variation among"},{"line_number":262,"context_line":"devices. So, instead of having the Nova virt driver choose a plugin as"},{"line_number":263,"context_line":"with os-vif, os-acc will let the operator specify a set of plugins in a"},{"line_number":264,"context_line":"configuration file (``setup.cfg``) and will load them through stevedore"},{"line_number":265,"context_line":"(similar to Neutron controller plugins). Out-of-tree plugins can be"},{"line_number":266,"context_line":"enabled in this way."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"Workflow"},{"line_number":269,"context_line":"--------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_5fa5879e","line":266,"range":{"start_line":262,"start_character":9,"end_line":266,"end_character":20},"in_reply_to":"5f7c97a3_e6297e9c","updated":"2018-06-28 18:33:16.000000000","message":"\u003e IIUC, that is not how os-vif works. Apparently, the Nova virt\n \u003e driver picks a plugin: https://github.com/openstack/nova/blob/master/nova/network/os_vif_util.py#L272\n\nOkay, I see.  So the `plugin` field of the VIF* object is a generic handle given to the class registered via stevedore.\n\nSo then why would we need can_handle or the \"try in alphabetical order\" thing below?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":288,"context_line":"   within each node."},{"line_number":289,"context_line":"#. The scheduler chooses a compute node and a specific device RP. It"},{"line_number":290,"context_line":"   then invokes Nova Compute (n-cpu) on that compute node via RPC."},{"line_number":291,"context_line":"#. The n-cpu notes that one of the requested resources is of the form"},{"line_number":292,"context_line":"   CUSTOM_ACCELERATOR_* and so decides to invoke os-acc. The interaction"},{"line_number":293,"context_line":"   with os-acc can happen concurrently with other tasks for spawning the"},{"line_number":294,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_8823faaa","line":291,"range":{"start_line":291,"start_character":7,"end_line":291,"end_character":12},"updated":"2018-06-28 00:46:16.000000000","message":"I think this should be the virt driver, more specifically.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":288,"context_line":"   within each node."},{"line_number":289,"context_line":"#. The scheduler chooses a compute node and a specific device RP. It"},{"line_number":290,"context_line":"   then invokes Nova Compute (n-cpu) on that compute node via RPC."},{"line_number":291,"context_line":"#. The n-cpu notes that one of the requested resources is of the form"},{"line_number":292,"context_line":"   CUSTOM_ACCELERATOR_* and so decides to invoke os-acc. The interaction"},{"line_number":293,"context_line":"   with os-acc can happen concurrently with other tasks for spawning the"},{"line_number":294,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_621a7fcc","line":291,"range":{"start_line":291,"start_character":7,"end_line":291,"end_character":12},"in_reply_to":"5f7c97a3_16e54c3e","updated":"2018-08-14 17:14:37.000000000","message":"Please take a look at change set 9, and see if this is still an issue.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":288,"context_line":"   within each node."},{"line_number":289,"context_line":"#. The scheduler chooses a compute node and a specific device RP. It"},{"line_number":290,"context_line":"   then invokes Nova Compute (n-cpu) on that compute node via RPC."},{"line_number":291,"context_line":"#. The n-cpu notes that one of the requested resources is of the form"},{"line_number":292,"context_line":"   CUSTOM_ACCELERATOR_* and so decides to invoke os-acc. The interaction"},{"line_number":293,"context_line":"   with os-acc can happen concurrently with other tasks for spawning the"},{"line_number":294,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_98a010f6","line":291,"range":{"start_line":291,"start_character":7,"end_line":291,"end_character":12},"in_reply_to":"5f7c97a3_62b1c844","updated":"2018-06-29 02:23:10.000000000","message":"OK, then I\u0027ll leave the spec as is. Regarding accelerator names, I believe Alex Xu has volunteered to do it: http://lists.openstack.org/pipermail/openstack-dev/2018-June/131003.html","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":288,"context_line":"   within each node."},{"line_number":289,"context_line":"#. The scheduler chooses a compute node and a specific device RP. It"},{"line_number":290,"context_line":"   then invokes Nova Compute (n-cpu) on that compute node via RPC."},{"line_number":291,"context_line":"#. The n-cpu notes that one of the requested resources is of the form"},{"line_number":292,"context_line":"   CUSTOM_ACCELERATOR_* and so decides to invoke os-acc. The interaction"},{"line_number":293,"context_line":"   with os-acc can happen concurrently with other tasks for spawning the"},{"line_number":294,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_62b1c844","line":291,"range":{"start_line":291,"start_character":7,"end_line":291,"end_character":12},"in_reply_to":"5f7c97a3_69d8df51","updated":"2018-06-28 18:33:16.000000000","message":"Yes, I have thoughts - I believe I started a comment on that and then deleted it :)\n\nMy initial reaction is that it is the business of the virt driver how to identify and deal with accelerator resources.  It\u0027s not something you need to (or even should) specify in this document.\n\nBut that leaves us with a bit of a disconnect as far as guaranteeing that we actually deal with the resource.  IOW, cyborg went and created the RPs and inventory, the flavor went and asked for the accelerator, the scheduler want and allocated it, and the allocation is included in what we send to the virt driver.  But nothing through that whole process defines or restricts what the resource can be called or how it should be treated - and most especially, who\u0027s responsible for it.  So like, the virt driver knows how to look for certain things within the allocation and it knows what to do with them.  But anything it doesn\u0027t recognize, it just... ignores.\n\nOr does it?  Maybe we don\u0027t have this situation yet for resource providers, but we certainly have it for extra_specs.  You can put anything you want in there - including misspelling something that *is* meaningful - and if it\u0027s not in the list of things compute manager and/or virt recognize, it\u0027s ignored.\n\nSo maybe my original statement stands.  The virt driver needs to know how to deal with these resources, which includes knowing that it needs to look for them in the allocations, which includes knowing *how* to look for them.  So as long as cyborg is deterministic about how they\u0027re presented, such that virt drivers are able to do all of that, we should be good.\n\nAll of that said, keying off a RC name may or may not be the recommended best practice here.  Another way would be to recognize allocations whose RPs represent cyborg-owned devices.\n\nIncidentally, I think I\u0027ve said this before, but I hope CUSTOM_ACCELERATOR_FOO will actually be ACCELERATOR_FOO (i.e. registered as a standard resource class) before any of this goes into production.  You should propose those RCs as soon as possible, if you haven\u0027t already.\n\naaaand sorry for the ramble :)","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":288,"context_line":"   within each node."},{"line_number":289,"context_line":"#. The scheduler chooses a compute node and a specific device RP. It"},{"line_number":290,"context_line":"   then invokes Nova Compute (n-cpu) on that compute node via RPC."},{"line_number":291,"context_line":"#. The n-cpu notes that one of the requested resources is of the form"},{"line_number":292,"context_line":"   CUSTOM_ACCELERATOR_* and so decides to invoke os-acc. The interaction"},{"line_number":293,"context_line":"   with os-acc can happen concurrently with other tasks for spawning the"},{"line_number":294,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_69d8df51","line":291,"range":{"start_line":291,"start_character":7,"end_line":291,"end_character":12},"in_reply_to":"5f7c97a3_8823faaa","updated":"2018-06-28 06:12:44.000000000","message":"Yes. One question that was raised earlier: should the name of custom traits be interpreted this way? If we decide to support only well-known accelerator types, ike ACCELERATOR_GPU, and no custom types, then adding any new device type (i.e. resource class) will require a Nova change. Any thoughts?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":288,"context_line":"   within each node."},{"line_number":289,"context_line":"#. The scheduler chooses a compute node and a specific device RP. It"},{"line_number":290,"context_line":"   then invokes Nova Compute (n-cpu) on that compute node via RPC."},{"line_number":291,"context_line":"#. The n-cpu notes that one of the requested resources is of the form"},{"line_number":292,"context_line":"   CUSTOM_ACCELERATOR_* and so decides to invoke os-acc. The interaction"},{"line_number":293,"context_line":"   with os-acc can happen concurrently with other tasks for spawning the"},{"line_number":294,"context_line":"   instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_16e54c3e","line":291,"range":{"start_line":291,"start_character":7,"end_line":291,"end_character":12},"in_reply_to":"5f7c97a3_98a010f6","updated":"2018-07-27 21:36:42.000000000","message":"Perhaps what\u0027s needed is a mention that accelerator support will be added to the nova support matrix, so that each driver can declare support individually. Each driver will then be responsible for documenting the specifics of its implementation wrt what the operator needs to do to make it work. But codifying a naming convention (as you\u0027ve done) ought to result in those docs being fairly similar to each other, and ought to make the virt driver implementations able to reference each other to some degree to figure out how to add support.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":299,"context_line":"#. The os-acc loads an appropriate plugin for that request, as discussed"},{"line_number":300,"context_line":"   in `Os-acc - Plugin API`_."},{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_283426df","line":302,"range":{"start_line":302,"start_character":38,"end_line":302,"end_character":54},"updated":"2018-06-28 00:46:16.000000000","message":"I thought this happened on L295-298","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":299,"context_line":"#. The os-acc loads an appropriate plugin for that request, as discussed"},{"line_number":300,"context_line":"   in `Os-acc - Plugin API`_."},{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_e9eb8fda","line":302,"range":{"start_line":302,"start_character":38,"end_line":302,"end_character":54},"in_reply_to":"5f7c97a3_283426df","updated":"2018-06-28 06:12:44.000000000","message":"n-cpu --\u003e os-acc --\u003e plugin. It is a chain of calls to get the VAN object. What the plugin returns to os-acc is passed back to n-cpu.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":299,"context_line":"#. The os-acc loads an appropriate plugin for that request, as discussed"},{"line_number":300,"context_line":"   in `Os-acc - Plugin API`_."},{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_16ceacb6","line":302,"range":{"start_line":302,"start_character":38,"end_line":302,"end_character":54},"in_reply_to":"5f7c97a3_e9eb8fda","updated":"2018-07-27 21:36:42.000000000","message":"okay, I get it","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_08936aa6","line":306,"range":{"start_line":304,"start_character":17,"end_line":306,"end_character":40},"updated":"2018-06-28 00:46:16.000000000","message":"No, this part needs to be done in os-acc *outside* of the plugin.  The general flow for any given operation should be:\n\n os-acc consumer (e.g. nova virt driver):\n     VANObj \u003d os_acc.do_thing(VANObj)\n         os_acc.do_thing:\n             pre ...\n             VANObj \u003d plugin.do_thing(VANObj)\n                 plugin.do_thing:\n                     magic magic magic\n                     update VANObj\n                     return VANObj\n             post (e.g. persist VANObj changes)\n\nIn other words, the plugin should *only* interact with cyborg through changes to the VAN object.  It should *not* call the cyborg API/agent directly.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_2990a74f","line":306,"range":{"start_line":304,"start_character":17,"end_line":306,"end_character":40},"in_reply_to":"5f7c97a3_08936aa6","updated":"2018-06-28 06:12:44.000000000","message":"Agreed that the update of Cyborg db can happen in the os-acc outside the plugin. However, it is too restrictive to say that the plugin should not call Cyborg. A specific plugin may choose to delegate some or all its work to Cyborg agent/driver, while another may not.\n\nJust curious: a GPU plugin for Power could invoke a Power/GPU-specific Cyborg driver through the Cyborg agent, and its plug() API could do what is required for Power-based hypervisor(s). Do you see that as a viable implementation?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_82466422","line":306,"range":{"start_line":304,"start_character":17,"end_line":306,"end_character":40},"in_reply_to":"5f7c97a3_2990a74f","updated":"2018-06-28 18:33:16.000000000","message":"\u003e it is too restrictive to say that the\n \u003e plugin should not call Cyborg. A specific plugin may choose to\n \u003e delegate some or all its work to Cyborg agent/driver, while another\n \u003e may not.\n\nCan you give me an example of an operation/flow for which a plugin method should need to call cyborg?\n\nIMO, if the plugin needs to invoke cyborg, we\u0027ve done a poor job of delineating the responsibilities of the plugin method.  (Never mind the potential for circular code paths.)\n\n \u003e Just curious: a GPU plugin for Power could invoke a\n \u003e Power/GPU-specific Cyborg driver through the Cyborg agent, and its\n \u003e plug() API could do what is required for Power-based hypervisor(s).\n \u003e Do you see that as a viable implementation?\n\nSorry, this isn\u0027t parsing for me.  What\u0027s a \"driver\" in this context, as opposed to a \"plugin\"?  And while you\u0027re at it, what\u0027s \"the Cyborg agent\"?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_b80e2c8a","line":306,"range":{"start_line":304,"start_character":17,"end_line":306,"end_character":40},"in_reply_to":"5f7c97a3_82466422","updated":"2018-06-29 02:23:10.000000000","message":"\u003e Can you give me an example of an operation/flow for \n\u003e which a plugin method should need to call cyborg?\n\nCyborg already has vendor-specific drivers to discover and enumerate devices like FPGAs. Instead of having yet another vendor-specific component, the plugin may call the Cyborg agent, which will delegate to the Cyborg driver for device-specific actions. So, the plugin\u0027s getVAN() is delegated, whereas the plug() is still, um, in the plugin because it is hypervisor-specific, not vendor-specific.\n\n \u003e IMO, if the plugin needs to invoke cyborg, we\u0027ve done a poor job of\n \u003e delineating the responsibilities of the plugin method.  (Never mind\n \u003e the potential for circular code paths.)\n\nPlease see the sequence diagram: each step seems to have a clear responsibility IMHO. \n   The plugin handles device-specific and hypervisor-specific actions. Of these, it may delegate the device-specific parts to a device/vendor-specific driver.\n\n \u003e What\u0027s a \"driver\" in this\n \u003e context, as opposed to a \"plugin\"?  And while you\u0027re at it, what\u0027s\n \u003e \"the Cyborg agent\"?\n\nThe Cyborg architecture has a daemon in the compute node, analogous to n-cpu, called the agent. Please see https://wiki.openstack.org/wiki/Cyborg#Pike.2FQueens_Initial_Architecture","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_f6245061","line":306,"range":{"start_line":304,"start_character":17,"end_line":306,"end_character":40},"in_reply_to":"5f7c97a3_9acf1f27","updated":"2018-07-27 21:36:42.000000000","message":"\u003e At the risk of touching a nerve :), I am going to suggest that it\n \u003e is best for Power to provide Cyborg driver(s) for the supported\n \u003e devices, possibly in addition to the plugin.\n\nYeah, that\u0027s what I figured we would end up with. And that seems like the right thing.\n\n \u003e * We want one entity that creates resource classes for GPUs and\n \u003e other\n \u003e devices, rather than PowerVM virt driver (or the plugin) for Power,\n \u003e and\n \u003e Cyborg driver/agent for others.\n\n\u003c3\n\n \u003e * RBAC for accelerator access should be in a single entity. For\n \u003e example,\n \u003e if we want to prevent some tenants from using accelerators or\n \u003e programming\n \u003e them, all those checks are better done in one entity. That is going\n \u003e to be\n \u003e Cyborg for most things, so Power may want to leverage that.\n\n\u003c3\n\n \u003e * If/when we get to FPGAs on Power, it is better to have bitstream\n \u003e management and programming in a single flow, independent of\n \u003e architecture/hypervisor.\n\nYes, BUT that flow will almost certainly need to be brokered through Power-specific code at some stage. That code should be part of the driver or plugin (not sure which is the most appropriate, or tbh what the delineation is between the driver and the plugin).\n\n \u003e * Not an expert here, but it seems like PowerKVM is managed by\n \u003e libvirt\n \u003e than PowerVM virt driver \n\nCorrect.\n\n \u003e May be there are benefits to unify the workflows for both?\n\nI\u0027ll need to call in the cavalry on this one. To my understanding, there has been almost no overlap between PowerVM and PowerKVM historically, but that may be changing. If you\u0027re suggesting PowerVM and PowerKVM share (some subset of) Cyborg driver code, that *may* be possible, yes. But again, I don\u0027t know.\n\n \u003e The Cyborg driver for Power is free to call its favorite REST APIs,\n \u003e for discovery or anything else. We can work to make the Cyborg\n \u003e agent-driver API sufficiently flexible to handle Power\u0027s needs.\n\n\u003c3\n\n \u003e Once you have a Cyborg driver, it is not clear what you would need\n \u003e the plugin for. But we can say that the plugin is still in the\n \u003e picture -- it is up to the plugin/Cyborg-driver author(s) to split\n \u003e responsibilities.\n\nYeah, this is what I said above - what are the respective roles and responsibilities of the driver vs. the plugin?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"5cfd984c993def9052f55f764ffb5fd26c17f642","unresolved":false,"context_lines":[{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_f717b8ed","line":306,"range":{"start_line":304,"start_character":17,"end_line":306,"end_character":40},"in_reply_to":"5f7c97a3_b80e2c8a","updated":"2018-06-29 21:33:04.000000000","message":"Okay, so the driver is to discover and enumerate devices.\n\nThese things will need to be platform-specific as well.  I don\u0027t know about other hypervisors like VMWare, but I know for sure you\u0027re not going to use the same mechanisms on Power as you do on libvirt.\n\nSo what does that model look like?  I would expect the Power-specific plugin (or a separate thing, a Power-specific driver) would need to be involved in the \"discover and enumerate\" process.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":301,"context_line":"#. The os-acc then calls a well-known API in the plugin to create the"},{"line_number":302,"context_line":"   Device Half of the attach and thus get a VAN object. A reference to"},{"line_number":303,"context_line":"   the plugin may be stored in the VAN for use in subsequent calls for"},{"line_number":304,"context_line":"   the same VAN. The plugin *may* call the Cyborg agent on the node to"},{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_9acf1f27","line":306,"range":{"start_line":304,"start_character":17,"end_line":306,"end_character":40},"in_reply_to":"5f7c97a3_f717b8ed","updated":"2018-07-06 02:23:20.000000000","message":"The Cyborg drivers, as currently envisioned, encapsulate device-specific actions, not only for discovery/enumeration, but also for everything else: resets, programming, obtaining performance counters, etc.\n\nAt the risk of touching a nerve :), I am going to suggest that it is best for Power to provide Cyborg driver(s) for the supported devices, possibly in addition to the plugin. Please bear with me as I explain:\n\n* We want one entity that creates resource classes for GPUs and other\n  devices, rather than PowerVM virt driver (or the plugin) for Power, and \n  Cyborg driver/agent for others.\n* RBAC for accelerator access should be in a single entity. For example,\n  if we want to prevent some tenants from using accelerators or programming\n  them, all those checks are better done in one entity. That is going to be \n  Cyborg for most things, so Power may want to leverage that.\n* If/when we get to FPGAs on Power, it is better to have bitstream\n  management and programming in a single flow, independent of \n  architecture/hypervisor.\n* Not an expert here, but it seems like PowerKVM is managed by libvirt\n  than PowerVM virt driver (https://www.ibm.com/support/knowledgecenter/en/SSZJY4_3.1.0/liabo/liaboconcept.htm). May be there are benefits to unify the workflows for both?\n\nThe Cyborg driver for Power is free to call its favorite REST APIs, for discovery or anything else. We can work to make the Cyborg agent-driver API sufficiently flexible to handle Power\u0027s needs.\n\nOnce you have a Cyborg driver, it is not clear what you would need the plugin for. But we can say that the plugin is still in the picture -- it is up to the plugin/Cyborg-driver author(s) to split responsibilities.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":305,"context_line":"   do the job, but *must* call the Cyborg agent to have the VAN object"},{"line_number":306,"context_line":"   persisted (via the Cyborg conductor)."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"   The process of getting a VAN object also serves to bind a specific"},{"line_number":309,"context_line":"   accelerator instance in the chosen device RP to that VAN. It may also"},{"line_number":310,"context_line":"   initiate programming in the case of FPGAs."},{"line_number":311,"context_line":"#. The n-cpu then calls the ``plug()`` API in os-acc for the Instance"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c8e1f22d","line":308,"range":{"start_line":308,"start_character":54,"end_line":308,"end_character":58},"updated":"2018-06-28 00:46:16.000000000","message":"indeed","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":310,"context_line":"   initiate programming in the case of FPGAs."},{"line_number":311,"context_line":"#. The n-cpu then calls the ``plug()`` API in os-acc for the Instance"},{"line_number":312,"context_line":"   Half of the attach, i.e., to associate the VAN with the instance."},{"line_number":313,"context_line":"   For example, in libvirt-based x86 systems, this may return an XML"},{"line_number":314,"context_line":"   snippet that includes the PCI function encapsulated by the VAN."},{"line_number":315,"context_line":"   The instance - VAN association is persisted by Nova."},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_08c12a82","line":314,"range":{"start_line":313,"start_character":46,"end_line":314,"end_character":66},"updated":"2018-06-28 00:46:16.000000000","message":"...embedded in the VAN object.  The return from the plugin\u0027s plug method should be a VAN object, period.  The whole point of this plugin model is to provide a *consistent* framework for interaction.  It\u0027s counterproductive if the \"generic\" parts of the code need to have platform/vendor-specific special cases.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":310,"context_line":"   initiate programming in the case of FPGAs."},{"line_number":311,"context_line":"#. The n-cpu then calls the ``plug()`` API in os-acc for the Instance"},{"line_number":312,"context_line":"   Half of the attach, i.e., to associate the VAN with the instance."},{"line_number":313,"context_line":"   For example, in libvirt-based x86 systems, this may return an XML"},{"line_number":314,"context_line":"   snippet that includes the PCI function encapsulated by the VAN."},{"line_number":315,"context_line":"   The instance - VAN association is persisted by Nova."},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_49f2c39d","line":314,"range":{"start_line":313,"start_character":46,"end_line":314,"end_character":66},"in_reply_to":"5f7c97a3_08c12a82","updated":"2018-06-28 06:12:44.000000000","message":"OK. I\u0027ll update to say the VAN object has a hypervisor-specific part that the plug() populates.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"d56dea2295b66c7c1e325335f37a1cabbb823125","unresolved":false,"context_lines":[{"line_number":312,"context_line":"   Half of the attach, i.e., to associate the VAN with the instance."},{"line_number":313,"context_line":"   For example, in libvirt-based x86 systems, this may return an XML"},{"line_number":314,"context_line":"   snippet that includes the PCI function encapsulated by the VAN."},{"line_number":315,"context_line":"   The instance - VAN association is persisted by Nova."},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_a956c004","line":315,"range":{"start_line":315,"start_character":7,"end_line":315,"end_character":55},"updated":"2018-06-30 02:18:24.000000000","message":"Do we need to modify nova db to make this happen?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":312,"context_line":"   Half of the attach, i.e., to associate the VAN with the instance."},{"line_number":313,"context_line":"   For example, in libvirt-based x86 systems, this may return an XML"},{"line_number":314,"context_line":"   snippet that includes the PCI function encapsulated by the VAN."},{"line_number":315,"context_line":"   The instance - VAN association is persisted by Nova."},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_ba49bb8e","line":315,"range":{"start_line":315,"start_character":7,"end_line":315,"end_character":55},"in_reply_to":"5f7c97a3_a956c004","updated":"2018-07-06 02:23:20.000000000","message":"Yes, as stated in line 791.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":314,"context_line":"   snippet that includes the PCI function encapsulated by the VAN."},{"line_number":315,"context_line":"   The instance - VAN association is persisted by Nova."},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c8ba520a","line":318,"range":{"start_line":317,"start_character":3,"end_line":318,"end_character":60},"updated":"2018-06-28 00:46:16.000000000","message":"IMO this statement is not needed.  The two are completely analogous.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":314,"context_line":"   snippet that includes the PCI function encapsulated by the VAN."},{"line_number":315,"context_line":"   The instance - VAN association is persisted by Nova."},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_2cad1532","line":318,"range":{"start_line":317,"start_character":3,"end_line":318,"end_character":60},"in_reply_to":"5f7c97a3_c8ba520a","updated":"2018-06-28 06:12:44.000000000","message":"An attachment for a port or an accelerator has a \u0027VM side\u0027 and the other side\u0027. The os-vif plug() deals with the other side, i.e. bridge/OVS/..., not the VM side. The os-acc plug() deals with the VM side, not the other side (i.e. the device).","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_e8b70e21","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"updated":"2018-06-28 00:46:16.000000000","message":"nono, the plug() operation *performs* the attach.  That\u0027s its whole purpose in life.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"7aa4cae0d18f2338ec7e5f4cc70924c1bc6edac2","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_962a8575","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"in_reply_to":"5f7c97a3_1a6d4f2e","updated":"2018-07-19 08:23:14.000000000","message":"It still not clear for me what would be the return and of plug and the purpose. The driver whill have the necessary information to generate any hypervisor configuration to build the guest. For libvirt that will be the domain XML. Then a call will be done to a occ plugin with the VAN and instanceinfo object, so the plugin can finalize some tweaks in host.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_1a6d4f2e","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"in_reply_to":"5f7c97a3_77d96854","updated":"2018-07-06 02:23:20.000000000","message":"We are going to have a single os-acc python code base for x86, Power, etc. That code has to do the right thing for each arch. So, instead of having os-acc look up the arch and hardcode some action, I would rather have the plugin say clearly what remains to be done -- because the plugin is indeed arch-specific and knows that.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"5cfd984c993def9052f55f764ffb5fd26c17f642","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_77d96854","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"in_reply_to":"5f7c97a3_78aad41a","updated":"2018-06-29 21:33:04.000000000","message":"I would imagine it would know by virtue of the developer writing it thus, and the developer does that based on knowing what the plugin does.  Frankly, I was surprised the virt driver would ever need to do anything after the plug(); in Power, it just won\u0027t.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_a21bb7c7","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"in_reply_to":"5f7c97a3_962a8575","updated":"2018-08-14 17:14:37.000000000","message":"It returns an updated VAN. In change set 9, the plug() has been renamed as attach_for_instance() and it explicitly returns an array of VANs, once for each requested accelerator.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c86b1bb7","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"in_reply_to":"5f7c97a3_acfa850e","updated":"2018-06-28 18:33:16.000000000","message":"Okay, I walked through some code and I think I understand the flow.  The sticking point is \"the return value of ``plug()``\".  In os-vif, plug doesn\u0027t return anything.  The libvirt driver queries the VIF* object for the XML snippet after the fact, and uses that to do the attach.  The model for os-acc should be the same in the sense that the contract of plug() does *not* entail a return value.  And there is also no *requirement* that the attach be performed by the caller of plug().  For example, in PowerVM, the whole operation is likely to be done within the plug() method.\n\nIn summary:\n\n- plug() should not return anything.\n- The wording here should be more suggestive than prescriptive (\"Drivers may wish to query the VAN object after the plug() to glean information necessary to complete the operation.\")","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_78aad41a","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"in_reply_to":"5f7c97a3_c86b1bb7","updated":"2018-06-29 02:23:10.000000000","message":"Cool, I think we are converging. I would phrase it a bit differently though.\n* The plugin should return a VAN object. Though it takes a VAN object as a parameter, it may return a VAN object of a child class e.g. VANTypePCI (inherited from VANBase). The virt driver can ignore the return value if not needed.\n\n* How would the virt driver know whether any work remains to be done after plug() returns? The type of the returned VAN object can indicate that. If it is VANTypePCI, virt driver should do more work; if it is VANTypePower (or whatever), it is done.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":316,"context_line":""},{"line_number":317,"context_line":"   NOTE: The os-vif ``plug()`` API attaches a Neutron port to a bridge,"},{"line_number":318,"context_line":"   OVS, etc. The os-acc ``plug()`` call does something else."},{"line_number":319,"context_line":"#. Nova virt driver uses the return value of ``plug()``"},{"line_number":320,"context_line":"   to complete the attach of the accelerator to the instance."},{"line_number":321,"context_line":"#. The n-cpu spawns the VM with the accelerator attached to it."},{"line_number":322,"context_line":"#. When the instance is deleted, or a single accelerator is detached"},{"line_number":323,"context_line":"   (through some mechanism TBD):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_acfa850e","line":320,"range":{"start_line":319,"start_character":3,"end_line":320,"end_character":61},"in_reply_to":"5f7c97a3_e8b70e21","updated":"2018-06-28 06:12:44.000000000","message":"Taking libvirt as an example, the XML snippet for a PCI device representing an accelerator needs to be inserted into the overall domain XML for the VM. The plug() does not have the overall domain XML , but can provide the XML snippet that needs to be inserted into the domain XML. \n\nIn light of this, can you clarify what you mean by \u0027plug() performs the attach.\u0027 ?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":325,"context_line":"   #.  n-cpu calls ``unplug()`` API in os-acc, passing the relevant VAN(s)."},{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_a837d6af","line":328,"range":{"start_line":328,"start_character":10,"end_line":328,"end_character":61},"updated":"2018-06-28 00:46:16.000000000","message":"no","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":325,"context_line":"   #.  n-cpu calls ``unplug()`` API in os-acc, passing the relevant VAN(s)."},{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_585b5854","line":328,"range":{"start_line":328,"start_character":10,"end_line":328,"end_character":61},"in_reply_to":"5f7c97a3_28de4f10","updated":"2018-06-29 02:23:10.000000000","message":"If there is already another entity that is doing device/platform-specific things, the developer may want to put all device/vendor/platform things in one place -- i.e. the driver -- and delegate from the plugin to the driver.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"5cfd984c993def9052f55f764ffb5fd26c17f642","unresolved":false,"context_lines":[{"line_number":325,"context_line":"   #.  n-cpu calls ``unplug()`` API in os-acc, passing the relevant VAN(s)."},{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_f7ec98f5","line":328,"range":{"start_line":328,"start_character":10,"end_line":328,"end_character":61},"in_reply_to":"5f7c97a3_585b5854","updated":"2018-06-29 21:33:04.000000000","message":"Wait, now we\u0027re talking about the plugin delegating to the *virt* driver?  Now I\u0027m really confused.\n\nI\u0027m going to stop arguing this point and let it shake out in the implementation.  Basically setting up for an \"I told you so\".","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":325,"context_line":"   #.  n-cpu calls ``unplug()`` API in os-acc, passing the relevant VAN(s)."},{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_ec171d46","line":328,"range":{"start_line":328,"start_character":10,"end_line":328,"end_character":61},"in_reply_to":"5f7c97a3_a837d6af","updated":"2018-06-28 06:12:44.000000000","message":"This is too restrictive. Plugins should be allowed to choose their implementation.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":325,"context_line":"   #.  n-cpu calls ``unplug()`` API in os-acc, passing the relevant VAN(s)."},{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_28de4f10","line":328,"range":{"start_line":328,"start_character":10,"end_line":328,"end_character":61},"in_reply_to":"5f7c97a3_ec171d46","updated":"2018-06-28 18:33:16.000000000","message":"Again, I don\u0027t understand under what circumstances it would be necessary to do\n\n driver \u003d\u003e os-acc \u003d\u003e cyborg: pre\n                  \u003d\u003e plugin: do_thing \u003d\u003e cyborg: foo\n                  \u003d\u003e cyborg post\n\nrather than\n\n driver \u003d\u003e os-acc \u003d\u003e cyborg: pre (possibly including foo)\n                  \u003d\u003e plugin: do_thing\n                  \u003d\u003e cyborg: post (possibly including foo)\n\nIf there\u0027s something platform-specific that do_thing needs to do, it should... just do it.  If do_thing needs to do platform-specific stuff, then generic stuff, then more platform-specific stuff, then we\u0027ve defined things wrong an we should split do_thing into separate pieces so we can preserve the separation of responsibilities and the unidirectionality of the communication channels.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":325,"context_line":"   #.  n-cpu calls ``unplug()`` API in os-acc, passing the relevant VAN(s)."},{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_fa6f3326","line":328,"range":{"start_line":328,"start_character":10,"end_line":328,"end_character":61},"in_reply_to":"5f7c97a3_f7ec98f5","updated":"2018-07-06 02:23:20.000000000","message":"No, Cyborg driver. Sure, I don\u0027t mind saying \u0027I told you so\u0027 :) . Let me hash this out in the next update (overdue at this point).","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"},{"line_number":332,"context_line":"      plugin calls Cyborg API directly.)"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_6841be0a","line":329,"range":{"start_line":329,"start_character":6,"end_line":329,"end_character":63},"updated":"2018-06-28 00:46:16.000000000","message":"no, see above, this is done in the non-plugin os-acc code as a pre/post step to invoking the plugin\u0027s unplug method.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"},{"line_number":332,"context_line":"      plugin calls Cyborg API directly.)"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_82755bfc","line":329,"range":{"start_line":329,"start_character":6,"end_line":329,"end_character":63},"in_reply_to":"5f7c97a3_0c15b93d","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":326,"context_line":"   #. The os-acc calls the right plugin to get the accelerator instances"},{"line_number":327,"context_line":"      detached."},{"line_number":328,"context_line":"   #. The plugin *may* call the Cyborg agent to do the detach, but"},{"line_number":329,"context_line":"      *must* call Cyborg to record the release of the resources. (For"},{"line_number":330,"context_line":"      the latter, either the plugin calls Cyborg agent as part of"},{"line_number":331,"context_line":"      delegating work and the agent calls the Cyborg REST API, or the"},{"line_number":332,"context_line":"      plugin calls Cyborg API directly.)"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_0c15b93d","line":329,"range":{"start_line":329,"start_character":6,"end_line":329,"end_character":63},"in_reply_to":"5f7c97a3_6841be0a","updated":"2018-06-28 06:12:44.000000000","message":"Sure, I can update this part.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":353,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027can_handle(...)\u0027]"},{"line_number":354,"context_line":"    \u0027Nova Compute\u0027 \u003c- \u0027os-acc\u0027 [label\u003d\u0027\u0027]"},{"line_number":355,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":356,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027plugin\u0027       [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":357,"context_line":"    \u0027plugin\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":358,"context_line":"    \u0027Cyborg agent\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d"},{"line_number":359,"context_line":"        \u0027* May call Glance to fetch bitstream"},{"line_number":360,"context_line":"         * May reprogram the device"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_6828de32","line":357,"range":{"start_line":356,"start_character":16,"end_line":357,"end_character":15},"updated":"2018-06-28 00:46:16.000000000","message":"strike.\n\nThis part shouldn\u0027t need to talk to the plugin.\n\nAnd the plugin should never talk to the agent.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":353,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027can_handle(...)\u0027]"},{"line_number":354,"context_line":"    \u0027Nova Compute\u0027 \u003c- \u0027os-acc\u0027 [label\u003d\u0027\u0027]"},{"line_number":355,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":356,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027plugin\u0027       [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":357,"context_line":"    \u0027plugin\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":358,"context_line":"    \u0027Cyborg agent\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d"},{"line_number":359,"context_line":"        \u0027* May call Glance to fetch bitstream"},{"line_number":360,"context_line":"         * May reprogram the device"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_5b79122e","line":357,"range":{"start_line":356,"start_character":16,"end_line":357,"end_character":15},"in_reply_to":"5f7c97a3_085ef378","updated":"2018-06-29 02:23:10.000000000","message":"This is not quite what I had in mind. I\u0027ll take another stab in the next update.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":353,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027can_handle(...)\u0027]"},{"line_number":354,"context_line":"    \u0027Nova Compute\u0027 \u003c- \u0027os-acc\u0027 [label\u003d\u0027\u0027]"},{"line_number":355,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":356,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027plugin\u0027       [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":357,"context_line":"    \u0027plugin\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":358,"context_line":"    \u0027Cyborg agent\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d"},{"line_number":359,"context_line":"        \u0027* May call Glance to fetch bitstream"},{"line_number":360,"context_line":"         * May reprogram the device"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_ec5e3d12","line":357,"range":{"start_line":356,"start_character":16,"end_line":357,"end_character":15},"in_reply_to":"5f7c97a3_6828de32","updated":"2018-06-28 06:12:44.000000000","message":"\u003e This part shouldn\u0027t need to talk to the plugin.\n\nWhy not? The plugin needs to populate the VAN with the appropriate info \n\n \u003e And the plugin should never talk to the agent.\n\nI should clarify that calling the Cyborg agent is an optional thing.  I wouldn\u0027t say \u0027never\u0027.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":353,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027can_handle(...)\u0027]"},{"line_number":354,"context_line":"    \u0027Nova Compute\u0027 \u003c- \u0027os-acc\u0027 [label\u003d\u0027\u0027]"},{"line_number":355,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":356,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027plugin\u0027       [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":357,"context_line":"    \u0027plugin\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d\u0027getVAN(...)\u0027]"},{"line_number":358,"context_line":"    \u0027Cyborg agent\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d"},{"line_number":359,"context_line":"        \u0027* May call Glance to fetch bitstream"},{"line_number":360,"context_line":"         * May reprogram the device"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_085ef378","line":357,"range":{"start_line":356,"start_character":16,"end_line":357,"end_character":15},"in_reply_to":"5f7c97a3_ec5e3d12","updated":"2018-06-28 18:33:16.000000000","message":"Okay, then the steps should be reversed.  Again, this flow should be shaped as discussed above.  In seqdiag terms, something like:\n\n \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\u0027getVan()\u0027]\n \u0027os-acc\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d\u0027generic pre-getVAN stuff as necessary\u0027]\n \u0027os-acc\u0027 \u003c-- \u0027Cyborg agent\u0027\n \u0027os-acc\u0027 -\u003e \u0027plugin\u0027 [label\u003d\u0027getVan()\u0027]\n \u0027os-acc\u0027 \u003c-- \u0027plugin\u0027\n \u0027os-acc\u0027 -\u003e \u0027Cyborg agent\u0027 [label\u003d\u0027generic post-getVAN stuff as necessary\u0027]\n \u0027os-acc\u0027 \u003c-- \u0027Cyborg agent\u0027\n \u0027Nova Compute\u0027 \u003c-- \u0027os-acc\u0027","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":379,"context_line":"may have one or more VANs. The number of VANs may be fixed (and"},{"line_number":380,"context_line":"statically discoverable, as with PCI functions) or not known initially"},{"line_number":381,"context_line":"(and a new one can be created dynamically on request, as with"},{"line_number":382,"context_line":"mediated devices)."},{"line_number":383,"context_line":""},{"line_number":384,"context_line":"A VAN is somewhat analogous to a VIF, though the analogy is not exact."},{"line_number":385,"context_line":"In particular, it is a versioned object, as shown below, which can be"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_76a12092","line":382,"updated":"2018-07-27 21:36:42.000000000","message":"++","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     VERSION \u003d \"1.0\""},{"line_number":391,"context_line":"     fields \u003d {"},{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c899f27a","line":394,"range":{"start_line":393,"start_character":8,"end_line":394,"end_character":46},"updated":"2018-06-28 00:46:16.000000000","message":"This concerns me a bit: it means that in order to create a plugin for a new hypervisor type, I would need to submit a code change to os-acc itself.  I would rather be able to avoid that.  Can this just be a string field, garbage-in garbage-out, that we simply use to match up with whatever is requested?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     VERSION \u003d \"1.0\""},{"line_number":391,"context_line":"     fields \u003d {"},{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_3b521637","line":394,"range":{"start_line":393,"start_character":8,"end_line":394,"end_character":46},"in_reply_to":"5f7c97a3_08ecb390","updated":"2018-06-29 02:23:10.000000000","message":"Broadly the idea is that os-acc tells the plugin about the hypervisor, and the plugin returns the data for that hypervisor (or just does the action for that hypervisor).\n \nYour point is that the installed plugins must be only for the relevant hypervisor. However:\n\n* It is not desirable to rely on the operator\u0027s manual configuration to always be correct. In the can_handle () call, the plugin gets to decide whether the specified hypervisor is indeed something it can handle.\n\n* I may have a single plugin that works for both libvirt+qemu and linvirt+Xen, with the differences handled by the plugin.\n\nMore broadly, I suspect many hypervisors would require the os-acc or virt driver to interpret what plug() returns, so the goal that adding new hypervisors should not require changes in os-acc may be a long shot.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     VERSION \u003d \"1.0\""},{"line_number":391,"context_line":"     fields \u003d {"},{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_08ecb390","line":394,"range":{"start_line":393,"start_character":8,"end_line":394,"end_character":46},"in_reply_to":"5f7c97a3_2c20b5b1","updated":"2018-06-28 18:33:16.000000000","message":"Looking again, I\u0027m not sure what purpose this field is serving.  Can you explain?  I would think the subclass is defined in a plugin that knows what hypervisor it works on; and that plugin was installed by the operator on a system with that hypervisor.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"7aa4cae0d18f2338ec7e5f4cc70924c1bc6edac2","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     VERSION \u003d \"1.0\""},{"line_number":391,"context_line":"     fields \u003d {"},{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c206f821","line":394,"range":{"start_line":393,"start_character":8,"end_line":394,"end_character":46},"in_reply_to":"5f7c97a3_3b521637","updated":"2018-07-19 08:23:14.000000000","message":"What kind of difference do you see based on the hypervisor? os-acc plug will probably perform actions in the host OS - it is not expected that it tweak the hypervisor. That part should stay the responsibility of the Nova drivers","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     VERSION \u003d \"1.0\""},{"line_number":391,"context_line":"     fields \u003d {"},{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_42c183e4","line":394,"range":{"start_line":393,"start_character":8,"end_line":394,"end_character":46},"in_reply_to":"5f7c97a3_c206f821","updated":"2018-08-14 17:14:37.000000000","message":"Please see change set 9. In short, configuring the device end of the attach may also depend on the hypervisor/architecture. For example, in x86, a PCI device is represented with a BDF, whereas Power apparently represents them differently.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     VERSION \u003d \"1.0\""},{"line_number":391,"context_line":"     fields \u003d {"},{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_2c20b5b1","line":394,"range":{"start_line":393,"start_character":8,"end_line":394,"end_character":46},"in_reply_to":"5f7c97a3_c899f27a","updated":"2018-06-28 06:12:44.000000000","message":"I am not terribly against a string field -- but are there so many hypervisors that this is a problem? We can even pre-add \nAN enum gives better checking.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14131,"name":"shaohef","email":"shaohe.feng@intel.com","username":"shaohefeng"},"change_message_id":"ab088451c4124b8636b6a7503c92f0fd045872fc","unresolved":false,"context_lines":[{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"It can be extended, for example, as below to handle different"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c067cbe9","line":395,"range":{"start_line":395,"start_character":8,"end_line":395,"end_character":29},"updated":"2018-07-19 13:03:43.000000000","message":"Van can index to is plugin by this fields.\n\nVice versa VanPlugin can also index to Van by its getVan() function?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":392,"context_line":"        id: UUIDField() # VAN ID"},{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"It can be extended, for example, as below to handle different"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_5d05e40f","line":395,"range":{"start_line":395,"start_character":8,"end_line":395,"end_character":29},"in_reply_to":"5f7c97a3_c067cbe9","updated":"2018-08-14 17:14:37.000000000","message":"The VAN is passed to the getVAN() as a parameter. BTW, the getVAN() has been renamed as prepareVAN() in change set 9.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"It can be extended, for example, as below to handle different"},{"line_number":399,"context_line":"mechanisms (VANfactors_)::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_28766641","line":396,"range":{"start_line":396,"start_character":5,"end_line":396,"end_character":6},"updated":"2018-06-28 00:46:16.000000000","message":"We should also have a \"profile\" field of some kind.  This would house an instance of a platform-/vendor-/type-specific subclass of e.g. VANProfileBase containing configuration information.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"It can be extended, for example, as below to handle different"},{"line_number":399,"context_line":"mechanisms (VANfactors_)::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_51433340","line":396,"range":{"start_line":396,"start_character":5,"end_line":396,"end_character":6},"in_reply_to":"5f7c97a3_28766641","updated":"2018-06-29 02:23:10.000000000","message":"The plug() API returns a VAN object. (I know you said Power plugin will do the attach itself, but other hypervisors will need a return value.) That returned VAN will have to contain data related to the mechanism (PCI, mdev, ... ) and the hypervisor. That could be the profile. \n\nSo, the profile idea makes sense to me, except that all vendor-related aspects should be in the plugin (Device Half of the attach), not in the VAN profile (Instance Half of the attach). I will include that.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"It can be extended, for example, as below to handle different"},{"line_number":399,"context_line":"mechanisms (VANfactors_)::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_d6f7d48d","line":396,"range":{"start_line":396,"start_character":5,"end_line":396,"end_character":6},"in_reply_to":"5f7c97a3_51433340","updated":"2018-07-27 21:36:42.000000000","message":"\u003e should be in the plugin... not in the VAN profile\n\nSorry, I don\u0027t understand. The plugin would need to *use* the information from the VAN profile, nah?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":393,"context_line":"        hypervisor: AccHypervisorEnumField()"},{"line_number":394,"context_line":"            # \u0027LibvirtQemu\u0027, LibvirtXen\u0027, etc."},{"line_number":395,"context_line":"        plugin: StringField() # name of plugin class to load"},{"line_number":396,"context_line":"     }"},{"line_number":397,"context_line":""},{"line_number":398,"context_line":"It can be extended, for example, as below to handle different"},{"line_number":399,"context_line":"mechanisms (VANfactors_)::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_dd101451","line":396,"range":{"start_line":396,"start_character":5,"end_line":396,"end_character":6},"in_reply_to":"5f7c97a3_d6f7d48d","updated":"2018-08-14 17:14:37.000000000","message":"Please check if change set 9 addresses this.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":440,"context_line":""},{"line_number":441,"context_line":"The main factors involved in attaching an accelerator (see VANfactors_)"},{"line_number":442,"context_line":"are the device type, hypervisor type, mechanism, and system"},{"line_number":443,"context_line":"architecture. It is proposed to have a modular design, where functionality"},{"line_number":444,"context_line":"relating to a specific factor is provided by classes specific to that"},{"line_number":445,"context_line":"factor. For example, there would be classes for specific device types::"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":" class VANImplTypeGPU(Object): # methods here are only examples"},{"line_number":448,"context_line":"    def create_vgpu(...):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_6800de67","line":445,"range":{"start_line":443,"start_character":14,"end_line":445,"end_character":7},"updated":"2018-06-28 00:46:16.000000000","message":"The content from here to L483 may be attempting to go into too much detail on something that\u0027s not part of the os-acc contract itself.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":440,"context_line":""},{"line_number":441,"context_line":"The main factors involved in attaching an accelerator (see VANfactors_)"},{"line_number":442,"context_line":"are the device type, hypervisor type, mechanism, and system"},{"line_number":443,"context_line":"architecture. It is proposed to have a modular design, where functionality"},{"line_number":444,"context_line":"relating to a specific factor is provided by classes specific to that"},{"line_number":445,"context_line":"factor. For example, there would be classes for specific device types::"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":" class VANImplTypeGPU(Object): # methods here are only examples"},{"line_number":448,"context_line":"    def create_vgpu(...):"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_d170a3a7","line":445,"range":{"start_line":443,"start_character":14,"end_line":445,"end_character":7},"in_reply_to":"5f7c97a3_6800de67","updated":"2018-06-29 02:23:10.000000000","message":"This doc introduces the notion of plugins, so we may want to flesh that out a bit.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"d56dea2295b66c7c1e325335f37a1cabbb823125","unresolved":false,"context_lines":[{"line_number":517,"context_line":""},{"line_number":518,"context_line":"    def unplug(self, van, instance_info):"},{"line_number":519,"context_line":"       pass"},{"line_number":520,"context_line":""},{"line_number":521,"context_line":"The semantics of these APIs are specified below."},{"line_number":522,"context_line":""},{"line_number":523,"context_line":"* initialize()"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_096014cf","line":520,"updated":"2018-06-30 02:18:24.000000000","message":"Do we need a reset() and deleteVAN()? unless both of these are covered implicitly by other steps.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":517,"context_line":""},{"line_number":518,"context_line":"    def unplug(self, van, instance_info):"},{"line_number":519,"context_line":"       pass"},{"line_number":520,"context_line":""},{"line_number":521,"context_line":"The semantics of these APIs are specified below."},{"line_number":522,"context_line":""},{"line_number":523,"context_line":"* initialize()"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_fa52d335","line":520,"in_reply_to":"5f7c97a3_096014cf","updated":"2018-07-06 02:23:20.000000000","message":"Are you talking of device reset? If so, that is not a call from n-cpu to os-acc, which is what this section covers.\n\nSimilarly, VANs can be deleted, but I don\u0027t see why n-cpu would call os-acc to delete a VAN. It would call unplug() which, among other things, would remove the db entries for that VAN and then delete the VAN.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":536,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":537,"context_line":"    if Cyborg Agent cannot be contacted."},{"line_number":538,"context_line":""},{"line_number":539,"context_line":"* getVAN(self, device_rp, extra_specs, instance_info)"},{"line_number":540,"context_line":""},{"line_number":541,"context_line":"  * device_rp: The nested RP selected by the scheduler is"},{"line_number":542,"context_line":"      needed for Cyborg to correlate it to a deployable. Read-only."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_561984cf","line":539,"range":{"start_line":539,"start_character":2,"end_line":539,"end_character":8},"updated":"2018-07-27 21:36:42.000000000","message":"This is what you\u0027ve been calling the \"device half\", right?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":536,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":537,"context_line":"    if Cyborg Agent cannot be contacted."},{"line_number":538,"context_line":""},{"line_number":539,"context_line":"* getVAN(self, device_rp, extra_specs, instance_info)"},{"line_number":540,"context_line":""},{"line_number":541,"context_line":"  * device_rp: The nested RP selected by the scheduler is"},{"line_number":542,"context_line":"      needed for Cyborg to correlate it to a deployable. Read-only."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_9d2ffc86","line":539,"range":{"start_line":539,"start_character":2,"end_line":539,"end_character":8},"in_reply_to":"5f7c97a3_561984cf","updated":"2018-08-14 17:14:37.000000000","message":"Yes. It has been renamed as prepareVAN() in change set 9.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":544,"context_line":"  * extra_specs: the extra_specs field in the flavor, including"},{"line_number":545,"context_line":"      fields interpreted by Cyborg but not Nova. Read-only."},{"line_number":546,"context_line":""},{"line_number":547,"context_line":"  * instance_info: Nova\u0027s per-instance dictionary. Read-only."},{"line_number":548,"context_line":""},{"line_number":549,"context_line":"    * The VM image UUID may be used to query Glance for metadata about"},{"line_number":550,"context_line":"      accelerator requirements that may be stored with the VM image"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_3615e89b","line":547,"range":{"start_line":547,"start_character":39,"end_line":547,"end_character":49},"updated":"2018-07-27 21:36:42.000000000","message":"Not a dictionary, a versioned object as in os-vif","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":544,"context_line":"  * extra_specs: the extra_specs field in the flavor, including"},{"line_number":545,"context_line":"      fields interpreted by Cyborg but not Nova. Read-only."},{"line_number":546,"context_line":""},{"line_number":547,"context_line":"  * instance_info: Nova\u0027s per-instance dictionary. Read-only."},{"line_number":548,"context_line":""},{"line_number":549,"context_line":"    * The VM image UUID may be used to query Glance for metadata about"},{"line_number":550,"context_line":"      accelerator requirements that may be stored with the VM image"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_1d59ac33","line":547,"range":{"start_line":547,"start_character":39,"end_line":547,"end_character":49},"in_reply_to":"5f7c97a3_3615e89b","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":593,"context_line":"    plugins, this call attaches a Neutron port to a bridge, OVS or"},{"line_number":594,"context_line":"    SR-IOV function. The functionality here is different.)"},{"line_number":595,"context_line":""},{"line_number":596,"context_line":"  * Returns a hypervisor-specific value on success. For example, for"},{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c84a52ac","line":598,"range":{"start_line":596,"start_character":4,"end_line":598,"end_character":33},"updated":"2018-06-28 00:46:16.000000000","message":"No, see above.  Should return the VAN, which has been updated with whatever hypervisor-specific data is necessary, embedded within the \u0027profile\u0027 attribute of the VAN.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":593,"context_line":"    plugins, this call attaches a Neutron port to a bridge, OVS or"},{"line_number":594,"context_line":"    SR-IOV function. The functionality here is different.)"},{"line_number":595,"context_line":""},{"line_number":596,"context_line":"  * Returns a hypervisor-specific value on success. For example, for"},{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c841db65","line":598,"range":{"start_line":596,"start_character":4,"end_line":598,"end_character":33},"in_reply_to":"5f7c97a3_0cdff966","updated":"2018-06-28 18:33:16.000000000","message":"[Later] Having looked at os-vif, it should in fact return nothing.  The caller can query the VAN object (which was updated by plug()) if something is needed after the fact.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":593,"context_line":"    plugins, this call attaches a Neutron port to a bridge, OVS or"},{"line_number":594,"context_line":"    SR-IOV function. The functionality here is different.)"},{"line_number":595,"context_line":""},{"line_number":596,"context_line":"  * Returns a hypervisor-specific value on success. For example, for"},{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_7d6220da","line":598,"range":{"start_line":596,"start_character":4,"end_line":598,"end_character":33},"in_reply_to":"5f7c97a3_91a78b0a","updated":"2018-08-14 17:14:37.000000000","message":"In change set 9, this returns an attach profile as part of the VAN.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":593,"context_line":"    plugins, this call attaches a Neutron port to a bridge, OVS or"},{"line_number":594,"context_line":"    SR-IOV function. The functionality here is different.)"},{"line_number":595,"context_line":""},{"line_number":596,"context_line":"  * Returns a hypervisor-specific value on success. For example, for"},{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_91a78b0a","line":598,"range":{"start_line":596,"start_character":4,"end_line":598,"end_character":33},"in_reply_to":"5f7c97a3_c841db65","updated":"2018-06-29 02:23:10.000000000","message":"The returned VAN object may belong to a child class of the VAN object that was passed in. E.g. a VANBase is passed in, and VANTypePCI object is returned. Also, it is cleaner to have a return value -- the plugin could clone the original VAN object or otherwise do whatever it pleases.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":593,"context_line":"    plugins, this call attaches a Neutron port to a bridge, OVS or"},{"line_number":594,"context_line":"    SR-IOV function. The functionality here is different.)"},{"line_number":595,"context_line":""},{"line_number":596,"context_line":"  * Returns a hypervisor-specific value on success. For example, for"},{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_0cdff966","line":598,"range":{"start_line":596,"start_character":4,"end_line":598,"end_character":33},"in_reply_to":"5f7c97a3_c84a52ac","updated":"2018-06-28 06:12:44.000000000","message":"OK","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."},{"line_number":602,"context_line":""},{"line_number":603,"context_line":"* unplug(self, van, instance_info)"},{"line_number":604,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_a8c7d611","line":601,"range":{"start_line":600,"start_character":4,"end_line":601,"end_character":62},"updated":"2018-06-28 00:46:16.000000000","message":"see above, the plug() method should *do* the attach, end of story.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."},{"line_number":602,"context_line":""},{"line_number":603,"context_line":"* unplug(self, van, instance_info)"},{"line_number":604,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_f67f70db","line":601,"range":{"start_line":600,"start_character":4,"end_line":601,"end_character":62},"in_reply_to":"5f7c97a3_6cc78d9a","updated":"2018-07-27 21:36:42.000000000","message":"Okay, I\u0027m willing to see how this shakes out in implementation.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":597,"context_line":"    libvirt, it may return an XML snippet with PCI passthrough data."},{"line_number":598,"context_line":"    Throws exceptions on failure."},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * NOTE: Nova virt drivers need to use the return value of ``plug()``"},{"line_number":601,"context_line":"    to complete the attach of the accelerator to the instance."},{"line_number":602,"context_line":""},{"line_number":603,"context_line":"* unplug(self, van, instance_info)"},{"line_number":604,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_6cc78d9a","line":601,"range":{"start_line":600,"start_character":4,"end_line":601,"end_character":62},"in_reply_to":"5f7c97a3_a8c7d611","updated":"2018-06-28 06:12:44.000000000","message":"Please see my response to earlier comment.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":608,"context_line":""},{"line_number":609,"context_line":"  * As part of this call, Cyborg Agent will clean up internal resources,"},{"line_number":610,"context_line":"    call the appropriate Cyborg driver to clean up the device resources"},{"line_number":611,"context_line":"    and update its data structures persistently."},{"line_number":612,"context_line":""},{"line_number":613,"context_line":"  * Returns None on success, throws exceptions on failure."},{"line_number":614,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_d653546d","line":611,"updated":"2018-07-27 21:36:42.000000000","message":"So does this clean up both the \"instance half\" and the \"device half\"?\n\nI wonder if we should make it symmetrical: have a destroyVAN to correspond to getVAN. That way if we need to \"reuse\" the *same* VAN (e.g. for that goofy libvirt reboot thing referenced above) we have a way of doing it.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":608,"context_line":""},{"line_number":609,"context_line":"  * As part of this call, Cyborg Agent will clean up internal resources,"},{"line_number":610,"context_line":"    call the appropriate Cyborg driver to clean up the device resources"},{"line_number":611,"context_line":"    and update its data structures persistently."},{"line_number":612,"context_line":""},{"line_number":613,"context_line":"  * Returns None on success, throws exceptions on failure."},{"line_number":614,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_1da76c0d","line":611,"in_reply_to":"5f7c97a3_d653546d","updated":"2018-08-14 17:14:37.000000000","message":"Yes","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":623,"context_line":"flexible mechanism to handle plugins. That is proposed below."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"For an attach request, the os-acc will choose a plugin by querying every"},{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_68ea1e75","line":626,"range":{"start_line":626,"start_character":17,"end_line":626,"end_character":29},"updated":"2018-06-28 00:46:16.000000000","message":"Oh, please no.  Poor zVM.  Should be in the order in which they\u0027re specified in the cfg.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":623,"context_line":"flexible mechanism to handle plugins. That is proposed below."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"For an attach request, the os-acc will choose a plugin by querying every"},{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_96d63cc4","line":626,"range":{"start_line":626,"start_character":17,"end_line":626,"end_character":29},"in_reply_to":"5f7c97a3_1a18efca","updated":"2018-07-27 21:36:42.000000000","message":"I think so. Again, I think I\u0027ll have to see how the impl shakes out.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":623,"context_line":"flexible mechanism to handle plugins. That is proposed below."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"For an attach request, the os-acc will choose a plugin by querying every"},{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_f3228ca3","line":626,"range":{"start_line":626,"start_character":17,"end_line":626,"end_character":29},"in_reply_to":"5f7c97a3_4c6c119d","updated":"2018-06-28 18:33:16.000000000","message":"At a glance, you\u0027re right, it appears not to be deterministic.  However, looking closer at how os-vif is being used, it looks like the driver gets to pick a plugin by name (i.e. the handle it registers to ExtensionManager).  If there\u0027s more than one alternative registered, I imagine it would be up to the driver to decide which ones it should attempt to load, and in what order.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":623,"context_line":"flexible mechanism to handle plugins. That is proposed below."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"For an attach request, the os-acc will choose a plugin by querying every"},{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_4c6c119d","line":626,"range":{"start_line":626,"start_character":17,"end_line":626,"end_character":29},"in_reply_to":"5f7c97a3_68ea1e75","updated":"2018-06-28 06:12:44.000000000","message":"Haha. I would not expect so many plugins on a given host that this is a problem.\n\nAlso, does stevedore guarantee that a loop like:\nhttps://github.com/openstack/os-vif/blob/master/os_vif/__init__.py#L41 \nwill list the plugins in the order of the config file?\n\nI just picked \u0027alphabetical\u0027 as one way to ensure a deterministic order, and by analogy with rc scripts in Linux. Any deterministic order is good enough.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"5cfd984c993def9052f55f764ffb5fd26c17f642","unresolved":false,"context_lines":[{"line_number":623,"context_line":"flexible mechanism to handle plugins. That is proposed below."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"For an attach request, the os-acc will choose a plugin by querying every"},{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_b7862016","line":626,"range":{"start_line":626,"start_character":17,"end_line":626,"end_character":29},"in_reply_to":"5f7c97a3_b1ee471f","updated":"2018-06-29 21:33:04.000000000","message":"The nova virt drivers don\u0027t know about specific plugins.  They know a name - basically an alias - and the operator has to install an appropriate plugin and configure it with that name.  The operator (obviously) has to know what she\u0027s doing and get it right; but could theoretically install one of any number of different plugins under that name.\n\nI do see the merit in being able to install several different plugins that the driver would be able to try.  But again, the driver would be trying *names* (not specific plugins) based on known criteria - or at least in a particular order determined at dev time.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"53abd863746a51c36705ab1815d755fc163bef83","unresolved":false,"context_lines":[{"line_number":623,"context_line":"flexible mechanism to handle plugins. That is proposed below."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"For an attach request, the os-acc will choose a plugin by querying every"},{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_1a18efca","line":626,"range":{"start_line":626,"start_character":17,"end_line":626,"end_character":29},"in_reply_to":"5f7c97a3_b7862016","updated":"2018-07-06 02:23:20.000000000","message":"To clarify, it is not the Nova virt driver that is trying each plugin -- it is os-acc. So, shall we say this point is agreed upon?","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":623,"context_line":"flexible mechanism to handle plugins. That is proposed below."},{"line_number":624,"context_line":""},{"line_number":625,"context_line":"For an attach request, the os-acc will choose a plugin by querying every"},{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_b1ee471f","line":626,"range":{"start_line":626,"start_character":17,"end_line":626,"end_character":29},"in_reply_to":"5f7c97a3_f3228ca3","updated":"2018-06-29 02:23:10.000000000","message":"I\u0027d have to disagree :(. There are few os-vif plugins and Nova virt knows about them, so it is ok for the virt driver to pick a os-vif plugin. The number of os-acc plugins could be much larger, and the virt driver would not (should not) know about them, as they could be vendor-specific. That is why we have the plugin chain model.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"},{"line_number":630,"context_line":"``setup.cfg`` file (similar to rc scripts in Linux)."},{"line_number":631,"context_line":""},{"line_number":632,"context_line":"Since the number of os-acc plugins in a given host is expected to be"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_689f7e02","line":629,"range":{"start_line":629,"start_character":44,"end_line":629,"end_character":62},"updated":"2018-06-28 00:46:16.000000000","message":"ick, just go in order specified","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":626,"context_line":"loaded plugin in alphabetical order of plugin name to check if it can"},{"line_number":627,"context_line":"handle that request. The first plugin to \u0027claim\u0027 the request gets to handle"},{"line_number":628,"context_line":"it. If there are two or more plugins that can potentially handle the"},{"line_number":629,"context_line":"request, the operator can prioritize one by naming it suitably in the"},{"line_number":630,"context_line":"``setup.cfg`` file (similar to rc scripts in Linux)."},{"line_number":631,"context_line":""},{"line_number":632,"context_line":"Since the number of os-acc plugins in a given host is expected to be"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_ec687d89","line":629,"range":{"start_line":629,"start_character":44,"end_line":629,"end_character":62},"in_reply_to":"5f7c97a3_689f7e02","updated":"2018-06-28 06:12:44.000000000","message":"Please see above","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"04cb273f411084cbbefbe4b5d7003ab465277356","unresolved":false,"context_lines":[{"line_number":650,"context_line":"     def unplug(self, van, instance_info):"},{"line_number":651,"context_line":"        raise NotImplementedError"},{"line_number":652,"context_line":""},{"line_number":653,"context_line":"     def describe(self):"},{"line_number":654,"context_line":"        raise NotImplementedError"},{"line_number":655,"context_line":""},{"line_number":656,"context_line":"     @classmethod"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_36f1686b","line":653,"range":{"start_line":653,"start_character":9,"end_line":653,"end_character":17},"updated":"2018-07-27 21:36:42.000000000","message":"what\u0027s this? It\u0027s not described :P","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":650,"context_line":"     def unplug(self, van, instance_info):"},{"line_number":651,"context_line":"        raise NotImplementedError"},{"line_number":652,"context_line":""},{"line_number":653,"context_line":"     def describe(self):"},{"line_number":654,"context_line":"        raise NotImplementedError"},{"line_number":655,"context_line":""},{"line_number":656,"context_line":"     @classmethod"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_7de54037","line":653,"range":{"start_line":653,"start_character":9,"end_line":653,"end_character":17},"in_reply_to":"5f7c97a3_36f1686b","updated":"2018-08-14 17:14:37.000000000","message":"It was supposed to be the same method as in os-vif. Dropped it in change set 9.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":678,"context_line":"To handle the device/vendor-specific aspects, which are already handled by"},{"line_number":679,"context_line":"Cyborg drivers, there are two options:"},{"line_number":680,"context_line":""},{"line_number":681,"context_line":"* Have the plugin call into Cyborg agent, which in turn calls into the"},{"line_number":682,"context_line":"  Cyborg drivers. Since the plugin has to call Cyborg anyway to have"},{"line_number":683,"context_line":"  the VAN persisted, this is the most reasonable option."},{"line_number":684,"context_line":""},{"line_number":685,"context_line":"* Have Cyborg drivers provide a plugin interface as well, so that they"},{"line_number":686,"context_line":"  can be loaded by both the plugin and Cyborg agent."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_c8549225","line":683,"range":{"start_line":681,"start_character":0,"end_line":683,"end_character":56},"updated":"2018-06-28 00:46:16.000000000","message":"see above","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":678,"context_line":"To handle the device/vendor-specific aspects, which are already handled by"},{"line_number":679,"context_line":"Cyborg drivers, there are two options:"},{"line_number":680,"context_line":""},{"line_number":681,"context_line":"* Have the plugin call into Cyborg agent, which in turn calls into the"},{"line_number":682,"context_line":"  Cyborg drivers. Since the plugin has to call Cyborg anyway to have"},{"line_number":683,"context_line":"  the VAN persisted, this is the most reasonable option."},{"line_number":684,"context_line":""},{"line_number":685,"context_line":"* Have Cyborg drivers provide a plugin interface as well, so that they"},{"line_number":686,"context_line":"  can be loaded by both the plugin and Cyborg agent."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_732e7c8d","line":683,"range":{"start_line":681,"start_character":0,"end_line":683,"end_character":56},"in_reply_to":"5f7c97a3_0c14d9e8","updated":"2018-06-28 18:33:16.000000000","message":"I think I again need to understand what a \"driver\" is in this context.  And again, I\u0027m taking a firm stance on the unidirectionality of the communication sequences.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":678,"context_line":"To handle the device/vendor-specific aspects, which are already handled by"},{"line_number":679,"context_line":"Cyborg drivers, there are two options:"},{"line_number":680,"context_line":""},{"line_number":681,"context_line":"* Have the plugin call into Cyborg agent, which in turn calls into the"},{"line_number":682,"context_line":"  Cyborg drivers. Since the plugin has to call Cyborg anyway to have"},{"line_number":683,"context_line":"  the VAN persisted, this is the most reasonable option."},{"line_number":684,"context_line":""},{"line_number":685,"context_line":"* Have Cyborg drivers provide a plugin interface as well, so that they"},{"line_number":686,"context_line":"  can be loaded by both the plugin and Cyborg agent."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_31e23732","line":683,"range":{"start_line":681,"start_character":0,"end_line":683,"end_character":56},"in_reply_to":"5f7c97a3_732e7c8d","updated":"2018-06-29 02:23:10.000000000","message":"Please see the link to Cyborg architecture, including drivers, that I mentioned in a previous comment:\nhttps://wiki.openstack.org/wiki/Cyborg#Pike.2FQueens_Initial_Architecture","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":678,"context_line":"To handle the device/vendor-specific aspects, which are already handled by"},{"line_number":679,"context_line":"Cyborg drivers, there are two options:"},{"line_number":680,"context_line":""},{"line_number":681,"context_line":"* Have the plugin call into Cyborg agent, which in turn calls into the"},{"line_number":682,"context_line":"  Cyborg drivers. Since the plugin has to call Cyborg anyway to have"},{"line_number":683,"context_line":"  the VAN persisted, this is the most reasonable option."},{"line_number":684,"context_line":""},{"line_number":685,"context_line":"* Have Cyborg drivers provide a plugin interface as well, so that they"},{"line_number":686,"context_line":"  can be loaded by both the plugin and Cyborg agent."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_0c14d9e8","line":683,"range":{"start_line":681,"start_character":0,"end_line":683,"end_character":56},"in_reply_to":"5f7c97a3_c8549225","updated":"2018-06-28 06:12:44.000000000","message":"Will drop the \u0027most reasonable\u0027 part. But the explicit premise of this paragraph is that a plugin may want to leverage existing functionality in Cyborg drivers. If that is not the case, it can do anything anyway.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":700,"context_line":""},{"line_number":701,"context_line":"No changes are required in Glance to support Cyborg."},{"line_number":702,"context_line":""},{"line_number":703,"context_line":"In the future, when Glance images can have traits, the accelerator"},{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_a831160d","line":703,"range":{"start_line":703,"start_character":0,"end_line":703,"end_character":13},"updated":"2018-06-28 00:46:16.000000000","message":"this is already a thing, I think.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":700,"context_line":""},{"line_number":701,"context_line":"No changes are required in Glance to support Cyborg."},{"line_number":702,"context_line":""},{"line_number":703,"context_line":"In the future, when Glance images can have traits, the accelerator"},{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_3d3a6898","line":703,"range":{"start_line":703,"start_character":0,"end_line":703,"end_character":13},"in_reply_to":"5f7c97a3_a831160d","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":701,"context_line":"No changes are required in Glance to support Cyborg."},{"line_number":702,"context_line":""},{"line_number":703,"context_line":"In the future, when Glance images can have traits, the accelerator"},{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_48364205","line":704,"range":{"start_line":704,"start_character":61,"end_line":704,"end_character":71},"updated":"2018-06-28 00:46:16.000000000","message":"or in addition to","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":701,"context_line":"No changes are required in Glance to support Cyborg."},{"line_number":702,"context_line":""},{"line_number":703,"context_line":"In the future, when Glance images can have traits, the accelerator"},{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_5d3de48c","line":704,"range":{"start_line":704,"start_character":61,"end_line":704,"end_character":71},"in_reply_to":"5f7c97a3_2ce2f503","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":701,"context_line":"No changes are required in Glance to support Cyborg."},{"line_number":702,"context_line":""},{"line_number":703,"context_line":"In the future, when Glance images can have traits, the accelerator"},{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_2ce2f503","line":704,"range":{"start_line":704,"start_character":61,"end_line":704,"end_character":71},"in_reply_to":"5f7c97a3_48364205","updated":"2018-06-28 06:12:44.000000000","message":"OK","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"},{"line_number":708,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":709,"context_line":"Cyborg needs to provide REST APIs to update allocations/release of"},{"line_number":710,"context_line":"VAN objects. They are as below::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_a81a7679","line":707,"range":{"start_line":707,"start_character":0,"end_line":707,"end_character":22},"updated":"2018-06-28 00:46:16.000000000","message":"I don\u0027t understand this section.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"},{"line_number":708,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":709,"context_line":"Cyborg needs to provide REST APIs to update allocations/release of"},{"line_number":710,"context_line":"VAN objects. They are as below::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_b3d5f449","line":707,"range":{"start_line":707,"start_character":0,"end_line":707,"end_character":22},"in_reply_to":"5f7c97a3_6c7f8d17","updated":"2018-06-28 18:33:16.000000000","message":"Okay.  This is out of the scope I\u0027m concerned with, but have you considered making the VAN the first-class object in the URI?  That is, rather than having to define for each {type} a separate\n\n /{type}/{deployable_uuid}/vans\n\nroute and handler, you would instead just have\n\n /vans\n\nThen the {type} and {deployable_uuid}, as well as all the other attributes (i.e. the serialized VAN* object, I would imagine) would be part of the payload.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"41f3b7a123b9d77c73c2e82b651ac178fe9294ea","unresolved":false,"context_lines":[{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"},{"line_number":708,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":709,"context_line":"Cyborg needs to provide REST APIs to update allocations/release of"},{"line_number":710,"context_line":"VAN objects. They are as below::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f79a3b5_3d6348b4","line":707,"range":{"start_line":707,"start_character":0,"end_line":707,"end_character":22},"in_reply_to":"5f7c97a3_71cbcf72","updated":"2018-08-14 17:14:37.000000000","message":"Done","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e6df2c265544f469e37246c545f94a594e980cd2","unresolved":false,"context_lines":[{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"},{"line_number":708,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":709,"context_line":"Cyborg needs to provide REST APIs to update allocations/release of"},{"line_number":710,"context_line":"VAN objects. They are as below::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_6c7f8d17","line":707,"range":{"start_line":707,"start_character":0,"end_line":707,"end_character":22},"in_reply_to":"5f7c97a3_a81a7679","updated":"2018-06-28 06:12:44.000000000","message":"Cyborg DB is used to store the VAN objects persistently. This is the REST API to update it. The os-acc could call this API.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":704,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":705,"context_line":"traits in the flavor."},{"line_number":706,"context_line":""},{"line_number":707,"context_line":"Cyborg REST API Update"},{"line_number":708,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":709,"context_line":"Cyborg needs to provide REST APIs to update allocations/release of"},{"line_number":710,"context_line":"VAN objects. They are as below::"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_71cbcf72","line":707,"range":{"start_line":707,"start_character":0,"end_line":707,"end_character":22},"in_reply_to":"5f7c97a3_b3d5f449","updated":"2018-06-29 02:23:10.000000000","message":"I think I agree that the device type doesn\u0027t belong here. \n\nFor a PUT or DELETE, it is important to say which device RP (Deployable) is being targeted. But I guess it could be in the payload as a mandatory field. I\u0027ll have to think about what that means.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"bef9418eed22f779db310c953dc63d7a9038d838","unresolved":false,"context_lines":[{"line_number":715,"context_line":"     URL Params:"},{"line_number":716,"context_line":"       None"},{"line_number":717,"context_line":""},{"line_number":718,"context_line":"     Data Params:"},{"line_number":719,"context_line":"       VAN UUID"},{"line_number":720,"context_line":""},{"line_number":721,"context_line":"     Success Response:"},{"line_number":722,"context_line":"       Code: 200 OK"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_b3f174e1","line":719,"range":{"start_line":718,"start_character":5,"end_line":719,"end_character":15},"updated":"2018-06-28 18:33:16.000000000","message":"You may want to talk about this with the API-SIG folks, but I guess I was expecting to see something more like:\n\n GET /vans\n    no payload\n    return a list of VAN objects\n GET /vans/{van_uuid}\n    no payload\n    return a single VAN\n PUT /vans/{van_uuid}\n    payload: { serialized VAN object }\n    create or update a VAN\n DELETE /vans/{van_uuid}\n    no payload\n    delete a VAN","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"50f8aa342387aa620ed3b9d6ba075c4b009b0013","unresolved":false,"context_lines":[{"line_number":715,"context_line":"     URL Params:"},{"line_number":716,"context_line":"       None"},{"line_number":717,"context_line":""},{"line_number":718,"context_line":"     Data Params:"},{"line_number":719,"context_line":"       VAN UUID"},{"line_number":720,"context_line":""},{"line_number":721,"context_line":"     Success Response:"},{"line_number":722,"context_line":"       Code: 200 OK"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_11bdbbd6","line":719,"range":{"start_line":718,"start_character":5,"end_line":719,"end_character":15},"in_reply_to":"5f7c97a3_33900443","updated":"2018-06-29 02:23:10.000000000","message":"The main functionality we need is the add/delete of VANs to a single device RP (Deployable in Cyborg parlance). The Deployable UUID has to figure here.","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"6b9598c234b3e959171fb2a00d827e122e484d0e","unresolved":false,"context_lines":[{"line_number":715,"context_line":"     URL Params:"},{"line_number":716,"context_line":"       None"},{"line_number":717,"context_line":""},{"line_number":718,"context_line":"     Data Params:"},{"line_number":719,"context_line":"       VAN UUID"},{"line_number":720,"context_line":""},{"line_number":721,"context_line":"     Success Response:"},{"line_number":722,"context_line":"       Code: 200 OK"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_33900443","line":719,"range":{"start_line":718,"start_character":5,"end_line":719,"end_character":15},"in_reply_to":"5f7c97a3_b3f174e1","updated":"2018-06-28 18:40:21.000000000","message":"Yes, in broad strokes that\u0027s a nice simple way to talk about them","commit_id":"74ede43ce86a2db447dedbf2ecf8f9fc626f55df"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"3f79a3b5_7b66b3bd","updated":"2018-08-24 21:38:48.000000000","message":"this spec should not be updated in place. a copy of the spec should be made in specs/stein/approved/ and the edits should be applied there so the the changes between releases are clear.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"3f79a3b5_f6b0f1df","in_reply_to":"3f79a3b5_7b66b3bd","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Cyborg is an OpenStack project that aims to provide a general purpose"},{"line_number":12,"context_line":"management framework for accelerators, such as FPGAs and GPUs. For"},{"line_number":13,"context_line":"scheduling an instance that needs accelerators, Cyborg shall work"},{"line_number":14,"context_line":"with Nova in three areas:"},{"line_number":15,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_552ce6aa","line":12,"range":{"start_line":12,"start_character":24,"end_line":12,"end_character":61},"updated":"2018-08-24 21:38:48.000000000","message":"i would maybe rephrase this to \"programmable devices\".","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Cyborg is an OpenStack project that aims to provide a general purpose"},{"line_number":12,"context_line":"management framework for accelerators, such as FPGAs and GPUs. For"},{"line_number":13,"context_line":"scheduling an instance that needs accelerators, Cyborg shall work"},{"line_number":14,"context_line":"with Nova in three areas:"},{"line_number":15,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_bbfd166f","line":12,"range":{"start_line":12,"start_character":24,"end_line":12,"end_character":61},"in_reply_to":"3f79a3b5_552ce6aa","updated":"2018-09-02 02:58:43.000000000","message":"Cyborg aims to include non-programmable devices such as HPTS cards as well.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":13,"context_line":"scheduling an instance that needs accelerators, Cyborg shall work"},{"line_number":14,"context_line":"with Nova in three areas:"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators"},{"line_number":17,"context_line":"  as resource classes and devices as resource providers in Placement."},{"line_number":18,"context_line":"  When a device is discovered, Cyborg shall update resource inventories"},{"line_number":19,"context_line":"  for those providers in Placement."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_40042615","line":16,"range":{"start_line":16,"start_character":55,"end_line":16,"end_character":67},"updated":"2018-08-24 21:38:48.000000000","message":"the type of an acclerator","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":13,"context_line":"scheduling an instance that needs accelerators, Cyborg shall work"},{"line_number":14,"context_line":"with Nova in three areas:"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators"},{"line_number":17,"context_line":"  as resource classes and devices as resource providers in Placement."},{"line_number":18,"context_line":"  When a device is discovered, Cyborg shall update resource inventories"},{"line_number":19,"context_line":"  for those providers in Placement."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_3b1fc6b7","line":16,"range":{"start_line":16,"start_character":55,"end_line":16,"end_character":67},"in_reply_to":"3f79a3b5_40042615","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators"},{"line_number":17,"context_line":"  as resource classes and devices as resource providers in Placement."},{"line_number":18,"context_line":"  When a device is discovered, Cyborg shall update resource inventories"},{"line_number":19,"context_line":"  for those providers in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: In the controller node, Cyborg *may*"},{"line_number":22,"context_line":"  provide a weigher that prioritizes hosts based on available"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_60b60241","line":19,"range":{"start_line":18,"start_character":1,"end_line":19,"end_character":35},"updated":"2018-08-24 21:38:48.000000000","message":"that would be one model but perhaps cyborg should not interact with placement directly. what might be a better model would be for nova in the virt driver to call cyborg as a lib as part of the update_provider_tree function to add the devices to the other inventories it is createing for the compute node.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators"},{"line_number":17,"context_line":"  as resource classes and devices as resource providers in Placement."},{"line_number":18,"context_line":"  When a device is discovered, Cyborg shall update resource inventories"},{"line_number":19,"context_line":"  for those providers in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: In the controller node, Cyborg *may*"},{"line_number":22,"context_line":"  provide a weigher that prioritizes hosts based on available"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_9bd7fabf","line":19,"range":{"start_line":18,"start_character":1,"end_line":19,"end_character":35},"in_reply_to":"3f79a3b5_60b60241","updated":"2018-09-02 02:58:43.000000000","message":"To handle PCI hot adds and deletes, Cyborg would periodically scan the compute node. There would be operator configurations to control the period and other factors. Further, these periods and knobs may be different for devices than for other resources. Instead of embedding those config knobs in every virt driver, it would be better to let Cyborg be the central place for all such things.\n\nSecondly, when Cyborg discovers devices, it would update its own db first before updating placement. There are several considerations in that flow to handle the deltas from one update to the next. It would be cleaner to have that logic be in Cyborg itself.\n\nBased on the above two, it is not obvious to me that coupling device updates to virt driver updates is the best way forward. The virt driver could update the top-level RP of the node while Cyborg updates device-level children RPs.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":18,"context_line":"  When a device is discovered, Cyborg shall update resource inventories"},{"line_number":19,"context_line":"  for those providers in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: In the controller node, Cyborg *may*"},{"line_number":22,"context_line":"  provide a weigher that prioritizes hosts based on available"},{"line_number":23,"context_line":"  accelerator resources."},{"line_number":24,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_20d48a47","line":21,"range":{"start_line":21,"start_character":33,"end_line":21,"end_character":56},"updated":"2018-08-24 21:38:48.000000000","message":"nit: this is technically where the weigher would be executed but i dont think this add value.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":18,"context_line":"  When a device is discovered, Cyborg shall update resource inventories"},{"line_number":19,"context_line":"  for those providers in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: In the controller node, Cyborg *may*"},{"line_number":22,"context_line":"  provide a weigher that prioritizes hosts based on available"},{"line_number":23,"context_line":"  accelerator resources."},{"line_number":24,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_fbf14e6c","line":21,"range":{"start_line":21,"start_character":33,"end_line":21,"end_character":56},"in_reply_to":"3f79a3b5_20d48a47","updated":"2018-09-02 02:58:43.000000000","message":"Since this spec is for the compute node, we are calling out what else is involved in the flow in the controller. This is to set the context.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  for those providers in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: In the controller node, Cyborg *may*"},{"line_number":22,"context_line":"  provide a weigher that prioritizes hosts based on available"},{"line_number":23,"context_line":"  accelerator resources."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Attachment of accelerators to instances: In the compute node, Cyborg"},{"line_number":26,"context_line":"  shall interact with Nova through a library named os-acc, analogous to"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_e025325d","line":23,"range":{"start_line":22,"start_character":37,"end_line":23,"end_character":24},"updated":"2018-08-24 21:38:48.000000000","message":"so the effect of this weigher would be to sort the filtered host by the number of available acclerators. if that is the case then to be consistent can i suggest teh weigher is written such that a positive multiplier results in spreading and a negitive multipler results in packing as is done in other weighers.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  for those providers in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: In the controller node, Cyborg *may*"},{"line_number":22,"context_line":"  provide a weigher that prioritizes hosts based on available"},{"line_number":23,"context_line":"  accelerator resources."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Attachment of accelerators to instances: In the compute node, Cyborg"},{"line_number":26,"context_line":"  shall interact with Nova through a library named os-acc, analogous to"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_5c3866ec","line":23,"range":{"start_line":22,"start_character":37,"end_line":23,"end_character":24},"in_reply_to":"3f79a3b5_e025325d","updated":"2018-09-02 02:58:43.000000000","message":"The weigher is intended to prioritize nodes that have free instances of requested function types, region types, etc. Sure, the weigher should follow precedents where appropriate. Given all the other tasks, it is not clear if we will get around to this optimization in Stein.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Attachment of accelerators to instances: In the compute node, Cyborg"},{"line_number":26,"context_line":"  shall interact with Nova through a library named os-acc, analogous to"},{"line_number":27,"context_line":"  os-vif (for Neutron) and os-brick (for Cinder). The behavior with"},{"line_number":28,"context_line":"  respect to accelerators for various instance operations (like start,"},{"line_number":29,"context_line":"  pause and reboot) needs to be defined explicitly. For FPGAs, the"},{"line_number":30,"context_line":"  interaction with Glance for programming needs to be laid out."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"The first two aspects are addressed in [#CyborgNovaSched]_. This spec"},{"line_number":33,"context_line":"addresses the third. Specifically:"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a0a41aa7","line":30,"range":{"start_line":27,"start_character":49,"end_line":30,"end_character":63},"updated":"2018-08-24 21:38:48.000000000","message":"this reads as a todo.\ni would expect this to be removed with the actul workflow specified instead  before this spec update is merged.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Attachment of accelerators to instances: In the compute node, Cyborg"},{"line_number":26,"context_line":"  shall interact with Nova through a library named os-acc, analogous to"},{"line_number":27,"context_line":"  os-vif (for Neutron) and os-brick (for Cinder). The behavior with"},{"line_number":28,"context_line":"  respect to accelerators for various instance operations (like start,"},{"line_number":29,"context_line":"  pause and reboot) needs to be defined explicitly. For FPGAs, the"},{"line_number":30,"context_line":"  interaction with Glance for programming needs to be laid out."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"The first two aspects are addressed in [#CyborgNovaSched]_. This spec"},{"line_number":33,"context_line":"addresses the third. Specifically:"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_fcfff2fe","line":30,"range":{"start_line":27,"start_character":49,"end_line":30,"end_character":63},"in_reply_to":"3f79a3b5_a0a41aa7","updated":"2018-09-02 02:58:43.000000000","message":"Not a to-do. This is the overall problem statement, and the rest of the overview section states which of those are addressed in this spec, and which are in other specs. Will rephrase to clarify.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":29,"context_line":"  pause and reboot) needs to be defined explicitly. For FPGAs, the"},{"line_number":30,"context_line":"  interaction with Glance for programming needs to be laid out."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"The first two aspects are addressed in [#CyborgNovaSched]_. This spec"},{"line_number":33,"context_line":"addresses the third. Specifically:"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"* It defines the behavior with respect to accelerators for various"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_4067464e","line":32,"range":{"start_line":32,"start_character":0,"end_line":32,"end_character":60},"updated":"2018-08-24 21:38:48.000000000","message":"i think there are several asspects of the first two aspects, discoverablity and scheduling/placement that need futher refinement. i will focus on attachment for the rest of this spec but im not sure cyborg should interact with placement directly.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"* It defines the behavior with respect to accelerators for various"},{"line_number":36,"context_line":"  instance operations (such as start, pause and reboot). This specifies"},{"line_number":37,"context_line":"  the situations when Nova must invoke os-acc to attach/detach"},{"line_number":38,"context_line":"  accelerators."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"* It defines the structure of os-acc. It also defines the Virtual"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a08adabb","line":37,"range":{"start_line":37,"start_character":49,"end_line":37,"end_character":62},"updated":"2018-08-24 21:38:48.000000000","message":"we might need to decouple this futher.\n\ni had orgninally enviouned at least 7 seperate operations.\n\ninvetory_devices -\u003e called via update_porvider_tree in virt driver.\nallocate_device -\u003e assigns a acclorator_fuction to a device in the db.\nprepare_device -\u003e instructs cyborg to prepare the assigned acclorater function for the guest to use.\nattach_device -\u003e virt driver fuction that ataches the device to the vm.\ndetach_device -\u003e inverse of attach_device\ncleanup_device -\u003e instructs cyborg reset the device.\ndeallocate_devices -\u003e releases the device in the db for another request to consume.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"* It defines the behavior with respect to accelerators for various"},{"line_number":36,"context_line":"  instance operations (such as start, pause and reboot). This specifies"},{"line_number":37,"context_line":"  the situations when Nova must invoke os-acc to attach/detach"},{"line_number":38,"context_line":"  accelerators."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"* It defines the structure of os-acc. It also defines the Virtual"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_c247b04b","line":37,"range":{"start_line":37,"start_character":49,"end_line":37,"end_character":62},"in_reply_to":"3f79a3b5_a08adabb","updated":"2018-09-02 02:58:43.000000000","message":"This section is a summary. As the rest of the spec explains, there are indeed more granular steps than attach/detach.\n\nOn a re-reading, the acquire/allocate vs attach distinction (and release/deallocate vs detach) needs to be better brought out in the spec. Will update.\n\nThe inventorying of devices is done by Cyborg, outside of os-acc, and is out of scope for this spec.\n\nThe allocate and prepare should not be separate steps. For a device with many accelerator instances and many VFs, the relationship may not be static: we may need to pick an accelerator, pick a VF and configure the device to associate the two. Only then can that VF be used to access that accelerator. For FPGAs, the device may need to be programmed before it has the accelerators of the right type. These are part of the prepareVAN() step in the spec.\n\nThere are many aspects and types of cleanup. Some of them may happen on detach and some on deallocate. They should be left to the device/vendor-specific implementation. IOW, detach and deallocate steps include the appropriate device updates for cleanup. So, we don\u0027t need a separate cleanup step.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":37,"context_line":"  the situations when Nova must invoke os-acc to attach/detach"},{"line_number":38,"context_line":"  accelerators."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"* It defines the structure of os-acc. It also defines the Virtual"},{"line_number":41,"context_line":"  Accelerator Nexus (VAN), the handle object representing a means by"},{"line_number":42,"context_line":"  which an accelerator can be attached to an instance (VM, container or"},{"line_number":43,"context_line":"  process)."},{"line_number":44,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_dbf02700","line":41,"range":{"start_line":40,"start_character":58,"end_line":41,"end_character":20},"updated":"2018-08-24 21:38:48.000000000","message":"is this a preexisting term as its rather confusing.\n\nfirstly virutal usually implies a software resouce but we are in general talking about physical resources when discribing a cyborg controled devices. \n\na nexus is a convergence/meeing point\nit also appears in a lot of trademaks.\ne.g. google nexus phone. Cisco nexus switchs ectra.\n\ncan i suggest using another term.\n\nfor example accelerator_context or accelerator_descriptor\n\nif you really want to keep the word virtual then\nvirtual accelerator connection (VAC) would be much more clear as that implies its a connection between the virutalised instance and the accelerator.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":37,"context_line":"  the situations when Nova must invoke os-acc to attach/detach"},{"line_number":38,"context_line":"  accelerators."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"* It defines the structure of os-acc. It also defines the Virtual"},{"line_number":41,"context_line":"  Accelerator Nexus (VAN), the handle object representing a means by"},{"line_number":42,"context_line":"  which an accelerator can be attached to an instance (VM, container or"},{"line_number":43,"context_line":"  process)."},{"line_number":44,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_428160fa","line":41,"range":{"start_line":40,"start_character":58,"end_line":41,"end_character":20},"in_reply_to":"3f79a3b5_dbf02700","updated":"2018-09-02 02:58:43.000000000","message":"Merriam Webster and other dictionaries define a nexus as a connection or a link. Better to stay away from hugely overloaded terms like context/descriptor. The term \u0027connection\u0027 is also apparently a trademark (https://trademarks.justia.com/872/42/connection-87242249.html).\n\nA VAN is a software object, and hence virtual.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":50,"context_line":"  This spec will address the interaction of Cyborg and Glance"},{"line_number":51,"context_line":"  in the compute node."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"This spec represents an update to the earlier Compute Node spec"},{"line_number":54,"context_line":"[#ComputeNodeSpec]_."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"This spec is common to all accelerators, including GPUs, High Precision"},{"line_number":57,"context_line":"Time Synchronization (HPTS) cards, etc. Since FPGAs have more aspects"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_7bb8f30f","line":54,"range":{"start_line":53,"start_character":0,"end_line":54,"end_character":20},"updated":"2018-08-24 21:38:48.000000000","message":"this can be deleted. it should be tracked in the history at the bottom of the doc on line 1033.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":50,"context_line":"  This spec will address the interaction of Cyborg and Glance"},{"line_number":51,"context_line":"  in the compute node."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"This spec represents an update to the earlier Compute Node spec"},{"line_number":54,"context_line":"[#ComputeNodeSpec]_."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"This spec is common to all accelerators, including GPUs, High Precision"},{"line_number":57,"context_line":"Time Synchronization (HPTS) cards, etc. Since FPGAs have more aspects"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_e2bd0cb1","line":54,"range":{"start_line":53,"start_character":0,"end_line":54,"end_character":20},"in_reply_to":"3f79a3b5_7bb8f30f","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":58,"context_line":"to be considered than other devices, some sections may focus on"},{"line_number":59,"context_line":"FPGA-specific factors. The spec calls out the FPGA-specific aspects."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Terminology"},{"line_number":62,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":63,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":64,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_9b05cf48","line":61,"range":{"start_line":61,"start_character":0,"end_line":61,"end_character":11},"updated":"2018-08-24 21:38:48.000000000","message":"i would move this higher to be honest as this context is need to properly understand the preable.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":58,"context_line":"to be considered than other devices, some sections may focus on"},{"line_number":59,"context_line":"FPGA-specific factors. The spec calls out the FPGA-specific aspects."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Terminology"},{"line_number":62,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":63,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":64,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_c0e30b34","line":61,"range":{"start_line":61,"start_character":0,"end_line":61,"end_character":11},"in_reply_to":"3f79a3b5_9b05cf48","updated":"2018-09-02 02:58:43.000000000","message":"Understood. But then most people would expect the overview to come first. I\u0027ll try to thread the needle by referring to this section early in the overview.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Terminology"},{"line_number":62,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":63,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":64,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"},{"line_number":65,"context_line":"  device itself or a virtualized version of it (e.g. vGPUs). For FPGAs, an"},{"line_number":66,"context_line":"  accelerator is either the entire device, a region within the device or a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_862ed689","line":63,"range":{"start_line":63,"start_character":2,"end_line":63,"end_character":13},"updated":"2018-08-24 21:38:48.000000000","message":"i would probably call this an accelerator function.\nwhen i think accelerator i think the phyical devices not the unit of scheduling.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Terminology"},{"line_number":62,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":63,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":64,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"},{"line_number":65,"context_line":"  device itself or a virtualized version of it (e.g. vGPUs). For FPGAs, an"},{"line_number":66,"context_line":"  accelerator is either the entire device, a region within the device or a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_428080bf","line":63,"range":{"start_line":63,"start_character":2,"end_line":63,"end_character":13},"in_reply_to":"3f79a3b5_862ed689","updated":"2018-09-02 02:58:43.000000000","message":"We have used the same terminology in previous specs. When the flavor requests a resource like CUSTOM_ACCELERATOR_GPU, this is the resource that is requested. Terms like \u0027function\u0027 are used with a more specific meaning.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":69,"context_line":"* Bitstream: An FPGA image, usually a binary file, possibly with"},{"line_number":70,"context_line":"  vendor-specific metadata. A bitstream may implement one or more functions."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"* Function: A specific functionality, such as matrix multiplication or video"},{"line_number":73,"context_line":"  transcoding, usually represented as a string or UUID. This term may be used"},{"line_number":74,"context_line":"  with multi-function devices, including FPGAs and other fixed function"},{"line_number":75,"context_line":"  hardware like Intel QuickAssist."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_668ffa74","line":72,"range":{"start_line":72,"start_character":2,"end_line":72,"end_character":10},"updated":"2018-08-24 21:38:48.000000000","message":"Function is a noun but the value of Function will actully be an ajative describing the functionality offloaded/delegated to the accelerator.\n\nperhaps offload-type or acceleration-category would be better.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":69,"context_line":"* Bitstream: An FPGA image, usually a binary file, possibly with"},{"line_number":70,"context_line":"  vendor-specific metadata. A bitstream may implement one or more functions."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"* Function: A specific functionality, such as matrix multiplication or video"},{"line_number":73,"context_line":"  transcoding, usually represented as a string or UUID. This term may be used"},{"line_number":74,"context_line":"  with multi-function devices, including FPGAs and other fixed function"},{"line_number":75,"context_line":"  hardware like Intel QuickAssist."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_b118bc1d","line":72,"range":{"start_line":72,"start_character":2,"end_line":72,"end_character":10},"in_reply_to":"3f79a3b5_668ffa74","updated":"2018-09-02 02:58:43.000000000","message":"We are referring to specific functions like a specific implementation of ipsec or gzip. Different implementations of, say, gzip will have different UUIDs.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":79,"context_line":"  Reconfiguration, the entire device constitutes one region. A region"},{"line_number":80,"context_line":"  may implement one or more functions."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Virtual Accelerator Nexus (VAN): A handle object representing a means"},{"line_number":83,"context_line":"  by which an accelerator can be attached to a VM, container or process."},{"line_number":84,"context_line":"  It may be based on PCI functions (PCI VF with SR-IOV), mediated"},{"line_number":85,"context_line":"  devices, Power architecture DRC Index (or location code or the like)."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_db0ca7ed","line":82,"range":{"start_line":82,"start_character":22,"end_line":82,"end_character":27},"updated":"2018-08-24 21:38:48.000000000","message":"Connection","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":79,"context_line":"  Reconfiguration, the entire device constitutes one region. A region"},{"line_number":80,"context_line":"  may implement one or more functions."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Virtual Accelerator Nexus (VAN): A handle object representing a means"},{"line_number":83,"context_line":"  by which an accelerator can be attached to a VM, container or process."},{"line_number":84,"context_line":"  It may be based on PCI functions (PCI VF with SR-IOV), mediated"},{"line_number":85,"context_line":"  devices, Power architecture DRC Index (or location code or the like)."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_9bfaafba","line":82,"range":{"start_line":82,"start_character":29,"end_line":82,"end_character":32},"updated":"2018-08-24 21:38:48.000000000","message":"VAC","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Virtual Accelerator Nexus (VAN): A handle object representing a means"},{"line_number":83,"context_line":"  by which an accelerator can be attached to a VM, container or process."},{"line_number":84,"context_line":"  It may be based on PCI functions (PCI VF with SR-IOV), mediated"},{"line_number":85,"context_line":"  devices, Power architecture DRC Index (or location code or the like)."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Here is an example diagram for an FPGA with multiple regions, and multiple"},{"line_number":88,"context_line":"accelerators in a region::"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_1b645f49","line":85,"range":{"start_line":84,"start_character":2,"end_line":85,"end_character":71},"updated":"2018-08-24 21:38:48.000000000","message":"this conflicts with the previous statement.\nthis is mixing the term to both refer to the logical data model object and the phyical dataplane concept of how the device and instance are connected.\n\nto put that more concreately using networking as a example.\nwhen using sriov we have two distict entities. \nThe virtual interface(VIF) and the sriov virtual function (VF)\n\nin this case we also need two terms.\n\na virtual acclerator connection(VAC) which is a python object that discripst how to conenct to the Accelerator Fuctions (AF).\n\nif you want to be more granular you might want to use the term Virtual Accelorator Fuction (VAF) and phyical acclerator function (PAF) to distigus the case were i am allocated all of a programmed resouces (PAF case) or am potentailly sharing it with others (VAF case).","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* Virtual Accelerator Nexus (VAN): A handle object representing a means"},{"line_number":83,"context_line":"  by which an accelerator can be attached to a VM, container or process."},{"line_number":84,"context_line":"  It may be based on PCI functions (PCI VF with SR-IOV), mediated"},{"line_number":85,"context_line":"  devices, Power architecture DRC Index (or location code or the like)."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Here is an example diagram for an FPGA with multiple regions, and multiple"},{"line_number":88,"context_line":"accelerators in a region::"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_1cc2e785","line":85,"range":{"start_line":84,"start_character":2,"end_line":85,"end_character":71},"in_reply_to":"3f79a3b5_1b645f49","updated":"2018-09-02 02:58:43.000000000","message":"The attach handle that a VAN represents may be *based on* PCI PF/VFs, mediated devices, etc. The former is logical, the latter may be (partly) physical. This is not unlike VIF vs. a PCI function.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":91,"context_line":" |       VM 0        |                        |  VM 1  |"},{"line_number":92,"context_line":" +-------------------+                        +--------+"},{"line_number":93,"context_line":" +--------+ +--------+                        +--------+"},{"line_number":94,"context_line":" |  VAN-0 | |  VAN-1 +\u003c---+                   |  VAN-N |"},{"line_number":95,"context_line":" +------+-+ +--------+    |                   +------+-+"},{"line_number":96,"context_line":"        ^                 |                          ^"},{"line_number":97,"context_line":" +------|-----------------|--------------------------|-----+"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a6c75296","line":94,"range":{"start_line":94,"start_character":4,"end_line":94,"end_character":9},"updated":"2018-08-24 21:38:48.000000000","message":"VAF or PAF depeneding on if ACC0 can be shared by two vms","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":91,"context_line":" |       VM 0        |                        |  VM 1  |"},{"line_number":92,"context_line":" +-------------------+                        +--------+"},{"line_number":93,"context_line":" +--------+ +--------+                        +--------+"},{"line_number":94,"context_line":" |  VAN-0 | |  VAN-1 +\u003c---+                   |  VAN-N |"},{"line_number":95,"context_line":" +------+-+ +--------+    |                   +------+-+"},{"line_number":96,"context_line":"        ^                 |                          ^"},{"line_number":97,"context_line":" +------|-----------------|--------------------------|-----+"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_7c619ba8","line":94,"range":{"start_line":94,"start_character":4,"end_line":94,"end_character":9},"in_reply_to":"3f79a3b5_a6c75296","updated":"2018-09-02 02:58:43.000000000","message":"Back to terminology: an accelerator is the unit that can be assigned to an instance. There is no concept of one accelerator being assigned to multiple VMs.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":111,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"The OpenStack Compute API [#ServerConcepts]_ mentions the list of operations"},{"line_number":114,"context_line":"that can be performed on an instance. Of these, some will not be supported by"},{"line_number":115,"context_line":"Cyborg in the initial releases."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"In general, when accelerators are detached from an instance, the associated"},{"line_number":118,"context_line":"resources are released. So, if the instance could be restarted after a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_4633deab","line":115,"range":{"start_line":114,"start_character":37,"end_line":115,"end_character":30},"updated":"2018-08-24 21:38:48.000000000","message":"this imply cyborg would be an impentation of the compute api which i dont think was the intent.\n\ni would replace this with the following.\n\nNot all operations valid for a server will be supported when\nusing an acclarort provided by cyborg.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":111,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"The OpenStack Compute API [#ServerConcepts]_ mentions the list of operations"},{"line_number":114,"context_line":"that can be performed on an instance. Of these, some will not be supported by"},{"line_number":115,"context_line":"Cyborg in the initial releases."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"In general, when accelerators are detached from an instance, the associated"},{"line_number":118,"context_line":"resources are released. So, if the instance could be restarted after a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_0ae4c40d","line":115,"range":{"start_line":114,"start_character":37,"end_line":115,"end_character":30},"in_reply_to":"3f79a3b5_4633deab","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":114,"context_line":"that can be performed on an instance. Of these, some will not be supported by"},{"line_number":115,"context_line":"Cyborg in the initial releases."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"In general, when accelerators are detached from an instance, the associated"},{"line_number":118,"context_line":"resources are released. So, if the instance could be restarted after a"},{"line_number":119,"context_line":"disruptive operation, either the accelerators should not be detached or"},{"line_number":120,"context_line":"the instance must get rescheduled on restart."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The list of supported operations (with the intended behaviors) are as follows:"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_c1cd985d","line":120,"range":{"start_line":117,"start_character":0,"end_line":120,"end_character":45},"updated":"2018-08-24 21:38:48.000000000","message":"when i read this first i read this as reboots would not be supported. on readiding after reading lines 128-136 im not sure that is what you ment.\n\ncan you first define what a disruptive operation is in this context?\n\ndetach and attach are distinct operation form allocate and deacllocate. i may want to detach a vf without deallocating it such as when we do a cold snapshot. it would be incorrect to deallocate the acclortor in this case but we do have to detach it while preforming the snapshot and then reattach it.\n\nit would be vaild for cyborg to reinitalise the acclerator function when we do this however so the guest application needs to support hotplug of the device.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":114,"context_line":"that can be performed on an instance. Of these, some will not be supported by"},{"line_number":115,"context_line":"Cyborg in the initial releases."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"In general, when accelerators are detached from an instance, the associated"},{"line_number":118,"context_line":"resources are released. So, if the instance could be restarted after a"},{"line_number":119,"context_line":"disruptive operation, either the accelerators should not be detached or"},{"line_number":120,"context_line":"the instance must get rescheduled on restart."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The list of supported operations (with the intended behaviors) are as follows:"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_0048232c","line":120,"range":{"start_line":117,"start_character":0,"end_line":120,"end_character":45},"in_reply_to":"3f79a3b5_c1cd985d","updated":"2018-09-02 02:58:43.000000000","message":"Will restate.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":125,"context_line":"  flavor must be attached to the instance. On deletion, those resources"},{"line_number":126,"context_line":"  are released."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"* When an instance is stopped, paused, suspended or locked, the"},{"line_number":129,"context_line":"  accelerator resources are left intact, and not detached from the instance."},{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_01363052","line":128,"range":{"start_line":128,"start_character":39,"end_line":128,"end_character":48},"updated":"2018-08-24 21:38:48.000000000","message":"for the libvirt driver map suspend to managed_save which reuiqres us to detach all passhtorugh devece before calling it so we must detach the accleartor frunctions in this case but we do not need to deallocate them","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":125,"context_line":"  flavor must be attached to the instance. On deletion, those resources"},{"line_number":126,"context_line":"  are released."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"* When an instance is stopped, paused, suspended or locked, the"},{"line_number":129,"context_line":"  accelerator resources are left intact, and not detached from the instance."},{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_76d94120","line":128,"range":{"start_line":128,"start_character":39,"end_line":128,"end_character":48},"in_reply_to":"3f79a3b5_01363052","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":126,"context_line":"  are released."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"* When an instance is stopped, paused, suspended or locked, the"},{"line_number":129,"context_line":"  accelerator resources are left intact, and not detached from the instance."},{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_410bc88c","line":129,"range":{"start_line":129,"start_character":49,"end_line":129,"end_character":57},"updated":"2018-08-24 21:38:48.000000000","message":"deallocated. they will be detached by qemu when it terminates.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":126,"context_line":"  are released."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"* When an instance is stopped, paused, suspended or locked, the"},{"line_number":129,"context_line":"  accelerator resources are left intact, and not detached from the instance."},{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_3605097b","line":129,"range":{"start_line":129,"start_character":49,"end_line":129,"end_character":57},"in_reply_to":"3f79a3b5_410bc88c","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are left attached."},{"line_number":134,"context_line":"  However, when it is shelved-offloaded, the accelerator resources are"},{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_010f1088","line":133,"range":{"start_line":133,"start_character":66,"end_line":133,"end_character":74},"updated":"2018-08-24 21:38:48.000000000","message":"allocated","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are left attached."},{"line_number":134,"context_line":"  However, when it is shelved-offloaded, the accelerator resources are"},{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_5608c58f","line":133,"range":{"start_line":133,"start_character":66,"end_line":133,"end_character":74},"in_reply_to":"3f79a3b5_010f1088","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are left attached."},{"line_number":134,"context_line":"  However, when it is shelved-offloaded, the accelerator resources are"},{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_611fe4b6","line":135,"range":{"start_line":135,"start_character":2,"end_line":135,"end_character":10},"updated":"2018-08-24 21:38:48.000000000","message":"deallocated","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are left attached."},{"line_number":134,"context_line":"  However, when it is shelved-offloaded, the accelerator resources are"},{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_f6feb169","line":135,"range":{"start_line":135,"start_character":2,"end_line":135,"end_character":10},"in_reply_to":"3f79a3b5_611fe4b6","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"},{"line_number":139,"context_line":"  restore, it is expected that the build operation will go through the"},{"line_number":140,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_217b2cf5","line":138,"range":{"start_line":138,"start_character":61,"end_line":138,"end_character":69},"updated":"2018-08-24 21:38:48.000000000","message":"detached and deallocated","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":135,"context_line":"  detached. On an unshelve after offload, the build operation will go"},{"line_number":136,"context_line":"  through the scheduler again, so it is equivalent to an instance start."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"},{"line_number":139,"context_line":"  restore, it is expected that the build operation will go through the"},{"line_number":140,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_16024d70","line":138,"range":{"start_line":138,"start_character":61,"end_line":138,"end_character":69},"in_reply_to":"3f79a3b5_217b2cf5","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":140,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":143,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":144,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":145,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":146,"context_line":"  precondition, accelerator resources are left attached to the instance"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_214ecc96","line":143,"range":{"start_line":143,"start_character":2,"end_line":143,"end_character":13},"updated":"2018-08-24 21:38:48.000000000","message":"why? ther is no such thing as a live rebuild.\n\n a rebuild is baically a delete followed by a spawn where we do not deallocate resouces.\n\nso when we start the rebuild we will first need to ask the schduler is the current node still valid for the new image.\nif the current host passes the schduler we will then showdown the instance which will detach but not deallocate the Acclerator functions then we will recreate the root disk with the new image and start the vm attaching the Acclerator resouces as normal.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":140,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":143,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":144,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":145,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":146,"context_line":"  precondition, accelerator resources are left attached to the instance"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_69c9c3aa","line":143,"range":{"start_line":143,"start_character":2,"end_line":143,"end_character":13},"in_reply_to":"3f79a3b5_214ecc96","updated":"2018-09-02 02:58:43.000000000","message":"Thanks. \"delete followed by a spawn\": Which doc are you quoting here? Will restate anyway.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":143,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":144,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":145,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":146,"context_line":"  precondition, accelerator resources are left attached to the instance"},{"line_number":147,"context_line":"  during the rebuild."},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_41518835","line":146,"range":{"start_line":146,"start_character":47,"end_line":146,"end_character":55},"updated":"2018-08-24 21:38:48.000000000","message":"allocated","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":143,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":144,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":145,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":146,"context_line":"  precondition, accelerator resources are left attached to the instance"},{"line_number":147,"context_line":"  during the rebuild."},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_d6f3759e","line":146,"range":{"start_line":146,"start_character":47,"end_line":146,"end_character":55},"in_reply_to":"3f79a3b5_41518835","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":147,"context_line":"  during the rebuild."},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"},{"line_number":150,"context_line":"  by re-scheduling and restart. The accelerator resources are detached"},{"line_number":151,"context_line":"  on termination and re-attached on when the instance is scheduled again."},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"* Cold migration: It is equivalent to a termination followed by re-scheduling"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_6157e427","line":150,"range":{"start_line":150,"start_character":62,"end_line":150,"end_character":70},"updated":"2018-08-24 21:38:48.000000000","message":"they are detached as part of shutdown and deallocated as part of the resize completion event. until resize is confirmed we need to keep the allocation on source node.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":147,"context_line":"  during the rebuild."},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"},{"line_number":150,"context_line":"  by re-scheduling and restart. The accelerator resources are detached"},{"line_number":151,"context_line":"  on termination and re-attached on when the instance is scheduled again."},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"* Cold migration: It is equivalent to a termination followed by re-scheduling"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_76f2a19c","line":150,"range":{"start_line":150,"start_character":62,"end_line":150,"end_character":70},"in_reply_to":"3f79a3b5_6157e427","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"},{"line_number":150,"context_line":"  by re-scheduling and restart. The accelerator resources are detached"},{"line_number":151,"context_line":"  on termination and re-attached on when the instance is scheduled again."},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"* Cold migration: It is equivalent to a termination followed by re-scheduling"},{"line_number":154,"context_line":"  and restart. The accelerator resources are detached on termination, and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_41366805","line":151,"range":{"start_line":151,"start_character":21,"end_line":151,"end_character":32},"updated":"2018-08-24 21:38:48.000000000","message":"the resized instance will be guarenteed to be on a new host unless the debug allow resize to same host option is set in the nova.conf so the same devices will not be reused.\n\nthe schduler will claim the new acclerator function in placment the the compute agent will call allocate_device folowed by prepare deivce on the new host the it will call* attach as part of spwan.\n\nNote: * the libvirt deriver woul not actully use the attach method to update the xml when boot it would jsut generate teh correct xml from the start. logically however it equvalent to saying we spawn the vm in a paused state and attach the device before unpausing and starting to execute the guest kernel.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"},{"line_number":150,"context_line":"  by re-scheduling and restart. The accelerator resources are detached"},{"line_number":151,"context_line":"  on termination and re-attached on when the instance is scheduled again."},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"* Cold migration: It is equivalent to a termination followed by re-scheduling"},{"line_number":154,"context_line":"  and restart. The accelerator resources are detached on termination, and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_89df7f39","line":151,"range":{"start_line":151,"start_character":21,"end_line":151,"end_character":32},"in_reply_to":"3f79a3b5_41366805","updated":"2018-09-02 02:58:43.000000000","message":"Thanks for the explanation. Again, the server operations doc does not state these so clearly. Will update.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":150,"context_line":"  by re-scheduling and restart. The accelerator resources are detached"},{"line_number":151,"context_line":"  on termination and re-attached on when the instance is scheduled again."},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"* Cold migration: It is equivalent to a termination followed by re-scheduling"},{"line_number":154,"context_line":"  and restart. The accelerator resources are detached on termination, and"},{"line_number":155,"context_line":"  re-attached on when the instance is scheduled again."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"* Evacuate: This is a forcible rebuild by the administrator. As the"},{"line_number":158,"context_line":"  semantics of evacuation are left open even without accelerators,"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_33861f68","line":155,"range":{"start_line":153,"start_character":18,"end_line":155,"end_character":54},"updated":"2018-08-24 21:38:48.000000000","message":"this is identical to resize except the flavor does not change.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":150,"context_line":"  by re-scheduling and restart. The accelerator resources are detached"},{"line_number":151,"context_line":"  on termination and re-attached on when the instance is scheduled again."},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"* Cold migration: It is equivalent to a termination followed by re-scheduling"},{"line_number":154,"context_line":"  and restart. The accelerator resources are detached on termination, and"},{"line_number":155,"context_line":"  re-attached on when the instance is scheduled again."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"* Evacuate: This is a forcible rebuild by the administrator. As the"},{"line_number":158,"context_line":"  semantics of evacuation are left open even without accelerators,"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_96edfdb8","line":155,"range":{"start_line":153,"start_character":18,"end_line":155,"end_character":54},"in_reply_to":"3f79a3b5_33861f68","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":154,"context_line":"  and restart. The accelerator resources are detached on termination, and"},{"line_number":155,"context_line":"  re-attached on when the instance is scheduled again."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"* Evacuate: This is a forcible rebuild by the administrator. As the"},{"line_number":158,"context_line":"  semantics of evacuation are left open even without accelerators,"},{"line_number":159,"context_line":"  Cyborg\u0027s behavior is also left undefined."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"* Set administrator password, trigger crash dump: These are supported and"},{"line_number":162,"context_line":"  are no-ops for accelerators."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_d3d9eb82","line":159,"range":{"start_line":157,"start_character":13,"end_line":159,"end_character":42},"updated":"2018-08-24 21:38:48.000000000","message":"there are some guarnetess we do deallocate the resouce claims in placemnt as far as i am aware. i also belive if the source compute node is running we will clean up the resouces track entries also. we will want to have the same behavior in this case.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":154,"context_line":"  and restart. The accelerator resources are detached on termination, and"},{"line_number":155,"context_line":"  re-attached on when the instance is scheduled again."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"* Evacuate: This is a forcible rebuild by the administrator. As the"},{"line_number":158,"context_line":"  semantics of evacuation are left open even without accelerators,"},{"line_number":159,"context_line":"  Cyborg\u0027s behavior is also left undefined."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"* Set administrator password, trigger crash dump: These are supported and"},{"line_number":162,"context_line":"  are no-ops for accelerators."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_89d3ffd5","line":159,"range":{"start_line":157,"start_character":13,"end_line":159,"end_character":42},"in_reply_to":"3f79a3b5_d3d9eb82","updated":"2018-09-02 02:58:43.000000000","message":"I found this: \nhttps://docs.openstack.org/nova/rocky/admin/evacuate.html\n\nApparently, this is equivalent to a rebuild and preserves many instance properties.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":164,"context_line":"The following two instance operations are allowed with some additional"},{"line_number":165,"context_line":"considerations noted below."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Reboot: The accelerator resources are left intact."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Booting off a snapshot: The accelerator resources are left intact."},{"line_number":170,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_934db33c","line":167,"range":{"start_line":167,"start_character":1,"end_line":167,"end_character":52},"updated":"2018-08-24 21:38:48.000000000","message":"by that you mean they are not deallocated.\non soft reboot the hyperviors(qemu) will remain running and the devices will not be detached but the guest kernel will reboot and the guest uefi/bios will reporbe the vms hardware and reset it.\n\nfor a hard reboot. the qemu process will terminate and be respawned so qemu will teatch and reattach to the accleration functions. but we should not deallocate and reallocate them and we should not need to clean an prepare the device.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":164,"context_line":"The following two instance operations are allowed with some additional"},{"line_number":165,"context_line":"considerations noted below."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Reboot: The accelerator resources are left intact."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Booting off a snapshot: The accelerator resources are left intact."},{"line_number":170,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_1637ad4f","line":167,"range":{"start_line":167,"start_character":1,"end_line":167,"end_character":52},"in_reply_to":"3f79a3b5_934db33c","updated":"2018-09-02 02:58:43.000000000","message":"Yes, soft vs hard reboot distinction should be stated clearly. Will restate.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Reboot: The accelerator resources are left intact."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Booting off a snapshot: The accelerator resources are left intact."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"In both these scenarios, the DMA buffers and other resources set up"},{"line_number":172,"context_line":"by the guest OS are lost after the operation. Any attempt by the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_b3fa2fba","line":169,"range":{"start_line":169,"start_character":1,"end_line":169,"end_character":68},"updated":"2018-08-24 21:38:48.000000000","message":"booting from a snapshot create a new vm so we will need to allocate and prepare new resouces.\n\nif you mean thaking a snappshot and then later resotring it to that snapshot then that is a rebuild operation.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Reboot: The accelerator resources are left intact."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Booting off a snapshot: The accelerator resources are left intact."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"In both these scenarios, the DMA buffers and other resources set up"},{"line_number":172,"context_line":"by the guest OS are lost after the operation. Any attempt by the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_d638553c","line":169,"range":{"start_line":169,"start_character":1,"end_line":169,"end_character":68},"in_reply_to":"3f79a3b5_b3fa2fba","updated":"2018-09-02 02:58:43.000000000","message":"Done","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Booting off a snapshot: The accelerator resources are left intact."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"In both these scenarios, the DMA buffers and other resources set up"},{"line_number":172,"context_line":"by the guest OS are lost after the operation. Any attempt by the"},{"line_number":173,"context_line":"device to access those resources after the operation can be problematic."},{"line_number":174,"context_line":"This issue is independent of OpenStack. So it is assumed that the"},{"line_number":175,"context_line":"hypervisor or the platform takes the necessary actions to prevent such"},{"line_number":176,"context_line":"scenarios, at least for well-behaved devices. For example, if the device"},{"line_number":177,"context_line":"is passed through to the VM via vfio, the hypervisor may issue a vfio reset,"},{"line_number":178,"context_line":"which should translate to a PCI function-level reset for the device."},{"line_number":179,"context_line":"A well-behaved device should abort DMA and other PCI transactions after"},{"line_number":180,"context_line":"such reset."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"The following instance operation is not supported for now:"},{"line_number":183,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_93be13ac","line":180,"range":{"start_line":171,"start_character":0,"end_line":180,"end_character":11},"updated":"2018-08-24 21:38:48.000000000","message":"really this should be handeld by the device driver not the hypervior. it is possible that qemu or another hyperviors could issue the device reset ioctl on behalf of the guest kernel but this is out of the scope of openstack and openstack cannont influnce this behavior.\n\nthat said i dont think its unreasonable to require all bitstreams programed by cyborg to support guest reboot.\n\nif the use of cyborg acclorators prevented a guest operating systemd from rebooting itself that would be a blocker for me in moving forward with nova inegration as it breaks the abstraction of a standard server that we provide.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Booting off a snapshot: The accelerator resources are left intact."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"In both these scenarios, the DMA buffers and other resources set up"},{"line_number":172,"context_line":"by the guest OS are lost after the operation. Any attempt by the"},{"line_number":173,"context_line":"device to access those resources after the operation can be problematic."},{"line_number":174,"context_line":"This issue is independent of OpenStack. So it is assumed that the"},{"line_number":175,"context_line":"hypervisor or the platform takes the necessary actions to prevent such"},{"line_number":176,"context_line":"scenarios, at least for well-behaved devices. For example, if the device"},{"line_number":177,"context_line":"is passed through to the VM via vfio, the hypervisor may issue a vfio reset,"},{"line_number":178,"context_line":"which should translate to a PCI function-level reset for the device."},{"line_number":179,"context_line":"A well-behaved device should abort DMA and other PCI transactions after"},{"line_number":180,"context_line":"such reset."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"The following instance operation is not supported for now:"},{"line_number":183,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_295d0b48","line":180,"range":{"start_line":171,"start_character":0,"end_line":180,"end_character":11},"in_reply_to":"3f79a3b5_93be13ac","updated":"2018-09-02 02:58:43.000000000","message":"As you noted yourself, the hypervisor needs to issue a reset or some notification to the device driver, to start the cleanup chain.\n\nNowhere does the spec say that reboots are unsupported. Rather it says that they are supported with some considerations on what hypervisor/driver need to do for safe operation.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":206,"context_line":"accelerators, and that translates into differences in the workflow"},{"line_number":207,"context_line":"and in the design of os-acc vs. os-vif:"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":210,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_936ff326","line":209,"range":{"start_line":209,"start_character":49,"end_line":209,"end_character":68},"updated":"2018-08-24 21:38:48.000000000","message":"IP are optional and macs are required but neither have anything to do with port binding as they are not provision during port binding they are provisions when the port is created in the neutron api.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":206,"context_line":"accelerators, and that translates into differences in the workflow"},{"line_number":207,"context_line":"and in the design of os-acc vs. os-vif:"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":210,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_6c81d1fb","line":209,"range":{"start_line":209,"start_character":49,"end_line":209,"end_character":68},"in_reply_to":"3f79a3b5_936ff326","updated":"2018-09-02 02:58:43.000000000","message":"Will clear up the terminology, but this comment misses the larger point: while Neutron ports need to be provisioned with something unique on a per-port basis, there is (as yet) no need to provision something unique for the accelerator per se. (We need to associate a VF or some attach handle with the accelerator, but that is different.)\n\nThe broader purpose of this section is to convey that the accelerator workflow must be designed based on its own requirements and characteristics, rather than blindly copying from another project.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":207,"context_line":"and in the design of os-acc vs. os-vif:"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":210,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"},{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_76b6c540","line":211,"range":{"start_line":210,"start_character":52,"end_line":211,"end_character":55},"updated":"2018-08-24 21:38:48.000000000","message":"this is incorrect. the port binding proces is when the neutron ask each of the loaded ml2 driver to attempt to bind the port. when asked to bind the port the ml2 driver reads the extended propertise on teh port such as the network it is to be attached to and the host it is to be bound too and validates the constratins. such as is there  a neutron agent/controler that can manage that host and is that netwrok reachable from that host. \n\nif the validation is successful the ml2 driver populates teh vif:binding-details metadata required to attach the guest to the network backend that it chose and set the vif-type to the correct value.\n\nthe vif details and vif type are then converted into an os-vif VIF object which is used by the virt drivers to create the vm. in the case of the libvirt driver the xml for the iterface is determined byt the vif info generated by the neutron ml2 driver. in legacy nova this was done by reading the vif type directly but now the type of the VIF object is used.\n\n\nthis is exactly the same workflow you would want to use for cyborg. in the cyborge case in stead of port bind you would do device allocate and return a virtual acclorator connection. the virt driver would then use that VAC/VAN to generate the libvirt xml required to attach the device based on the type of the VAN returned.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":207,"context_line":"and in the design of os-acc vs. os-vif:"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":210,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"},{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_4cfe5539","line":211,"range":{"start_line":210,"start_character":52,"end_line":211,"end_character":55},"in_reply_to":"3f79a3b5_76b6c540","updated":"2018-09-02 02:58:43.000000000","message":"First, allocation and preparation cannot be separated in the case of accelerators, as I said earlier. There are use cases where the device must be configured to associate an accelerator resource to an attach handle (e.g. VF). For FPGAs, the device may need to be programmed before it has the accelerators of the right type. So both steps go together. IIUC, you are correlating allocation to binding and preparation to plug(), so the analogy is not exact.\n\nSecondly, since Nova chooses a specific device RP based on traits, most of the constraints are satisfied already for accelerators.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":209,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":210,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"},{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"},{"line_number":215,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":216,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":217,"context_line":"  This means that os-acc must provide the PCI function data, if"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_3607edcc","line":214,"range":{"start_line":212,"start_character":2,"end_line":214,"end_character":52},"updated":"2018-08-24 21:38:48.000000000","message":"the assocaition of a acclerator to a pci endpoint would be part of allocate_device call.\n\nthe progrgraming would be part of the prepare_device call.\n\nboth could be done in prepare_device if the pci endpoint or mdev was reserved already in allocate.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":209,"context_line":"* Each Neutron port needs to be provisioned with a unique MAC and IP,"},{"line_number":210,"context_line":"  but there is no such need today for accelerators. This implies that"},{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"},{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"},{"line_number":215,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":216,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":217,"context_line":"  This means that os-acc must provide the PCI function data, if"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cc4ee595","line":214,"range":{"start_line":212,"start_character":2,"end_line":214,"end_character":52},"in_reply_to":"3f79a3b5_3607edcc","updated":"2018-09-02 02:58:43.000000000","message":"Already addressed in previous comments.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"},{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"},{"line_number":215,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":216,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":217,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":218,"context_line":"  any, for the Instance End of the attach."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_f60c15ed","line":215,"range":{"start_line":214,"start_character":53,"end_line":215,"end_character":41},"updated":"2018-08-24 21:38:48.000000000","message":"this is effectivly what plug does in os-vif.\n\nplug assocates the tap device that the will use with the vswitch it should be attached to.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":211,"context_line":"  the port binding step has no analog for accelerators."},{"line_number":212,"context_line":"* To create the Device End of the attach, the device may need to be"},{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"},{"line_number":215,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":216,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":217,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":218,"context_line":"  any, for the Instance End of the attach."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_8c16ad55","line":215,"range":{"start_line":214,"start_character":53,"end_line":215,"end_character":41},"in_reply_to":"3f79a3b5_f60c15ed","updated":"2018-09-02 02:58:43.000000000","message":"Reprogramming the FPGA creates new accelerators. (Although we represent them generically as CUSTOM_ACCELERATOR_FPGA resource class, the extra specs in the flavor may ask for specific functions, which can be realized only after programming.) This goes back to the allocation+preparation point.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"},{"line_number":215,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":216,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":217,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":218,"context_line":"  any, for the Instance End of the attach."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"The broad steps for attaching an accelerator to an instance are as below:"},{"line_number":221,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_36304d9f","line":218,"range":{"start_line":216,"start_character":2,"end_line":218,"end_character":42},"updated":"2018-08-24 21:38:48.000000000","message":"yes that would be part of the VAN the same way the interface name or pci address is part of the vif object","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":213,"context_line":"  configured in some way, such as associating the accelerator to a PCI"},{"line_number":214,"context_line":"  function in the hardware or reprogramming the FPGA. Whereas there is"},{"line_number":215,"context_line":"  no need to do that for NICs in Neutron."},{"line_number":216,"context_line":"* Nova handles PCI for Neutron while Cyborg handles it for accelerators."},{"line_number":217,"context_line":"  This means that os-acc must provide the PCI function data, if"},{"line_number":218,"context_line":"  any, for the Instance End of the attach."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"The broad steps for attaching an accelerator to an instance are as below:"},{"line_number":221,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_764701be","line":218,"range":{"start_line":216,"start_character":2,"end_line":218,"end_character":42},"in_reply_to":"3f79a3b5_36304d9f","updated":"2018-09-02 02:58:43.000000000","message":"In general, this section will be reworded to bring out the core points more clearly.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":"The broad steps for attaching an accelerator to an instance are as below:"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"#. Create a Virtual Accelerator Nexus (VAN). The VAN is a versioned object"},{"line_number":223,"context_line":"   that represents the attach handle: it is an abstraction of specific"},{"line_number":224,"context_line":"   handle types such as PCI BDFs, mediated device UUIDs, etc. (See `VANs`_)."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"#. Handle the Device End of the attach:"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_b623fd55","line":224,"range":{"start_line":222,"start_character":3,"end_line":224,"end_character":76},"updated":"2018-08-24 21:38:48.000000000","message":"in the os-vif/neutron workflow this is what port bind does.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":"The broad steps for attaching an accelerator to an instance are as below:"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"#. Create a Virtual Accelerator Nexus (VAN). The VAN is a versioned object"},{"line_number":223,"context_line":"   that represents the attach handle: it is an abstraction of specific"},{"line_number":224,"context_line":"   handle types such as PCI BDFs, mediated device UUIDs, etc. (See `VANs`_)."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"#. Handle the Device End of the attach:"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_affcbb0c","line":224,"range":{"start_line":222,"start_character":3,"end_line":224,"end_character":76},"in_reply_to":"3f79a3b5_b623fd55","updated":"2018-09-02 02:58:43.000000000","message":"Already addressed in previous comments.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":223,"context_line":"   that represents the attach handle: it is an abstraction of specific"},{"line_number":224,"context_line":"   handle types such as PCI BDFs, mediated device UUIDs, etc. (See `VANs`_)."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"#. Handle the Device End of the attach:"},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"   * Choose or create an accelerator in the chosen device RP (resource"},{"line_number":229,"context_line":"     provider) if there is more than one."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_16de713d","line":226,"range":{"start_line":226,"start_character":3,"end_line":226,"end_character":39},"updated":"2018-08-24 21:38:48.000000000","message":"this is more or less what plug does in os-vif","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":223,"context_line":"   that represents the attach handle: it is an abstraction of specific"},{"line_number":224,"context_line":"   handle types such as PCI BDFs, mediated device UUIDs, etc. (See `VANs`_)."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"#. Handle the Device End of the attach:"},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"   * Choose or create an accelerator in the chosen device RP (resource"},{"line_number":229,"context_line":"     provider) if there is more than one."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cf01f706","line":226,"range":{"start_line":226,"start_character":3,"end_line":226,"end_character":39},"in_reply_to":"3f79a3b5_16de713d","updated":"2018-09-02 02:58:43.000000000","message":"Already addressed in previous comments.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":234,"context_line":"     device, etc.)"},{"line_number":235,"context_line":"   * If needed, handle the driver bindings. For example, unbind a PCI"},{"line_number":236,"context_line":"     kernel driver and bind to vfio-pci."},{"line_number":237,"context_line":"   * Update the VAN to record the accelerator and attach handle association."},{"line_number":238,"context_line":"   * Persisethe VAN object and the instance-VAN association."},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"#. Handle any device-compute interactions."},{"line_number":241,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_36a36db4","line":238,"range":{"start_line":237,"start_character":5,"end_line":238,"end_character":60},"updated":"2018-08-24 21:38:48.000000000","message":"this should be done as part of allocting the device not as part of attaching it.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":234,"context_line":"     device, etc.)"},{"line_number":235,"context_line":"   * If needed, handle the driver bindings. For example, unbind a PCI"},{"line_number":236,"context_line":"     kernel driver and bind to vfio-pci."},{"line_number":237,"context_line":"   * Update the VAN to record the accelerator and attach handle association."},{"line_number":238,"context_line":"   * Persisethe VAN object and the instance-VAN association."},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"#. Handle any device-compute interactions."},{"line_number":241,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_8fa77f0c","line":238,"range":{"start_line":237,"start_character":5,"end_line":238,"end_character":60},"in_reply_to":"3f79a3b5_36a36db4","updated":"2018-09-02 02:58:43.000000000","message":"This is indeed the allocation/preparation step. The last step, \u0027Handle the Instance end of the attach\u0027, is what we called the attach/detach operation earlier. Perhaps the use of the noun \u0027attach\u0027 in the phrase \u0027device/instance end of the attach\u0027 is throwing you off. I\u0027ll try to use a better term.\n\nIf you are saying that the VAN should be persisted after the first step (\u0027Create a VAN\u0027 above), I considered it but it doesn\u0027t seem useful at that stage, as there is no info to recover if, say, the n-cpu crashes and restarts at that point.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":237,"context_line":"   * Update the VAN to record the accelerator and attach handle association."},{"line_number":238,"context_line":"   * Persisethe VAN object and the instance-VAN association."},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"#. Handle any device-compute interactions."},{"line_number":241,"context_line":""},{"line_number":242,"context_line":"   * As an example, say the device has two PCI virtual functions and they"},{"line_number":243,"context_line":"     need to be attached to different NUMA partitions in the compute node."},{"line_number":244,"context_line":"     This step may involve obtaining the anti-affinity requirement from the"},{"line_number":245,"context_line":"     device configuration, querying Nova for the NUMA node to which the"},{"line_number":246,"context_line":"     other PCI function is attached, if any, and setting the preferred or"},{"line_number":247,"context_line":"     required NUMA nodes for the current PCI function. (Note: This is"},{"line_number":248,"context_line":"     only an example. The exact workflow and Cyborg/Nova split are left"},{"line_number":249,"context_line":"     open intentionally. This spec focuses on the basics.)"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"   * As another example, say the device has a PCI function with multiple"},{"line_number":252,"context_line":"     interrupt vectors, and all of them need to be bound to different cores"},{"line_number":253,"context_line":"     in the same CPU set, socket or NUMA node. Similar to above, this step"},{"line_number":254,"context_line":"     would have to obtain that requirement from the device configuration,"},{"line_number":255,"context_line":"     query Nova for the needed info, and update the VAN with the preferences"},{"line_number":256,"context_line":"     or requirements."},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"   The result of this step is an updated VAN that can be used to attach"},{"line_number":259,"context_line":"   the accelerator to the instance with the needed constraints."},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"   The use cases of the device compute interactions are not yet well-"},{"line_number":262,"context_line":"   formulated. So, the considerations of this step in this document"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_f6fdf56d","line":259,"range":{"start_line":240,"start_character":0,"end_line":259,"end_character":63},"updated":"2018-08-24 21:38:48.000000000","message":"i dont think that makes sense. when nova asks cyborg to allocate the device it should specify its requirements inluding what numa nodes are vailid. could be done by passing the placement allocation to cyborg as once numa tology is modled in placement the numa affinity of the allocation will be determined by the allocation claimed in placement.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":237,"context_line":"   * Update the VAN to record the accelerator and attach handle association."},{"line_number":238,"context_line":"   * Persisethe VAN object and the instance-VAN association."},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"#. Handle any device-compute interactions."},{"line_number":241,"context_line":""},{"line_number":242,"context_line":"   * As an example, say the device has two PCI virtual functions and they"},{"line_number":243,"context_line":"     need to be attached to different NUMA partitions in the compute node."},{"line_number":244,"context_line":"     This step may involve obtaining the anti-affinity requirement from the"},{"line_number":245,"context_line":"     device configuration, querying Nova for the NUMA node to which the"},{"line_number":246,"context_line":"     other PCI function is attached, if any, and setting the preferred or"},{"line_number":247,"context_line":"     required NUMA nodes for the current PCI function. (Note: This is"},{"line_number":248,"context_line":"     only an example. The exact workflow and Cyborg/Nova split are left"},{"line_number":249,"context_line":"     open intentionally. This spec focuses on the basics.)"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"   * As another example, say the device has a PCI function with multiple"},{"line_number":252,"context_line":"     interrupt vectors, and all of them need to be bound to different cores"},{"line_number":253,"context_line":"     in the same CPU set, socket or NUMA node. Similar to above, this step"},{"line_number":254,"context_line":"     would have to obtain that requirement from the device configuration,"},{"line_number":255,"context_line":"     query Nova for the needed info, and update the VAN with the preferences"},{"line_number":256,"context_line":"     or requirements."},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"   The result of this step is an updated VAN that can be used to attach"},{"line_number":259,"context_line":"   the accelerator to the instance with the needed constraints."},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"   The use cases of the device compute interactions are not yet well-"},{"line_number":262,"context_line":"   formulated. So, the considerations of this step in this document"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_0fe12f2b","line":259,"range":{"start_line":240,"start_character":0,"end_line":259,"end_character":63},"in_reply_to":"3f79a3b5_f6fdf56d","updated":"2018-09-02 02:58:43.000000000","message":"? For the interrupt vectors example, how would Nova even know that constraint exists, so as to pass the NUMA nodes up front?\n\nMore broadly, the point here is that the future may bring use cases that involve device/compute interaction that we haven\u0027t anticipated yet, and so we need a flexible placeholder in the workflow to address them. I am not trying to solve NUMA affinity here.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":263,"context_line":"   are only at alpha level of maturity. They may be revisited later based"},{"line_number":264,"context_line":"   on real-world experience and feedback."},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"#. Handle the Instance End of the attach."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"   * Create the hypervisor-specific device definition from the VAN. For"},{"line_number":269,"context_line":"     example, for libvirt and PCI, this would be an XML snippet that"},{"line_number":270,"context_line":"     defines a PCI device."},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"   * Compose that into the instance definition. For libvirt, this involves"},{"line_number":273,"context_line":"     weaving the device XML snippet into the instance\u0027s XML."},{"line_number":274,"context_line":""},{"line_number":275,"context_line":".. _VANfactors:"},{"line_number":276,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_362dadfd","line":273,"range":{"start_line":266,"start_character":3,"end_line":273,"end_character":60},"updated":"2018-08-24 21:38:48.000000000","message":"this will be all handeled within the virt driver an will not be part of os-acc.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":272,"context_line":"   * Compose that into the instance definition. For libvirt, this involves"},{"line_number":273,"context_line":"     weaving the device XML snippet into the instance\u0027s XML."},{"line_number":274,"context_line":""},{"line_number":275,"context_line":".. _VANfactors:"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"The process of attaching a device to a VM and detaching it depends on"},{"line_number":278,"context_line":"many factors, as apparent from the above:"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_767485fe","line":275,"range":{"start_line":275,"start_character":3,"end_line":275,"end_character":14},"updated":"2018-08-24 21:38:48.000000000","message":"this is a rather confusing term\n\nit litrally expands to\nvirtual accelerator nexus factors which is a meaninless statement.\n\nwhy dont we just call the the VAN object \nor preable VAC object.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":272,"context_line":"   * Compose that into the instance definition. For libvirt, this involves"},{"line_number":273,"context_line":"     weaving the device XML snippet into the instance\u0027s XML."},{"line_number":274,"context_line":""},{"line_number":275,"context_line":".. _VANfactors:"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"The process of attaching a device to a VM and detaching it depends on"},{"line_number":278,"context_line":"many factors, as apparent from the above:"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_05acb5ec","line":275,"range":{"start_line":275,"start_character":3,"end_line":275,"end_character":14},"in_reply_to":"3f79a3b5_767485fe","updated":"2018-09-02 02:58:43.000000000","message":"This is just a label used for cross-reference. I could have called it \u0027foo\u0027. This is at best a nit.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":276,"context_line":""},{"line_number":277,"context_line":"The process of attaching a device to a VM and detaching it depends on"},{"line_number":278,"context_line":"many factors, as apparent from the above:"},{"line_number":279,"context_line":""},{"line_number":280,"context_line":"* Accelerator type: GPU, FPGA, etc."},{"line_number":281,"context_line":"* Device Virtualization Mechanism: plain PCI, SR-IOV, mediated device, etc."},{"line_number":282,"context_line":"* System architecture: x86, ARM, Power etc."},{"line_number":283,"context_line":"* Host Operating System."},{"line_number":284,"context_line":"* Hypervisor: libvirt+qemu, libvirt+Xen, etc."},{"line_number":285,"context_line":"* Vendor/device-specific actions."},{"line_number":286,"context_line":""},{"line_number":287,"context_line":"The term platform is used to mean a combination of system architecture, host"},{"line_number":288,"context_line":"OS and hypervisor."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_2901d205","line":285,"range":{"start_line":279,"start_character":0,"end_line":285,"end_character":33},"updated":"2018-08-24 21:38:48.000000000","message":"you are conflating several things here.\n\nthe system architecture and host operating system will have no bareing on the attachment of the a device to a guest.\n\nif i take a gpu and install it in an x86 system running cento7 with libvirt kvm and pass it though to a guest.\nand i then take the same gpu and install it in a arm system with ubuntu 18.04 and qemu/kvm how i pass it through to the guest will not change.\n\nin the general case the factors that will effect how a device are attached are limited to the following.\nacclerator_type, device_virtualisation_mechanisum  and hypervisor.\n\nin some rare cases there will be a vendoer specifc action that needs to be preformed tough that is rare that it is turely vendor or devices speficic.\n\n\nbowereing form os-vif you can create a ovo for each acclerator_type that contains a datapath filed\nwich is a class corresponding to the device_virtualisation_mechanisum\nand a device profile field that can contain vendore speficic info.\n\ne.g.\n\ndef class Acclerator_Type(oslo.versionedobject):\n    pass\n\ndef class Gpu(Acclerator_Type):\n    fields\u003d{\n        resolution\u003dfield.resolution(),\n        displays\u003dfield.integer(),\n        compute_apis\u003dfields.list(String)\n        .....\n    }\n    \ndef class fpga(Acclerator_Type):\n    fields\u003d{\n        ...\n    }\n\ndef class datapath(oslo.versionedobject):\n    pass\n\ndef class pcie(datapath):\n    fields\u003d{\n        address\u003dfield.string(),\n        address_type\u003dfiled.choice(BDF,UUID),\n    }\n\ndef class mdev(datapath):\n    fields\u003d{\n        mdev_path \u003d fields.path()\n    }\n\ndef class Device_Profile(oslo.versionedobject):\n    pass\n    \ndef class VendorA_DeviceX_Profile(Device_Profile):\n    fields\u003d{\n        my_special_data \u003d fields.string()\n    }\n    \n    \ndef VirtualAccleratorConnection(oslo.versionedobject):\n    fields \u003d {\n        acclerator_type \u003d fields.subclassof(Acclerator_type),\n        datapath \u003d fields.list(Datapath),\n        device_profile \u003d fields.subclassof(Device_profile),\n        }","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":276,"context_line":""},{"line_number":277,"context_line":"The process of attaching a device to a VM and detaching it depends on"},{"line_number":278,"context_line":"many factors, as apparent from the above:"},{"line_number":279,"context_line":""},{"line_number":280,"context_line":"* Accelerator type: GPU, FPGA, etc."},{"line_number":281,"context_line":"* Device Virtualization Mechanism: plain PCI, SR-IOV, mediated device, etc."},{"line_number":282,"context_line":"* System architecture: x86, ARM, Power etc."},{"line_number":283,"context_line":"* Host Operating System."},{"line_number":284,"context_line":"* Hypervisor: libvirt+qemu, libvirt+Xen, etc."},{"line_number":285,"context_line":"* Vendor/device-specific actions."},{"line_number":286,"context_line":""},{"line_number":287,"context_line":"The term platform is used to mean a combination of system architecture, host"},{"line_number":288,"context_line":"OS and hypervisor."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cf3017a2","line":285,"range":{"start_line":279,"start_character":0,"end_line":285,"end_character":33},"in_reply_to":"3f79a3b5_2901d205","updated":"2018-09-02 02:58:43.000000000","message":"\"limited to the following: accelerator_type, device_virtualization_mechanism and hypervisor\u0027. Clearly the process is going to be device-specific, as we need a device preparation step in the os-acc flow.\n\nPower architecture developers have pointed out that the PowerVM does indeed have a huge bearing on how devices are handled and attached to VMs. Much of this content follows from extensive discussions around that.\n\nThis section is basically motivation for the need for extension modules. Unless you disagree with that, this is probably a side discussion.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":304,"context_line":""},{"line_number":305,"context_line":"#. Handling the device-end of the attach: os-acc together with Cyborg drivers."},{"line_number":306,"context_line":""},{"line_number":307,"context_line":"#. Device-Compute interactions: os-acc together with os-acc extensions that"},{"line_number":308,"context_line":"   handle specific use cases such as NUMA affinity. (Such extensions"},{"line_number":309,"context_line":"   may invoke Cyborg drivers, Nova REST API, or whatever is needed to"},{"line_number":310,"context_line":"   do their job.)"},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"#. Handling the instance end of the attach: Nova virt driver."},{"line_number":313,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_69d82a1d","line":310,"range":{"start_line":307,"start_character":2,"end_line":310,"end_character":17},"updated":"2018-08-24 21:38:48.000000000","message":"numa affintiy will be handeled by placment in the future.\n\nin the imediate futre nova should passing the set of allowed numa as part of the allocate request with the placement allocation so that cyborge can assigin the acclerator funciton from the device corresponding to the resouce profiver selected by placement.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":304,"context_line":""},{"line_number":305,"context_line":"#. Handling the device-end of the attach: os-acc together with Cyborg drivers."},{"line_number":306,"context_line":""},{"line_number":307,"context_line":"#. Device-Compute interactions: os-acc together with os-acc extensions that"},{"line_number":308,"context_line":"   handle specific use cases such as NUMA affinity. (Such extensions"},{"line_number":309,"context_line":"   may invoke Cyborg drivers, Nova REST API, or whatever is needed to"},{"line_number":310,"context_line":"   do their job.)"},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"#. Handling the instance end of the attach: Nova virt driver."},{"line_number":313,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cf91f748","line":310,"range":{"start_line":307,"start_character":2,"end_line":310,"end_character":17},"in_reply_to":"3f79a3b5_69d82a1d","updated":"2018-09-02 02:58:43.000000000","message":"As the example about device IRQ affinity shows, Cyborg may have a role to play here. I don\u0027t want to design NUMA for devices in this spec.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":312,"context_line":"#. Handling the instance end of the attach: Nova virt driver."},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"Thus, os-acc uses two sets of extension modules: Cyborg drivers for device-"},{"line_number":315,"context_line":"specific actions and extensions for device-compute interactions. But both"},{"line_number":316,"context_line":"drivers and extensions will in general be platform-specific, with separate"},{"line_number":317,"context_line":"implementations for different architectures (x86, Power, etc.), server"},{"line_number":318,"context_line":"types and families (blade servers, hyperconverged or disaggregated"},{"line_number":319,"context_line":"servers), and hypervisors (qemu, PowerVM, etc.)"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":"It is desirable to allow Cyborg drivers and os-acc extensions to reside"},{"line_number":322,"context_line":"out of tree and to let the operator assemble the set needed for a given"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_891a66c3","line":319,"range":{"start_line":315,"start_character":64,"end_line":319,"end_character":47},"updated":"2018-08-24 21:38:48.000000000","message":"i have not seen a good usecase put forward as to why os-acc need to have any knoladge of the hypervisor.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":312,"context_line":"#. Handling the instance end of the attach: Nova virt driver."},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"Thus, os-acc uses two sets of extension modules: Cyborg drivers for device-"},{"line_number":315,"context_line":"specific actions and extensions for device-compute interactions. But both"},{"line_number":316,"context_line":"drivers and extensions will in general be platform-specific, with separate"},{"line_number":317,"context_line":"implementations for different architectures (x86, Power, etc.), server"},{"line_number":318,"context_line":"types and families (blade servers, hyperconverged or disaggregated"},{"line_number":319,"context_line":"servers), and hypervisors (qemu, PowerVM, etc.)"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":"It is desirable to allow Cyborg drivers and os-acc extensions to reside"},{"line_number":322,"context_line":"out of tree and to let the operator assemble the set needed for a given"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_4f9de755","line":319,"range":{"start_line":315,"start_character":64,"end_line":319,"end_character":47},"in_reply_to":"3f79a3b5_891a66c3","updated":"2018-09-02 02:58:43.000000000","message":"PowerVM developers have been asking for exactly that: I\u0027ll let them chime in.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":327,"context_line":"shall specify the drivers and extensions respectively. An example entry"},{"line_number":328,"context_line":"(not normative) for os-acc\u0027s `setup.cfg` is as below::"},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"  [entry_points]"},{"line_number":331,"context_line":"  os_acc \u003d"},{"line_number":332,"context_line":"     x86_affinity \u003d os_acc.numa_affinity.x86_libvirt"},{"line_number":333,"context_line":"  cyborg \u003d"},{"line_number":334,"context_line":"     x86_gpu \u003d cyborg.amd_gpu.x86"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"Workflow"},{"line_number":337,"context_line":"--------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_6957ca27","line":334,"range":{"start_line":330,"start_character":2,"end_line":334,"end_character":33},"updated":"2018-08-24 21:38:48.000000000","message":"i dont think either of these are good examples of something that should be plugable.\n\nos-acc will not need a hyperviors specific numa affinity plugin as nova can speify the numa nodes that are valid when it asks os-acc to allocate the device.\n\nthe types of device should also not be plugable but rather should be stored in tree in os-acc\n\nsilarly there is no reasonto list x86_gpu as an entry point as how you passthough a gpu will not change based on architecutre.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":327,"context_line":"shall specify the drivers and extensions respectively. An example entry"},{"line_number":328,"context_line":"(not normative) for os-acc\u0027s `setup.cfg` is as below::"},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"  [entry_points]"},{"line_number":331,"context_line":"  os_acc \u003d"},{"line_number":332,"context_line":"     x86_affinity \u003d os_acc.numa_affinity.x86_libvirt"},{"line_number":333,"context_line":"  cyborg \u003d"},{"line_number":334,"context_line":"     x86_gpu \u003d cyborg.amd_gpu.x86"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"Workflow"},{"line_number":337,"context_line":"--------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_ef45d3b5","line":334,"range":{"start_line":330,"start_character":2,"end_line":334,"end_character":33},"in_reply_to":"3f79a3b5_6957ca27","updated":"2018-09-02 02:58:43.000000000","message":"Totally disagree that all device software should be in-tree in os-acc or Cyborg. The Cyborg driver is not specific to just a device type but also a device family or vendor -- I would expect different GPU drivers, different FPGA drivers etc. Given the variety of devices, it is not reasonable to expect that all drivers will reside in-tree. We should allow out-of-tree device drivers.\n\nAs I made it clear, these are only examples. The reader is free to consider other examples.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":338,"context_line":"The end-to-end workflow for instance creation and scheduling with"},{"line_number":339,"context_line":"accelerators is as follows:"},{"line_number":340,"context_line":""},{"line_number":341,"context_line":"0. Cyborg agent and drivers discover devices and resources in each node."},{"line_number":342,"context_line":"   The agent propagates the data to the Cyborg conductor. The conductor"},{"line_number":343,"context_line":"   computes the difference from previous configuration, updates the"},{"line_number":344,"context_line":"   database with the diff and then updates Placement based on the diff."},{"line_number":345,"context_line":"   Further details are in [#CyborgNovaSched]_."},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"#. The user submits a instance creation request. An example::"},{"line_number":348,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a92642cd","line":345,"range":{"start_line":341,"start_character":2,"end_line":345,"end_character":46},"updated":"2018-08-24 21:38:48.000000000","message":"alternitive is the nova-compute agent call invetory_devices in os-acc as part of update_provider_tree on agent startup.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":338,"context_line":"The end-to-end workflow for instance creation and scheduling with"},{"line_number":339,"context_line":"accelerators is as follows:"},{"line_number":340,"context_line":""},{"line_number":341,"context_line":"0. Cyborg agent and drivers discover devices and resources in each node."},{"line_number":342,"context_line":"   The agent propagates the data to the Cyborg conductor. The conductor"},{"line_number":343,"context_line":"   computes the difference from previous configuration, updates the"},{"line_number":344,"context_line":"   database with the diff and then updates Placement based on the diff."},{"line_number":345,"context_line":"   Further details are in [#CyborgNovaSched]_."},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"#. The user submits a instance creation request. An example::"},{"line_number":348,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_0f436fb1","line":345,"range":{"start_line":341,"start_character":2,"end_line":345,"end_character":46},"in_reply_to":"3f79a3b5_a92642cd","updated":"2018-09-02 02:58:43.000000000","message":"Already addressed in previous comments.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":349,"context_line":"     openstack server create --flavor my-acc ..."},{"line_number":350,"context_line":"     Flavor my-acc includes:"},{"line_number":351,"context_line":"       resources:CUSTOM_ACCELERATOR_GPU\u003d1"},{"line_number":352,"context_line":"       traits:CUSTOM_GPU_VENDOR_X\u003drequired"},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"#. Nova API receives the request and sends it to the Nova conductor,"},{"line_number":355,"context_line":"   which in turn forwards it to the Nova scheduler."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_2c01e005","line":352,"range":{"start_line":352,"start_character":14,"end_line":352,"end_character":33},"updated":"2018-08-24 21:38:48.000000000","message":"vendors should not be tratis.\n\nyou should use different resouce classes for each device.\n\ne.g. CUSTOM_GPU_NVIDIA_GTX1080 AND CUSTOM_GPU_AMD_VEGAR64","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":349,"context_line":"     openstack server create --flavor my-acc ..."},{"line_number":350,"context_line":"     Flavor my-acc includes:"},{"line_number":351,"context_line":"       resources:CUSTOM_ACCELERATOR_GPU\u003d1"},{"line_number":352,"context_line":"       traits:CUSTOM_GPU_VENDOR_X\u003drequired"},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"#. Nova API receives the request and sends it to the Nova conductor,"},{"line_number":355,"context_line":"   which in turn forwards it to the Nova scheduler."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_0f4a8fab","line":352,"range":{"start_line":352,"start_character":14,"end_line":352,"end_character":33},"in_reply_to":"3f79a3b5_2c01e005","updated":"2018-09-02 02:58:43.000000000","message":"No. Quotas are per-RC and we don\u0027t necessarily want quotas for each device model -- but quotas for GPUs, FPGAs, or device types in general. \n\nIf you mean that there should be traits (not RCs) for device models, yes. The caveat is that, if the VM image has a device driver for a family of devices, there would need to be a trait that covers that family.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":362,"context_line":"   CUSTOM_ACCELERATOR_*. It therefore creates a generic VAN object and"},{"line_number":363,"context_line":"   populates it with information that subsequent steps need, such as"},{"line_number":364,"context_line":"   system architecture and hypervisor type (see WhyHavePlatformArchType_.)"},{"line_number":365,"context_line":"#. The n-cpu then invokes the `attach_for_instance()` API in os-acc (see"},{"line_number":366,"context_line":"   `Nova Compute - Os-acc API`_), passing info for all accelerators"},{"line_number":367,"context_line":"   requested for that instance, with the chosen device RPs, instance UUID,"},{"line_number":368,"context_line":"   and flavor extra specs. The interaction with os-acc can happen"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_6c3358f7","line":365,"range":{"start_line":365,"start_character":31,"end_line":365,"end_character":50},"updated":"2018-08-24 21:38:48.000000000","message":"allocate_for_instance()\n\nalso i dont know if we really want the iteration to happen in os-acc i think that would be better to be left to the caller e.g. nova virtdriver.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":362,"context_line":"   CUSTOM_ACCELERATOR_*. It therefore creates a generic VAN object and"},{"line_number":363,"context_line":"   populates it with information that subsequent steps need, such as"},{"line_number":364,"context_line":"   system architecture and hypervisor type (see WhyHavePlatformArchType_.)"},{"line_number":365,"context_line":"#. The n-cpu then invokes the `attach_for_instance()` API in os-acc (see"},{"line_number":366,"context_line":"   `Nova Compute - Os-acc API`_), passing info for all accelerators"},{"line_number":367,"context_line":"   requested for that instance, with the chosen device RPs, instance UUID,"},{"line_number":368,"context_line":"   and flavor extra specs. The interaction with os-acc can happen"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cf97b7eb","line":365,"range":{"start_line":365,"start_character":31,"end_line":365,"end_character":50},"in_reply_to":"3f79a3b5_6c3358f7","updated":"2018-09-02 02:58:43.000000000","message":"If we handle one VAN at a time, and something fails, there would have to be a rollback of some kind -- to release resources etc. Would we want to leave that to each virt driver? I thought it would be simpler to have that logic in one place, i.e, os-acc.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":367,"context_line":"   requested for that instance, with the chosen device RPs, instance UUID,"},{"line_number":368,"context_line":"   and flavor extra specs. The interaction with os-acc can happen"},{"line_number":369,"context_line":"   concurrently with other tasks for spawning the instance."},{"line_number":370,"context_line":"#. The `attach_for_instance` API in os-acc does the following for each"},{"line_number":371,"context_line":"   requested accelerator for that instance. (Note: it is possible to"},{"line_number":372,"context_line":"   optimize by performing certain tasks once for the instance rather"},{"line_number":373,"context_line":"   once for each accelerator.)"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_ecc9a8ab","line":370,"range":{"start_line":370,"start_character":8,"end_line":370,"end_character":27},"updated":"2018-08-24 21:38:48.000000000","message":"this does way to much.\n\nwe shoudl seperate this out into seperate calls\nfirst we allocate the devices for the instance and then for each call os-acc to prepare the device.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":367,"context_line":"   requested for that instance, with the chosen device RPs, instance UUID,"},{"line_number":368,"context_line":"   and flavor extra specs. The interaction with os-acc can happen"},{"line_number":369,"context_line":"   concurrently with other tasks for spawning the instance."},{"line_number":370,"context_line":"#. The `attach_for_instance` API in os-acc does the following for each"},{"line_number":371,"context_line":"   requested accelerator for that instance. (Note: it is possible to"},{"line_number":372,"context_line":"   optimize by performing certain tasks once for the instance rather"},{"line_number":373,"context_line":"   once for each accelerator.)"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_f60c7195","line":370,"range":{"start_line":370,"start_character":8,"end_line":370,"end_character":27},"in_reply_to":"3f79a3b5_ecc9a8ab","updated":"2018-09-02 02:58:43.000000000","message":"The next version will update this sequence substantially.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":377,"context_line":"   * Performs any actions needed for the resource class, i.e.,"},{"line_number":378,"context_line":"     accelerator category (GPUs, FPGAs, etc.)."},{"line_number":379,"context_line":"   * Queries Cyborg conductor via RPC to obtain info on the device RP"},{"line_number":380,"context_line":"     that Nova selected for that accelerator."},{"line_number":381,"context_line":"   * Queries Glance with VM UUID to obtain any image traits relevant to"},{"line_number":382,"context_line":"     accelerators."},{"line_number":383,"context_line":"   * Decides which Cyborg driver can handle that device type."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_8c8a7461","line":380,"range":{"start_line":380,"start_character":10,"end_line":380,"end_character":14},"updated":"2018-08-24 21:38:48.000000000","message":"placement","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":377,"context_line":"   * Performs any actions needed for the resource class, i.e.,"},{"line_number":378,"context_line":"     accelerator category (GPUs, FPGAs, etc.)."},{"line_number":379,"context_line":"   * Queries Cyborg conductor via RPC to obtain info on the device RP"},{"line_number":380,"context_line":"     that Nova selected for that accelerator."},{"line_number":381,"context_line":"   * Queries Glance with VM UUID to obtain any image traits relevant to"},{"line_number":382,"context_line":"     accelerators."},{"line_number":383,"context_line":"   * Decides which Cyborg driver can handle that device type."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_6fb5437e","line":380,"range":{"start_line":380,"start_character":10,"end_line":380,"end_character":14},"in_reply_to":"3f79a3b5_8c8a7461","updated":"2018-09-02 02:58:43.000000000","message":"Placement returns a set of allocation candidates, and Nova scheduler selects a specific RP.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":378,"context_line":"     accelerator category (GPUs, FPGAs, etc.)."},{"line_number":379,"context_line":"   * Queries Cyborg conductor via RPC to obtain info on the device RP"},{"line_number":380,"context_line":"     that Nova selected for that accelerator."},{"line_number":381,"context_line":"   * Queries Glance with VM UUID to obtain any image traits relevant to"},{"line_number":382,"context_line":"     accelerators."},{"line_number":383,"context_line":"   * Decides which Cyborg driver can handle that device type."},{"line_number":384,"context_line":"   * Calls that driver\u0027s `prepareVAN()` API to configure the device end"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cc94ec8d","line":381,"range":{"start_line":381,"start_character":24,"end_line":381,"end_character":27},"updated":"2018-08-24 21:38:48.000000000","message":"you mean image uuid. we already have this in teh virt driver so we could pass them in as part of the call to os-acc","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":378,"context_line":"     accelerator category (GPUs, FPGAs, etc.)."},{"line_number":379,"context_line":"   * Queries Cyborg conductor via RPC to obtain info on the device RP"},{"line_number":380,"context_line":"     that Nova selected for that accelerator."},{"line_number":381,"context_line":"   * Queries Glance with VM UUID to obtain any image traits relevant to"},{"line_number":382,"context_line":"     accelerators."},{"line_number":383,"context_line":"   * Decides which Cyborg driver can handle that device type."},{"line_number":384,"context_line":"   * Calls that driver\u0027s `prepareVAN()` API to configure the device end"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_2f866bbc","line":381,"range":{"start_line":381,"start_character":24,"end_line":381,"end_character":27},"in_reply_to":"3f79a3b5_cc94ec8d","updated":"2018-09-02 02:58:43.000000000","message":"Yes. Agreed.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":381,"context_line":"   * Queries Glance with VM UUID to obtain any image traits relevant to"},{"line_number":382,"context_line":"     accelerators."},{"line_number":383,"context_line":"   * Decides which Cyborg driver can handle that device type."},{"line_number":384,"context_line":"   * Calls that driver\u0027s `prepareVAN()` API to configure the device end"},{"line_number":385,"context_line":"     of the attach. This step also serves to bind a specific accelerator"},{"line_number":386,"context_line":"     instance in the chosen device RP to that VAN. It may also initiate"},{"line_number":387,"context_line":"     programming in the case of FPGAs."},{"line_number":388,"context_line":"   * Persists the VAN by calling the Cyborg conductor via an RPC API."},{"line_number":389,"context_line":"   * Decides what device-compute interactions need to be handled, and which"},{"line_number":390,"context_line":"     os-acc extension can handle that."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cc470c1d","line":387,"range":{"start_line":384,"start_character":5,"end_line":387,"end_character":38},"updated":"2018-08-24 21:38:48.000000000","message":"this should be a in a sperate prepare_devices_for_instance call.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":386,"context_line":"     instance in the chosen device RP to that VAN. It may also initiate"},{"line_number":387,"context_line":"     programming in the case of FPGAs."},{"line_number":388,"context_line":"   * Persists the VAN by calling the Cyborg conductor via an RPC API."},{"line_number":389,"context_line":"   * Decides what device-compute interactions need to be handled, and which"},{"line_number":390,"context_line":"     os-acc extension can handle that."},{"line_number":391,"context_line":"   * Calls the os-acc extension\u0027s `updateVAN()` API to handle device-compute"},{"line_number":392,"context_line":"     interactions. This step may query Nova and other components."},{"line_number":393,"context_line":"   * Persists the instance-VAN association by calling Nova conductor via an"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_acb09044","line":390,"range":{"start_line":389,"start_character":1,"end_line":390,"end_character":38},"updated":"2018-08-24 21:38:48.000000000","message":"there really should not need to be any.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":389,"context_line":"   * Decides what device-compute interactions need to be handled, and which"},{"line_number":390,"context_line":"     os-acc extension can handle that."},{"line_number":391,"context_line":"   * Calls the os-acc extension\u0027s `updateVAN()` API to handle device-compute"},{"line_number":392,"context_line":"     interactions. This step may query Nova and other components."},{"line_number":393,"context_line":"   * Persists the instance-VAN association by calling Nova conductor via an"},{"line_number":394,"context_line":"     RPC API."},{"line_number":395,"context_line":"   * Returns the updated VANs to n-cpu."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_ec5768c9","line":392,"range":{"start_line":392,"start_character":18,"end_line":392,"end_character":65},"updated":"2018-08-24 21:38:48.000000000","message":"you should not need to quey nova.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     os-acc extension can handle that."},{"line_number":391,"context_line":"   * Calls the os-acc extension\u0027s `updateVAN()` API to handle device-compute"},{"line_number":392,"context_line":"     interactions. This step may query Nova and other components."},{"line_number":393,"context_line":"   * Persists the instance-VAN association by calling Nova conductor via an"},{"line_number":394,"context_line":"     RPC API."},{"line_number":395,"context_line":"   * Returns the updated VANs to n-cpu."},{"line_number":396,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_8c66f4ba","line":393,"range":{"start_line":393,"start_character":54,"end_line":393,"end_character":58},"updated":"2018-08-24 21:38:48.000000000","message":"you mean cyborg.\nyou wont have acess to the nova conductor or nova db.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":390,"context_line":"     os-acc extension can handle that."},{"line_number":391,"context_line":"   * Calls the os-acc extension\u0027s `updateVAN()` API to handle device-compute"},{"line_number":392,"context_line":"     interactions. This step may query Nova and other components."},{"line_number":393,"context_line":"   * Persists the instance-VAN association by calling Nova conductor via an"},{"line_number":394,"context_line":"     RPC API."},{"line_number":395,"context_line":"   * Returns the updated VANs to n-cpu."},{"line_number":396,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_6f6a23ee","line":393,"range":{"start_line":393,"start_character":54,"end_line":393,"end_character":58},"in_reply_to":"3f79a3b5_8c66f4ba","updated":"2018-09-02 02:58:43.000000000","message":"Say a VM is getting suspended or deleted. Nova virt driver would call os-acc with the VANs associated with that VM to have them detached or deallocated, as needed. Wouldn\u0027t we want that info to be with Nova?\n\nThat said, in the next version, I\u0027ll have nova-compute do the persistence.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":401,"context_line":"#. The n-cpu then passes the VANs to the virt driver. The virt driver uses"},{"line_number":402,"context_line":"   the VAN data to insert the accelerator attach handles into the instance"},{"line_number":403,"context_line":"   definition."},{"line_number":404,"context_line":"#. The n-cpu calls the `postattach_for_instance()` API in os-acc, indicating"},{"line_number":405,"context_line":"   the success or failure of the VM spawning. This is expected to handle"},{"line_number":406,"context_line":"   any post-VM-boot operations on the device on success, and cleanups"},{"line_number":407,"context_line":"   or rollbacks on failure. The API does the following for each VAN:"},{"line_number":408,"context_line":""},{"line_number":409,"context_line":"   * Calls the `cleanupVAN()` API of the previous invoked Cyborg driver"},{"line_number":410,"context_line":"     (recorded in the VAN) with the success/failure status."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_8cfb54bc","line":407,"range":{"start_line":404,"start_character":3,"end_line":407,"end_character":68},"updated":"2018-08-24 21:38:48.000000000","message":"we i dont think we shoudl reqriue and post vm boot operations but if we did we should have seperate fuctions for sucess and failure.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":458,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"Call Nova conductor to persist instance-VAN\"];"},{"line_number":459,"context_line":"    \u0027Nova Compute\u0027 \u003c-- \u0027os-acc\u0027 [label\u003d\"updated VAN\"];"},{"line_number":460,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027Nova Compute\u0027 [label\u003d\"Spawn VM\"];"},{"line_number":461,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"postattach_for_instance()\"];"},{"line_number":462,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027Cyborg driver\u0027 [label\u003d\"cleanupVAN()\"];"},{"line_number":463,"context_line":"    \u0027os-acc\u0027 \u003c-- \u0027Cyborg driver\u0027 [label\u003d\"\"];"},{"line_number":464,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"On failure, call Nova to unbind instance-VAN\"];"},{"line_number":465,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"On failure, call Cyborg to release VAN\"];"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"  }"},{"line_number":468,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_ccccec8a","line":465,"range":{"start_line":461,"start_character":5,"end_line":465,"end_character":74},"updated":"2018-08-24 21:38:48.000000000","message":"this should to be two seperate code paths as they will be in nova.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":458,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"Call Nova conductor to persist instance-VAN\"];"},{"line_number":459,"context_line":"    \u0027Nova Compute\u0027 \u003c-- \u0027os-acc\u0027 [label\u003d\"updated VAN\"];"},{"line_number":460,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027Nova Compute\u0027 [label\u003d\"Spawn VM\"];"},{"line_number":461,"context_line":"    \u0027Nova Compute\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"postattach_for_instance()\"];"},{"line_number":462,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027Cyborg driver\u0027 [label\u003d\"cleanupVAN()\"];"},{"line_number":463,"context_line":"    \u0027os-acc\u0027 \u003c-- \u0027Cyborg driver\u0027 [label\u003d\"\"];"},{"line_number":464,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"On failure, call Nova to unbind instance-VAN\"];"},{"line_number":465,"context_line":"    \u0027os-acc\u0027 -\u003e \u0027os-acc\u0027 [label\u003d\"On failure, call Cyborg to release VAN\"];"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"  }"},{"line_number":468,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_af17db54","line":465,"range":{"start_line":461,"start_character":5,"end_line":465,"end_character":74},"in_reply_to":"3f79a3b5_ccccec8a","updated":"2018-09-02 02:58:43.000000000","message":"Yes, they are. The \u0027on failure\u0027 condition is supposed to indicate that. \n\nAnyway, the entire flow is being redone in the next version.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":466,"context_line":""},{"line_number":467,"context_line":"  }"},{"line_number":468,"context_line":""},{"line_number":469,"context_line":"VANs"},{"line_number":470,"context_line":"----"},{"line_number":471,"context_line":"As noted before, the VAN is an Oslo versioned object that represents the"},{"line_number":472,"context_line":"attach handle: it is an abstraction of specific handles such as PCI BDFs,"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_cccdcc5d","line":469,"range":{"start_line":469,"start_character":2,"end_line":469,"end_character":3},"updated":"2018-08-24 21:38:48.000000000","message":"we should restructor the document to seperate our the control flow/workflow and the datamodel.\n\nthis spec keeps jumping back between the two which make it very hard to digest.\n\nwe should really discribe the datamodle before going though the workflow so this should be much earlier.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":548,"context_line":"        . . ."},{"line_number":549,"context_line":"     }"},{"line_number":550,"context_line":""},{"line_number":551,"context_line":"  class VANAttachProfile_PCI_Power(VANAttachProfileBase):"},{"line_number":552,"context_line":"     fields \u003d {"},{"line_number":553,"context_line":"        LocationIndex: IntegerField() # only an example"},{"line_number":554,"context_line":"        . . ."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_c79925c4","line":551,"range":{"start_line":551,"start_character":8,"end_line":551,"end_character":34},"updated":"2018-08-24 21:38:48.000000000","message":"if your are using power kvm on linux you will use the bus device fuction adddress if you are are using powervm you will use the location index. the architecure does not tell you how to attach a pci device the hypervior does.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":558,"context_line":"-------------------------"},{"line_number":559,"context_line":""},{"line_number":560,"context_line":"Nova Compute calls the os-acc APIs if it sees that the requested flavor"},{"line_number":561,"context_line":"refers to CUSTOM_ACCELERATOR_* resource classes. The APIs are called"},{"line_number":562,"context_line":"asynchronously, as suggested below::"},{"line_number":563,"context_line":""},{"line_number":564,"context_line":"   with ThreadPoolExecutor(max_workers\u003dN) as executor:"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a7b8a910","line":561,"range":{"start_line":561,"start_character":9,"end_line":561,"end_character":31},"updated":"2018-08-24 21:38:48.000000000","message":"are you suggesting we resrve CUSTOM_ACCLERATOR for cyborg?","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":558,"context_line":"-------------------------"},{"line_number":559,"context_line":""},{"line_number":560,"context_line":"Nova Compute calls the os-acc APIs if it sees that the requested flavor"},{"line_number":561,"context_line":"refers to CUSTOM_ACCELERATOR_* resource classes. The APIs are called"},{"line_number":562,"context_line":"asynchronously, as suggested below::"},{"line_number":563,"context_line":""},{"line_number":564,"context_line":"   with ThreadPoolExecutor(max_workers\u003dN) as executor:"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_d64ff5bb","line":561,"range":{"start_line":561,"start_character":9,"end_line":561,"end_character":31},"in_reply_to":"3f79a3b5_a7b8a910","updated":"2018-09-02 02:58:43.000000000","message":"A Nova developer is looking at making ACCELERATOR_{GPU,FPGA} etc. as standard RCs in Nova. This is only till that point.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":580,"context_line":"    def initialize(self): # called by n-cpu at start"},{"line_number":581,"context_line":"       return None"},{"line_number":582,"context_line":""},{"line_number":583,"context_line":"    def attach_for_instance(self, device_rp[], extra_specs, instance_info,"},{"line_number":584,"context_line":"                            vans[]):"},{"line_number":585,"context_line":"       ..."},{"line_number":586,"context_line":"       return vans[]"},{"line_number":587,"context_line":""},{"line_number":588,"context_line":"    def postattach_for_instance(self, instance_info, vans[]):"},{"line_number":589,"context_line":"       return None"},{"line_number":590,"context_line":""},{"line_number":591,"context_line":"    def detach_for_instance(self, instance_info, vans[]):"},{"line_number":592,"context_line":"       return None"},{"line_number":593,"context_line":""},{"line_number":594,"context_line":"The semantics of these APIs are specified below."},{"line_number":595,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a770a941","line":592,"range":{"start_line":583,"start_character":7,"end_line":592,"end_character":18},"updated":"2018-08-24 21:38:48.000000000","message":"i think we want the iteration to be done client side.\nwe will need to support hot attach/detach of indviual instace as some point an passing a list of one for that cases seams wrong.\n\nas i have said previoulsy i would also have a different infaces for os-acc.\n\n    def initialize(self): # called by n-cpu at start\n       return None\n\n    def allocate_device(self,placement_rp,resouce_class,numa_nodes\u003d[])\n       ...\n       return van()\n\n    def prepare_device(self,van)\n       ...\n       return van\n\n    def cleanup_device(self,van) // no return\n       ...\n\n    def deallocate_device(self,van)// no return\n       ...\n\n    def inventory_devices(self, provider_tree)// no return\n       ...","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":597,"context_line":""},{"line_number":598,"context_line":"* initialize()"},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * Called once at startup of Nova compute. Loads Cyborg drivers and"},{"line_number":601,"context_line":"    os-acc extensions using stevedore\u0027s ``ExtensionManager``. Waits for"},{"line_number":602,"context_line":"    Cyborg Agent to be ready to accept requests, i.e., all devices"},{"line_number":603,"context_line":"    enumerated and traits published."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_e7c8e155","line":600,"range":{"start_line":600,"start_character":44,"end_line":600,"end_character":65},"updated":"2018-08-24 21:38:48.000000000","message":"the cyborge drivers should not be running in the nova compute agent process.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":598,"context_line":"* initialize()"},{"line_number":599,"context_line":""},{"line_number":600,"context_line":"  * Called once at startup of Nova compute. Loads Cyborg drivers and"},{"line_number":601,"context_line":"    os-acc extensions using stevedore\u0027s ``ExtensionManager``. Waits for"},{"line_number":602,"context_line":"    Cyborg Agent to be ready to accept requests, i.e., all devices"},{"line_number":603,"context_line":"    enumerated and traits published."},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"  * Nova compute may time out after a reasonable duration if Cyborg"},{"line_number":606,"context_line":"    agent cannot be contacted. The suggested timeout is 20 seconds. This"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_2701f96a","line":603,"range":{"start_line":601,"start_character":61,"end_line":603,"end_character":36},"updated":"2018-08-24 21:38:48.000000000","message":"we should not need to wait here. the init should load stevedor extentions for os-acc bue we should not connect to or start a cyborg agent at this point.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":602,"context_line":"    Cyborg Agent to be ready to accept requests, i.e., all devices"},{"line_number":603,"context_line":"    enumerated and traits published."},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"  * Nova compute may time out after a reasonable duration if Cyborg"},{"line_number":606,"context_line":"    agent cannot be contacted. The suggested timeout is 20 seconds. This"},{"line_number":607,"context_line":"    is configurable in nova.conf, and defaults to immediate timeout if"},{"line_number":608,"context_line":"    no value is specified (to handle deployments without Cyborg)."},{"line_number":609,"context_line":""},{"line_number":610,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":611,"context_line":"    if Cyborg Agent cannot be contacted."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_c72925e6","line":608,"range":{"start_line":605,"start_character":2,"end_line":608,"end_character":65},"updated":"2018-08-24 21:38:48.000000000","message":"i dont think it correct to block nova compute on waiting for cyborg for any timeout.\n\nthe majority of deployment will not have cyborg deployed.\nos-acc should not comunicate with the agent directly if we have a cyborge conductir. it sould communicate vai the rabbit mq message bus asyscronously.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":602,"context_line":"    Cyborg Agent to be ready to accept requests, i.e., all devices"},{"line_number":603,"context_line":"    enumerated and traits published."},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"  * Nova compute may time out after a reasonable duration if Cyborg"},{"line_number":606,"context_line":"    agent cannot be contacted. The suggested timeout is 20 seconds. This"},{"line_number":607,"context_line":"    is configurable in nova.conf, and defaults to immediate timeout if"},{"line_number":608,"context_line":"    no value is specified (to handle deployments without Cyborg)."},{"line_number":609,"context_line":""},{"line_number":610,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":611,"context_line":"    if Cyborg Agent cannot be contacted."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_764e21c2","line":608,"range":{"start_line":605,"start_character":2,"end_line":608,"end_character":65},"in_reply_to":"3f79a3b5_c72925e6","updated":"2018-09-02 02:58:43.000000000","message":"The os-acc structure and flow are being redone in the next version. That should address this and most of the points below.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":614,"context_line":""},{"line_number":615,"context_line":"* attach_for_instance(self, device_rp[], extra_specs, instance_info, vans[])"},{"line_number":616,"context_line":""},{"line_number":617,"context_line":"  All parameters are read-only except vans[]."},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"  * device_rp[]: This is an array with one entry for each request group in"},{"line_number":620,"context_line":"    the granular resource request syntax [#granular]_ that refers to an"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_c757e557","line":617,"range":{"start_line":617,"start_character":38,"end_line":617,"end_character":44},"updated":"2018-08-24 21:38:48.000000000","message":"the fuction prototype on line 586 implies the modifed value will be returned.\n\nfuctions should eiterh have side-effects (modify there inputs)\n\nor return results not both.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"  All parameters are read-only except vans[]."},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"  * device_rp[]: This is an array with one entry for each request group in"},{"line_number":620,"context_line":"    the granular resource request syntax [#granular]_ that refers to an"},{"line_number":621,"context_line":"    accelerator resource class (CUSTOM_ACCELERATOR_*). Note that there can be"},{"line_number":622,"context_line":"    request groups with no references to accelerators, and other groups which"},{"line_number":623,"context_line":"    ask for accelerator RCs and other RCs (e.g. local memory in FPGAs or"},{"line_number":624,"context_line":"    GPUs). The former would not be included in this array whereas the latter"},{"line_number":625,"context_line":"    would be."},{"line_number":626,"context_line":""},{"line_number":627,"context_line":"    Each entry is a dictionary with two fields:"},{"line_number":628,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_4765151d","line":625,"range":{"start_line":619,"start_character":1,"end_line":625,"end_character":13},"updated":"2018-08-24 21:38:48.000000000","message":"at this point i dont think we still the placement query. we only have the allocations as this would be executed on the compute node. the placement query is create in the scudler and it is not passed to the compute node.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"  All parameters are read-only except vans[]."},{"line_number":618,"context_line":""},{"line_number":619,"context_line":"  * device_rp[]: This is an array with one entry for each request group in"},{"line_number":620,"context_line":"    the granular resource request syntax [#granular]_ that refers to an"},{"line_number":621,"context_line":"    accelerator resource class (CUSTOM_ACCELERATOR_*). Note that there can be"},{"line_number":622,"context_line":"    request groups with no references to accelerators, and other groups which"},{"line_number":623,"context_line":"    ask for accelerator RCs and other RCs (e.g. local memory in FPGAs or"},{"line_number":624,"context_line":"    GPUs). The former would not be included in this array whereas the latter"},{"line_number":625,"context_line":"    would be."},{"line_number":626,"context_line":""},{"line_number":627,"context_line":"    Each entry is a dictionary with two fields:"},{"line_number":628,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_7fb703f7","line":625,"range":{"start_line":619,"start_character":1,"end_line":625,"end_character":13},"in_reply_to":"3f79a3b5_4765151d","updated":"2018-09-02 02:58:43.000000000","message":"The request groups etc. are part of the flavor extra_specs.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":626,"context_line":""},{"line_number":627,"context_line":"    Each entry is a dictionary with two fields:"},{"line_number":628,"context_line":""},{"line_number":629,"context_line":"    * request_group_number: index of the request group that references an"},{"line_number":630,"context_line":"      accelerator RC. The unnumbered group is considered to have index 0."},{"line_number":631,"context_line":"    * uuid: The resource provider UUID of the device RP assigned to that"},{"line_number":632,"context_line":"      request group."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a77989c5","line":629,"range":{"start_line":629,"start_character":6,"end_line":629,"end_character":26},"updated":"2018-08-24 21:38:48.000000000","message":"we may or may not have this.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":626,"context_line":""},{"line_number":627,"context_line":"    Each entry is a dictionary with two fields:"},{"line_number":628,"context_line":""},{"line_number":629,"context_line":"    * request_group_number: index of the request group that references an"},{"line_number":630,"context_line":"      accelerator RC. The unnumbered group is considered to have index 0."},{"line_number":631,"context_line":"    * uuid: The resource provider UUID of the device RP assigned to that"},{"line_number":632,"context_line":"      request group."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_d6d4d5bd","line":629,"range":{"start_line":629,"start_character":6,"end_line":629,"end_character":26},"in_reply_to":"3f79a3b5_a77989c5","updated":"2018-09-02 02:58:43.000000000","message":"Addressed in previous comment.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":626,"context_line":""},{"line_number":627,"context_line":"    Each entry is a dictionary with two fields:"},{"line_number":628,"context_line":""},{"line_number":629,"context_line":"    * request_group_number: index of the request group that references an"},{"line_number":630,"context_line":"      accelerator RC. The unnumbered group is considered to have index 0."},{"line_number":631,"context_line":"    * uuid: The resource provider UUID of the device RP assigned to that"},{"line_number":632,"context_line":"      request group."},{"line_number":633,"context_line":""},{"line_number":634,"context_line":"    Note that multiple request groups may be assigned the same device RP"},{"line_number":635,"context_line":"    if `group policy` is `none`."},{"line_number":636,"context_line":""},{"line_number":637,"context_line":"    Note also that the Cyborg-Nova scheduling spec [#cyborg-others]_ allows"},{"line_number":638,"context_line":"    for the keyword `othersN` to supplement the granular request syntax."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_4733d514","line":635,"range":{"start_line":629,"start_character":2,"end_line":635,"end_character":32},"updated":"2018-08-24 21:38:48.000000000","message":"the logic to do the assignment should not be in os-acc.\nif you put it in os acc why would we still need a cyborg agent.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":635,"context_line":"    if `group policy` is `none`."},{"line_number":636,"context_line":""},{"line_number":637,"context_line":"    Note also that the Cyborg-Nova scheduling spec [#cyborg-others]_ allows"},{"line_number":638,"context_line":"    for the keyword `othersN` to supplement the granular request syntax."},{"line_number":639,"context_line":"    This is used to convey requirements or preferences that Nova does not"},{"line_number":640,"context_line":"    understand, such as bitstream IDs."},{"line_number":641,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_87056d16","line":638,"range":{"start_line":638,"start_character":21,"end_line":638,"end_character":28},"updated":"2018-08-24 21:38:48.000000000","message":"at the time we are invoking this fuction we are on the compute node and scudling has been finished for a very long time. why is this relevent. aslso t add an otherN keywork you would need a seperate spec for the placement api change.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":635,"context_line":"    if `group policy` is `none`."},{"line_number":636,"context_line":""},{"line_number":637,"context_line":"    Note also that the Cyborg-Nova scheduling spec [#cyborg-others]_ allows"},{"line_number":638,"context_line":"    for the keyword `othersN` to supplement the granular request syntax."},{"line_number":639,"context_line":"    This is used to convey requirements or preferences that Nova does not"},{"line_number":640,"context_line":"    understand, such as bitstream IDs."},{"line_number":641,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_f681d1c4","line":638,"range":{"start_line":638,"start_character":21,"end_line":638,"end_character":28},"in_reply_to":"3f79a3b5_87056d16","updated":"2018-09-02 02:58:43.000000000","message":"The request contains the flavor extra specs, which contains terms like \u0027others\u0027 that Nova does not interpret. Cyborg interprets those.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":644,"context_line":""},{"line_number":645,"context_line":"  * instance_info: Nova\u0027s per-instance versioned object."},{"line_number":646,"context_line":""},{"line_number":647,"context_line":"    * The VM image UUID may be used to query Glance for metadata about"},{"line_number":648,"context_line":"      accelerator requirements that may be stored with the VM image"},{"line_number":649,"context_line":"      (i.e., traits in images instead of flavors)."},{"line_number":650,"context_line":""},{"line_number":651,"context_line":"    * The project ID may be used for policy-based decisions in the"},{"line_number":652,"context_line":"      future if more than one bitstream matches the need."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_02206b9a","line":649,"range":{"start_line":647,"start_character":5,"end_line":649,"end_character":50},"updated":"2018-08-24 21:38:48.000000000","message":"nova has the image metadata it can pass that in so you dont have to query it.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":648,"context_line":"      accelerator requirements that may be stored with the VM image"},{"line_number":649,"context_line":"      (i.e., traits in images instead of flavors)."},{"line_number":650,"context_line":""},{"line_number":651,"context_line":"    * The project ID may be used for policy-based decisions in the"},{"line_number":652,"context_line":"      future if more than one bitstream matches the need."},{"line_number":653,"context_line":""},{"line_number":654,"context_line":"  * vans[]: Array of VAN objects, one per requested accelerator. Note"},{"line_number":655,"context_line":"    that the number of requested accelerators may be equal to or more"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_a25c1719","line":652,"range":{"start_line":651,"start_character":5,"end_line":652,"end_character":57},"updated":"2018-08-24 21:38:48.000000000","message":"the bitstream is a glance image id. you wont be able to chaange that unless you make it a list..\n\ni would just remove this and readd it if it actully is need in the future.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":655,"context_line":"    that the number of requested accelerators may be equal to or more"},{"line_number":656,"context_line":"    than the number of request groups that refer to accelerator RCs."},{"line_number":657,"context_line":""},{"line_number":658,"context_line":"  * Returns: vans[], an array of VAN objects, one per requested accelerator."},{"line_number":659,"context_line":""},{"line_number":660,"context_line":"  * Summary: Prepares the attach handle for every requested accelerator,"},{"line_number":661,"context_line":"    possibly after configuring the device as needed, binds it to a VAN"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_62a03ff5","line":658,"range":{"start_line":658,"start_character":3,"end_line":658,"end_character":75},"updated":"2018-08-24 21:38:48.000000000","message":"since you are returning the arry you should make a copy of the input array instead of modifying it.\n\notherswise dont return anything an just modify the input array.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":695,"context_line":"Os-acc - Cyborg Driver API"},{"line_number":696,"context_line":"--------------------------"},{"line_number":697,"context_line":""},{"line_number":698,"context_line":"The os-acc loads all Cyborg drivers (and extensions) using stevedore\u0027s"},{"line_number":699,"context_line":"`ExtensionManager` class. All drivers shall be in the namespace `cyborg`."},{"line_number":700,"context_line":"The operator must configure the needed drivers in os-acc\u0027s `setup.cfg` file."},{"line_number":701,"context_line":""},{"line_number":702,"context_line":"For each VAN specified in `attach_for_instance()`, the os-acc will choose"},{"line_number":703,"context_line":"a driver by querying every loaded driver in alphabetical order of driver"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_828f9b73","line":700,"range":{"start_line":698,"start_character":0,"end_line":700,"end_character":76},"updated":"2018-08-24 21:38:48.000000000","message":"no the cyborg drivers shoudld run in the cyborge agent.\nyou should not need to load them into the nova-compute process.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":699,"context_line":"`ExtensionManager` class. All drivers shall be in the namespace `cyborg`."},{"line_number":700,"context_line":"The operator must configure the needed drivers in os-acc\u0027s `setup.cfg` file."},{"line_number":701,"context_line":""},{"line_number":702,"context_line":"For each VAN specified in `attach_for_instance()`, the os-acc will choose"},{"line_number":703,"context_line":"a driver by querying every loaded driver in alphabetical order of driver"},{"line_number":704,"context_line":"name through the `can_handle()` API to check if it can handle that"},{"line_number":705,"context_line":"request. The first driver to \u0027claim\u0027 the request gets to handle it. If"},{"line_number":706,"context_line":"there are two or more drivers that can potentially handle the request,"},{"line_number":707,"context_line":"the operator can prioritize one by naming it suitably in the `setup.cfg`"},{"line_number":708,"context_line":"file (similar to rc scripts in Linux)."},{"line_number":709,"context_line":""},{"line_number":710,"context_line":"Since the number of os-acc drivers in a given host is expected to be"},{"line_number":711,"context_line":"small (even if the number in the community is large), this may"},{"line_number":712,"context_line":"be efficient enough. If further efficiencies are needed in the future, we"},{"line_number":713,"context_line":"could:"},{"line_number":714,"context_line":""},{"line_number":715,"context_line":"* Cache the driver data in some form."},{"line_number":716,"context_line":"* Have each driver provide metadata about its parameters such as supported"},{"line_number":717,"context_line":"  device types, and organize separate driver chains. The resulting chains"},{"line_number":718,"context_line":"  will be shorter."},{"line_number":719,"context_line":""},{"line_number":720,"context_line":"The Cyborg drivers shall expose the following APIs to os-acc.::"},{"line_number":721,"context_line":""},{"line_number":722,"context_line":"  class CyborgBaseDriver(object):"},{"line_number":723,"context_line":""},{"line_number":724,"context_line":"     @classmethod"},{"line_number":725,"context_line":"     def load(cls, driver_name)"},{"line_number":726,"context_line":"        raise NotImplementedError"},{"line_number":727,"context_line":""},{"line_number":728,"context_line":"     @classmethod"},{"line_number":729,"context_line":"     def can_handle(cls, rc, van, device_rp, extra_specs, instance_info):"},{"line_number":730,"context_line":"        # rc: resource class of the accelerator e.g. CUSTOM_ACCELERATOR_GPU"},{"line_number":731,"context_line":"        # device_rp implicitly indicates the device family"},{"line_number":732,"context_line":"        # returns True/False"},{"line_number":733,"context_line":"        return False"},{"line_number":734,"context_line":""},{"line_number":735,"context_line":"     def prepareVAN(self, van, device_rp, extra_specs):"},{"line_number":736,"context_line":"        . . ."},{"line_number":737,"context_line":"        return van"},{"line_number":738,"context_line":""},{"line_number":739,"context_line":""},{"line_number":740,"context_line":"     def cleanupVAN(self, van, device_rp):"},{"line_number":741,"context_line":"        . . ."},{"line_number":742,"context_line":"        return None"},{"line_number":743,"context_line":""},{"line_number":744,"context_line":"     def deleteVAN(self, van, device_rp):"},{"line_number":745,"context_line":"        . . ."},{"line_number":746,"context_line":"        return None"},{"line_number":747,"context_line":""},{"line_number":748,"context_line":"     # Plus other methods exposed to Cyborg"},{"line_number":749,"context_line":""},{"line_number":750,"context_line":"See the workflow sequence described earlier (attach_) for the descriptions."},{"line_number":751,"context_line":""},{"line_number":752,"context_line":".. _FPGAProg:"},{"line_number":753,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_62659fb4","line":750,"range":{"start_line":702,"start_character":0,"end_line":750,"end_character":75},"updated":"2018-08-24 21:38:48.000000000","message":"if we execute this from os-acc we nolonger and we do the device assignment in os-acc as was suggesed before tehn we dont need any cyborg anymore.\n\nwe dont need an api, conducore, agent or database.\n\n\nthis wont work in the general case as you cannot assume the nova compute agent is running on the same physical sever as the fpga or has the right right permisions to execute the device specific driver code.\n\nos-acc should be making an rpc call to the cyborg agent here instaed.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":699,"context_line":"`ExtensionManager` class. All drivers shall be in the namespace `cyborg`."},{"line_number":700,"context_line":"The operator must configure the needed drivers in os-acc\u0027s `setup.cfg` file."},{"line_number":701,"context_line":""},{"line_number":702,"context_line":"For each VAN specified in `attach_for_instance()`, the os-acc will choose"},{"line_number":703,"context_line":"a driver by querying every loaded driver in alphabetical order of driver"},{"line_number":704,"context_line":"name through the `can_handle()` API to check if it can handle that"},{"line_number":705,"context_line":"request. The first driver to \u0027claim\u0027 the request gets to handle it. If"},{"line_number":706,"context_line":"there are two or more drivers that can potentially handle the request,"},{"line_number":707,"context_line":"the operator can prioritize one by naming it suitably in the `setup.cfg`"},{"line_number":708,"context_line":"file (similar to rc scripts in Linux)."},{"line_number":709,"context_line":""},{"line_number":710,"context_line":"Since the number of os-acc drivers in a given host is expected to be"},{"line_number":711,"context_line":"small (even if the number in the community is large), this may"},{"line_number":712,"context_line":"be efficient enough. If further efficiencies are needed in the future, we"},{"line_number":713,"context_line":"could:"},{"line_number":714,"context_line":""},{"line_number":715,"context_line":"* Cache the driver data in some form."},{"line_number":716,"context_line":"* Have each driver provide metadata about its parameters such as supported"},{"line_number":717,"context_line":"  device types, and organize separate driver chains. The resulting chains"},{"line_number":718,"context_line":"  will be shorter."},{"line_number":719,"context_line":""},{"line_number":720,"context_line":"The Cyborg drivers shall expose the following APIs to os-acc.::"},{"line_number":721,"context_line":""},{"line_number":722,"context_line":"  class CyborgBaseDriver(object):"},{"line_number":723,"context_line":""},{"line_number":724,"context_line":"     @classmethod"},{"line_number":725,"context_line":"     def load(cls, driver_name)"},{"line_number":726,"context_line":"        raise NotImplementedError"},{"line_number":727,"context_line":""},{"line_number":728,"context_line":"     @classmethod"},{"line_number":729,"context_line":"     def can_handle(cls, rc, van, device_rp, extra_specs, instance_info):"},{"line_number":730,"context_line":"        # rc: resource class of the accelerator e.g. CUSTOM_ACCELERATOR_GPU"},{"line_number":731,"context_line":"        # device_rp implicitly indicates the device family"},{"line_number":732,"context_line":"        # returns True/False"},{"line_number":733,"context_line":"        return False"},{"line_number":734,"context_line":""},{"line_number":735,"context_line":"     def prepareVAN(self, van, device_rp, extra_specs):"},{"line_number":736,"context_line":"        . . ."},{"line_number":737,"context_line":"        return van"},{"line_number":738,"context_line":""},{"line_number":739,"context_line":""},{"line_number":740,"context_line":"     def cleanupVAN(self, van, device_rp):"},{"line_number":741,"context_line":"        . . ."},{"line_number":742,"context_line":"        return None"},{"line_number":743,"context_line":""},{"line_number":744,"context_line":"     def deleteVAN(self, van, device_rp):"},{"line_number":745,"context_line":"        . . ."},{"line_number":746,"context_line":"        return None"},{"line_number":747,"context_line":""},{"line_number":748,"context_line":"     # Plus other methods exposed to Cyborg"},{"line_number":749,"context_line":""},{"line_number":750,"context_line":"See the workflow sequence described earlier (attach_) for the descriptions."},{"line_number":751,"context_line":""},{"line_number":752,"context_line":".. _FPGAProg:"},{"line_number":753,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_f120fb9d","line":750,"range":{"start_line":702,"start_character":0,"end_line":750,"end_character":75},"in_reply_to":"3f79a3b5_62659fb4","updated":"2018-09-02 02:58:43.000000000","message":"? Where would you persist the VANs if there is no Cyborg db? There is some disconnect here.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":900,"context_line":"and let them handle further sub-plugins for different devices. That"},{"line_number":901,"context_line":"is probably unnecessary."},{"line_number":902,"context_line":""},{"line_number":903,"context_line":"Data model impact"},{"line_number":904,"context_line":"-----------------"},{"line_number":905,"context_line":""},{"line_number":906,"context_line":"Cyborg needs to persist the VANs."}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_c2235355","line":903,"range":{"start_line":903,"start_character":0,"end_line":903,"end_character":17},"updated":"2018-08-24 21:38:48.000000000","message":"all of the version objecst should have been descibed in this section.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":919,"context_line":""},{"line_number":920,"context_line":"Nova may need a REST API to post and delete instance - VAN associations."},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"Security impact"},{"line_number":923,"context_line":"---------------"},{"line_number":924,"context_line":""},{"line_number":925,"context_line":"None"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_42a3c3b2","line":922,"range":{"start_line":922,"start_character":0,"end_line":922,"end_character":15},"updated":"2018-08-24 21:38:48.000000000","message":"the abiltiy to load plugins that modif the gues xml generation allows another attack vector for malitous code.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"7f9d97bd82fb520172d9b182f53e2b4d53641e73","unresolved":false,"context_lines":[{"line_number":935,"context_line":""},{"line_number":936,"context_line":"None"},{"line_number":937,"context_line":""},{"line_number":938,"context_line":"Performance Impact"},{"line_number":939,"context_line":"------------------"},{"line_number":940,"context_line":""},{"line_number":941,"context_line":"None"},{"line_number":942,"context_line":""},{"line_number":943,"context_line":"Other deployer impact"},{"line_number":944,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_42d5031b","line":941,"range":{"start_line":938,"start_character":1,"end_line":941,"end_character":4},"updated":"2018-08-24 21:38:48.000000000","message":"as descriped there will be a signifacte overhead to instance spwaning and most nova compute agents will have a 20 second delay instarting as they wait for a responce form a cyborge agent that does not exist.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c86bca3e8f8da2cb1acbdc0e742cc7cef2966895","unresolved":false,"context_lines":[{"line_number":935,"context_line":""},{"line_number":936,"context_line":"None"},{"line_number":937,"context_line":""},{"line_number":938,"context_line":"Performance Impact"},{"line_number":939,"context_line":"------------------"},{"line_number":940,"context_line":""},{"line_number":941,"context_line":"None"},{"line_number":942,"context_line":""},{"line_number":943,"context_line":"Other deployer impact"},{"line_number":944,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"3f79a3b5_36ef4988","line":941,"range":{"start_line":938,"start_character":1,"end_line":941,"end_character":4},"in_reply_to":"3f79a3b5_42d5031b","updated":"2018-09-02 02:58:43.000000000","message":"This is a misunderstanding. I am not saying that n-cpu will wait on each instance request. Anyway, the next version will update this significantly.","commit_id":"0192efc44b55594c2f783a0ccd45ed8c86c21000"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"0df3c618b2488f3f24aca413efb136c1305dfd11","unresolved":false,"context_lines":[{"line_number":1,"context_line":".."},{"line_number":2,"context_line":" This work is licensed under a Creative Commons Attribution 3.0 Unported"},{"line_number":3,"context_line":" License."},{"line_number":4,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"3f79a3b5_61dd48dc","line":1,"updated":"2018-09-24 23:50:59.000000000","message":"i have left this comment on previous review but its gnerally considerd\nbad pratice to update specs for previous releases as it\ndestorys the history of what was planned and when.\ncan you remove these changes form the review.","commit_id":"64c73a4e12430e4cdecd397584eda1bff10220d1"}],"doc/source/specs/rocky/compute-node.rst":[{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"5f7c97a3_92d80d25","line":95,"range":{"start_line":93,"start_character":0,"end_line":95,"end_character":56},"updated":"2018-06-28 00:46:16.000000000","message":"I\u0027m not convinced the VANs deserve to be part of this diagram.  They\u0027re really just a metadata object we pass around, containing configuration and attachment information for the accelerator.  E.g. for the plug operation, you pass me a VAN and an InstanceInfo, I attach the accelerator (not the VAN) to the instance, I (maybe) update the VAN, and I return it to you.","commit_id":"a99de8289a5a08c0f22945db65c78d8679cc43bb"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"5f7c97a3_b2dd0915","line":134,"range":{"start_line":134,"start_character":45,"end_line":134,"end_character":54},"updated":"2018-06-28 00:46:16.000000000","message":"accelerator","commit_id":"a99de8289a5a08c0f22945db65c78d8679cc43bb"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"5f7c97a3_52e275d6","line":151,"range":{"start_line":151,"start_character":34,"end_line":151,"end_character":36},"updated":"2018-06-28 00:46:16.000000000","message":"strike","commit_id":"a99de8289a5a08c0f22945db65c78d8679cc43bb"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a361a2e9a45746faaf64390b0ebd8b829be1fcad","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"5f7c97a3_12ccfd60","line":155,"range":{"start_line":155,"start_character":14,"end_line":155,"end_character":16},"updated":"2018-06-28 00:46:16.000000000","message":"strike","commit_id":"a99de8289a5a08c0f22945db65c78d8679cc43bb"}],"doc/source/specs/stein/approved/compute-node.rst":[{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"863543e541f252e3fb93ebd69503a4a02e612be7","unresolved":false,"context_lines":[{"line_number":284,"context_line":"   The result of this step is an updated VAN that can be used to attach"},{"line_number":285,"context_line":"   the accelerator to the instance with the needed constraints."},{"line_number":286,"context_line":""},{"line_number":287,"context_line":"   The specific use cases for this step, such as NUMA affinity for devices,"},{"line_number":288,"context_line":"   are out of scope for this spec and can be taken up in the future. A few"},{"line_number":289,"context_line":"   examples are provided below to illustrate what this step may involve:"},{"line_number":290,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"3f79a3b5_fe5b4e52","line":287,"range":{"start_line":287,"start_character":41,"end_line":287,"end_character":75},"updated":"2018-09-10 16:20:42.000000000","message":"You should choice device based on the NUMA affinity, then configure the device, so that should be done by the first step?","commit_id":"64c73a4e12430e4cdecd397584eda1bff10220d1"}],"doc/specs/rocky/compute-node.rst":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":115,"context_line":"Cyborg in Rocky."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"In general, when accelerators are detached from an instance, the associated"},{"line_number":118,"context_line":"resources are released. So, if the instance could be restarted after a"},{"line_number":119,"context_line":"disruptive operation, either the accelerators should not be detached or"},{"line_number":120,"context_line":"the instance must get rescheduled on restart."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The list of supported operations (with the intended behaviors) are as follows:"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_dc6821f5","line":120,"range":{"start_line":118,"start_character":24,"end_line":120,"end_character":45},"updated":"2018-06-25 15:07:46.000000000","message":"What does \u0027disruptive\u0027 means here? \nI don\u0027t think nova release any other resources (even external ones like cinder volume or neutron port) during instance restart.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":115,"context_line":"Cyborg in Rocky."},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"In general, when accelerators are detached from an instance, the associated"},{"line_number":118,"context_line":"resources are released. So, if the instance could be restarted after a"},{"line_number":119,"context_line":"disruptive operation, either the accelerators should not be detached or"},{"line_number":120,"context_line":"the instance must get rescheduled on restart."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The list of supported operations (with the intended behaviors) are as follows:"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_01a4e4b7","line":120,"range":{"start_line":118,"start_character":24,"end_line":120,"end_character":45},"in_reply_to":"5f7c97a3_dc6821f5","updated":"2018-06-27 03:54:51.000000000","message":"\u0027Disruptive\u0027 means the instance\u0027s operation is affected: e.g. reboot, deletion. On some of those, the accelerator resources are not (and should not be) detached - e.g. reboot. On others, they are detached - e.g. deletion. But, if you restore after a deletion, the instance should be rescheduled to get new accelerator resources.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are detached. On an"},{"line_number":134,"context_line":"  unshelve, it is expected that the build operation will go through the"},{"line_number":135,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":136,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_7c50ed2c","line":133,"range":{"start_line":133,"start_character":2,"end_line":133,"end_character":69},"updated":"2018-06-25 15:07:46.000000000","message":"I think you mean shelved and offloaded here as nova only release resource during shelve offload not during shelve.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are detached. On an"},{"line_number":134,"context_line":"  unshelve, it is expected that the build operation will go through the"},{"line_number":135,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":136,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_f34f2789","line":133,"range":{"start_line":133,"start_character":2,"end_line":133,"end_character":69},"in_reply_to":"5f7c97a3_7c50ed2c","updated":"2018-06-27 03:54:51.000000000","message":"Agreed","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are detached. On an"},{"line_number":134,"context_line":"  unshelve, it is expected that the build operation will go through the"},{"line_number":135,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_9c24e9c7","line":134,"range":{"start_line":133,"start_character":71,"end_line":134,"end_character":10},"updated":"2018-06-25 15:07:46.000000000","message":"On an unshelve, if the instance was offloaded after shelve, it is expected...","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":130,"context_line":"  So, when the instance is restarted, unpaused, resumed or unlocked,"},{"line_number":131,"context_line":"  there is nothing to do."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* When an instance is shelved, the accelerator resources are detached. On an"},{"line_number":134,"context_line":"  unshelve, it is expected that the build operation will go through the"},{"line_number":135,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_331fdf70","line":134,"range":{"start_line":133,"start_character":71,"end_line":134,"end_character":10},"in_reply_to":"5f7c97a3_9c24e9c7","updated":"2018-06-27 03:54:51.000000000","message":"Ok","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  unshelve, it is expected that the build operation will go through the"},{"line_number":135,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"},{"line_number":138,"context_line":"  restore, it is expected that the build operation will go through the"},{"line_number":139,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":142,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_e71d4662","line":139,"range":{"start_line":137,"start_character":71,"end_line":139,"end_character":60},"updated":"2018-06-25 15:07:46.000000000","message":"As far as I know restore in nova only allowed if the instance is not reclaimed yet from the compute it is originally scheduled on. This also means that the resources towards that compute hasn\u0027t been released yet. So a restore will not cause a reschedule as it doesn\u0027t need to do that.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  unshelve, it is expected that the build operation will go through the"},{"line_number":135,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* When an instance is deleted, the accelerator resources are detached. On a"},{"line_number":138,"context_line":"  restore, it is expected that the build operation will go through the"},{"line_number":139,"context_line":"  scheduler again, so it is equivalent to an instance start."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":142,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_0e4b9627","line":139,"range":{"start_line":137,"start_character":71,"end_line":139,"end_character":60},"in_reply_to":"5f7c97a3_e71d4662","updated":"2018-06-27 03:54:51.000000000","message":"The wording in https://developer.openstack.org/api-guide/compute/server_concepts.html \nis a bit confusing, for both shelve and delete. It starts by saying that the resources will be released, \nthen talks of a timeout that can affect that behavior. \n\nIs there a definitive statement on these operations?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":140,"context_line":""},{"line_number":141,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":142,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":143,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":144,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":145,"context_line":"  precondition, accelerator resources are left attached to the instance"},{"line_number":146,"context_line":"  during the rebuild."},{"line_number":147,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_8741f238","line":144,"range":{"start_line":143,"start_character":53,"end_line":144,"end_character":56},"updated":"2018-06-25 15:07:46.000000000","message":"Is there any other document describing the quiescing?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":140,"context_line":""},{"line_number":141,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":142,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":143,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":144,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":145,"context_line":"  precondition, accelerator resources are left attached to the instance"},{"line_number":146,"context_line":"  during the rebuild."},{"line_number":147,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_44364fe9","line":144,"range":{"start_line":143,"start_character":53,"end_line":144,"end_character":56},"in_reply_to":"5f7c97a3_8741f238","updated":"2018-06-27 03:54:51.000000000","message":"I don\u0027t think so. It will involve resetting the device in some vendor-specific way and preventing further access to the device from the VM. The easiest way to do that is for the VM\u0027s vendor driver to issue an accelerator reset, and ensure no further access.\n  If we want this in Nova/Cyborg, we could possibly require every Cyborg driver to provide an accelerator reset API. However, that would not ensure the VM will not access the device after the reset.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":141,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":142,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":143,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":144,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":145,"context_line":"  precondition, accelerator resources are left attached to the instance"},{"line_number":146,"context_line":"  during the rebuild."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_c764cae7","line":145,"range":{"start_line":144,"start_character":57,"end_line":145,"end_character":14},"updated":"2018-06-25 15:07:46.000000000","message":"What breaks such precondition is not fulfilled?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":141,"context_line":"* Rebuild: Prior to the instance image replacement, all device access must"},{"line_number":142,"context_line":"  be quiesced, i.e., accesses to devices from that instance must be"},{"line_number":143,"context_line":"  completed and further accesses must be prohibited. The mechanics of"},{"line_number":144,"context_line":"  such quiescing are outside the scope of this document. With that"},{"line_number":145,"context_line":"  precondition, accelerator resources are left attached to the instance"},{"line_number":146,"context_line":"  during the rebuild."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"* Resize (with change of flavor): It is equivalent to a termination followed"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_a475e399","line":145,"range":{"start_line":144,"start_character":57,"end_line":145,"end_character":14},"in_reply_to":"5f7c97a3_c764cae7","updated":"2018-06-27 03:54:51.000000000","message":"The consequences depend on the implementation. For Intel FPGAs, the accelerator or the region containing the accelerator may go into error state, and cannot be used till it is reset and the errors are cleared. Other vendors can comment on their devices.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":155,"context_line":""},{"line_number":156,"context_line":"* Evacuate: This is a forcible rebuild by the administrator. As the"},{"line_number":157,"context_line":"  semantics of evacuation are left open even without accelerators,"},{"line_number":158,"context_line":"  Cyborg\u0027s behavior is also left undefined."},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"* Set administrator password, trigger crash dump: These are supported and"},{"line_number":161,"context_line":"  are no-ops for accelerators."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_e779c681","line":158,"updated":"2018-06-25 15:07:46.000000000","message":"For me the evacuation is defined as \u0027rebuild the instance on another host as the current host is unavailable\u0027. I think from cyborg perspective it is equivalent to the new schedule and build on the new host.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":155,"context_line":""},{"line_number":156,"context_line":"* Evacuate: This is a forcible rebuild by the administrator. As the"},{"line_number":157,"context_line":"  semantics of evacuation are left open even without accelerators,"},{"line_number":158,"context_line":"  Cyborg\u0027s behavior is also left undefined."},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"* Set administrator password, trigger crash dump: These are supported and"},{"line_number":161,"context_line":"  are no-ops for accelerators."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_057610d9","line":158,"in_reply_to":"5f7c97a3_e779c681","updated":"2018-06-27 03:54:51.000000000","message":"There doesn\u0027t seem to be any official document giving a clear semantics. Would you recommend that we define this operation in this spec for Cyborg as you said?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":168,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":169,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":170,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"},{"line_number":171,"context_line":"  the instance."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Booting off a snapshot: The accelerator resources are left intact. But"},{"line_number":174,"context_line":"  the snapshot may have been taken when the attached accelerators were"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_272ede42","line":171,"updated":"2018-06-25 15:07:46.000000000","message":"Not allowing the user to reboot a stuck instance from the OpenStack API seems like a serious limitation to me. What else the user can do with this VM to get it back? \n\nCould nova do detach and attach (or an explicit reset) during reboot to notify cyborg to do a device reset?\n\nCould cyborg listen to reboot notifications from nova and reset the device?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":168,"context_line":"  buffers and other resources are lost after reboot, so any attempt by"},{"line_number":169,"context_line":"  the device to access those resources can be problematic. The instance"},{"line_number":170,"context_line":"  can reset the accelerator(s) after reboot, but that requires trusting"},{"line_number":171,"context_line":"  the instance."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Booting off a snapshot: The accelerator resources are left intact. But"},{"line_number":174,"context_line":"  the snapshot may have been taken when the attached accelerators were"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_c5549824","line":171,"in_reply_to":"5f7c97a3_272ede42","updated":"2018-06-27 03:54:51.000000000","message":"The user could ssh to an instance and reboot it, without Nova knowing about it.\nAlso, this spec does allow the user to reboot.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":175,"context_line":"  in a particular state. When booting off a previous snapshot, the"},{"line_number":176,"context_line":"  current configuration and state of accelerators may not match the"},{"line_number":177,"context_line":"  snapshot. The instance can reset the accelerator(s) to a clean state"},{"line_number":178,"context_line":"  after the boot, but that requires trusting the instance."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"The following instance operations are not supported in this release:"},{"line_number":181,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_27d33e3a","line":178,"updated":"2018-06-25 15:07:46.000000000","message":"ditto: can nova let cyborg know that a reset is needed?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":175,"context_line":"  in a particular state. When booting off a previous snapshot, the"},{"line_number":176,"context_line":"  current configuration and state of accelerators may not match the"},{"line_number":177,"context_line":"  snapshot. The instance can reset the accelerator(s) to a clean state"},{"line_number":178,"context_line":"  after the boot, but that requires trusting the instance."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"The following instance operations are not supported in this release:"},{"line_number":181,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_0821454d","line":178,"in_reply_to":"5f7c97a3_27d33e3a","updated":"2018-06-27 03:54:51.000000000","message":"Yes. We could require Cyborg drivers to implement a accelerator reset. Nova -\u003e os-acc -\u003e plugin -\u003e Cyborg agent -\u003e Cyborg driver path can be used to address this.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":14131,"name":"shaohef","email":"shaohe.feng@intel.com","username":"shaohefeng"},"change_message_id":"b0984e59bba2276dd81865bb6d1243d365d78bf2","unresolved":false,"context_lines":[{"line_number":322,"context_line":"   (through some mechanism TBD), n-cpu calls ``unplug()`` API in os-acc,"},{"line_number":323,"context_line":"   passing the relevant VAN(s). The os-acc calls the right plugin to get"},{"line_number":324,"context_line":"   the accelerator instances detached. The plugin *may* call the Cyborg"},{"line_number":325,"context_line":"   agent to do the detach, but *must* call Cyborg to record the release"},{"line_number":326,"context_line":"   of the resources."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"VANs and plugins"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_41874a62","line":325,"range":{"start_line":325,"start_character":38,"end_line":325,"end_character":49},"updated":"2018-06-25 12:27:04.000000000","message":"here \"call Cyborg\" means by cyborg API?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"d3b48400e74a21e66b0cce44c2a9b78ab8c8110d","unresolved":false,"context_lines":[{"line_number":322,"context_line":"   (through some mechanism TBD), n-cpu calls ``unplug()`` API in os-acc,"},{"line_number":323,"context_line":"   passing the relevant VAN(s). The os-acc calls the right plugin to get"},{"line_number":324,"context_line":"   the accelerator instances detached. The plugin *may* call the Cyborg"},{"line_number":325,"context_line":"   agent to do the detach, but *must* call Cyborg to record the release"},{"line_number":326,"context_line":"   of the resources."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"VANs and plugins"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_b6dc6174","line":325,"range":{"start_line":325,"start_character":38,"end_line":325,"end_character":49},"in_reply_to":"5f7c97a3_41874a62","updated":"2018-06-25 14:41:29.000000000","message":"Either the plugin calls Cyborg agent to delegate work and the agent calls the Cyborg REST API, or the plugin calls Cyborg API directly.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":323,"context_line":"   passing the relevant VAN(s). The os-acc calls the right plugin to get"},{"line_number":324,"context_line":"   the accelerator instances detached. The plugin *may* call the Cyborg"},{"line_number":325,"context_line":"   agent to do the detach, but *must* call Cyborg to record the release"},{"line_number":326,"context_line":"   of the resources."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"VANs and plugins"},{"line_number":329,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_a703ce41","line":326,"updated":"2018-06-25 15:07:46.000000000","message":"And nova also needs to modify/delete the instance allocation in placement.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":323,"context_line":"   passing the relevant VAN(s). The os-acc calls the right plugin to get"},{"line_number":324,"context_line":"   the accelerator instances detached. The plugin *may* call the Cyborg"},{"line_number":325,"context_line":"   agent to do the detach, but *must* call Cyborg to record the release"},{"line_number":326,"context_line":"   of the resources."},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"VANs and plugins"},{"line_number":329,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_e5fd1409","line":326,"in_reply_to":"5f7c97a3_a703ce41","updated":"2018-06-27 03:54:51.000000000","message":"Agreed. Will add.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":440,"context_line":"-------------------------"},{"line_number":441,"context_line":""},{"line_number":442,"context_line":"Nova Compute calls the os-acc APIs if it sees that the requested flavor"},{"line_number":443,"context_line":"refers to CUSTOM_ACCELERATOR_* resource classes. The APIs are called"},{"line_number":444,"context_line":"asynchronously, as suggested below::"},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"   with ThreadPoolExecutor(max_workers\u003dN) as executor:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_872572c1","line":443,"range":{"start_line":443,"start_character":9,"end_line":443,"end_character":30},"updated":"2018-06-25 15:07:46.000000000","message":"If we require nova to handle a specific subset of resources differently then I think having that subset defined only as a CUSTOM_ resource class prefix is not enough. CUSTOM_ resource can be defined by any deployer not knowing that actually CUSTOM_ACCELERATOR_* is reserved for a very specific purpose.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":440,"context_line":"-------------------------"},{"line_number":441,"context_line":""},{"line_number":442,"context_line":"Nova Compute calls the os-acc APIs if it sees that the requested flavor"},{"line_number":443,"context_line":"refers to CUSTOM_ACCELERATOR_* resource classes. The APIs are called"},{"line_number":444,"context_line":"asynchronously, as suggested below::"},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"   with ThreadPoolExecutor(max_workers\u003dN) as executor:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_481dfd5f","line":443,"range":{"start_line":443,"start_character":9,"end_line":443,"end_character":30},"in_reply_to":"5f7c97a3_872572c1","updated":"2018-06-27 03:54:51.000000000","message":"OTOH, if we require the device types to be known to Nova, the addition of every new device type in Cyborg will require a Nova change. They may be relatively rare, but is Nova ok with that?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":479,"context_line":"  .. _initialize:"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"  * Called once at startup of Nova compute. Loads os-acc plugins using"},{"line_number":482,"context_line":"    stevedore\u0027s ``ExtensionManager``. Waits for Cyborg Agent to be ready"},{"line_number":483,"context_line":"    to accept requests, i.e., all devices enumerated and traits published."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":486,"context_line":"    if Cyborg Agent cannot be contacted."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_e2521412","line":483,"range":{"start_line":482,"start_character":38,"end_line":483,"end_character":22},"updated":"2018-06-25 15:07:46.000000000","message":"Does it mean that nova-compute cannot start if the cyborg agents are not running?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":479,"context_line":"  .. _initialize:"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"  * Called once at startup of Nova compute. Loads os-acc plugins using"},{"line_number":482,"context_line":"    stevedore\u0027s ``ExtensionManager``. Waits for Cyborg Agent to be ready"},{"line_number":483,"context_line":"    to accept requests, i.e., all devices enumerated and traits published."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":486,"context_line":"    if Cyborg Agent cannot be contacted."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_4852dd13","line":483,"range":{"start_line":482,"start_character":38,"end_line":483,"end_character":22},"in_reply_to":"5f7c97a3_e2521412","updated":"2018-06-27 03:54:51.000000000","message":"We can specify a reasonable timeout - say 20 seconds?\n\nAlso, I should say that initialize() is also called asynchronously, so that n-cpu can do other things while waiting.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":482,"context_line":"    stevedore\u0027s ``ExtensionManager``. Waits for Cyborg Agent to be ready"},{"line_number":483,"context_line":"    to accept requests, i.e., all devices enumerated and traits published."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":486,"context_line":"    if Cyborg Agent cannot be contacted."},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"* getVAN(self, device_rp, extra_specs, instance_info)"},{"line_number":489,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_02a4f005","line":486,"range":{"start_line":485,"start_character":29,"end_line":486,"end_character":40},"updated":"2018-06-25 15:07:46.000000000","message":"OK so there is at least a timeout and after that we let nova-compute to start up without cyborg-agent. Is there a way for the deployer of nova to state that cyborg is not used so therefore nova-compute does not have to wait for the timeout at every restart in such situation?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":482,"context_line":"    stevedore\u0027s ``ExtensionManager``. Waits for Cyborg Agent to be ready"},{"line_number":483,"context_line":"    to accept requests, i.e., all devices enumerated and traits published."},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"  * Returns None on success. Throws ``CyborgAgentUnavailable`` exception"},{"line_number":486,"context_line":"    if Cyborg Agent cannot be contacted."},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"* getVAN(self, device_rp, extra_specs, instance_info)"},{"line_number":489,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_08406534","line":486,"range":{"start_line":485,"start_character":29,"end_line":486,"end_character":40},"in_reply_to":"5f7c97a3_02a4f005","updated":"2018-06-27 03:54:51.000000000","message":"Can the deployer just drop the timeout value to 1 second or something?","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":487,"context_line":""},{"line_number":488,"context_line":"* getVAN(self, device_rp, extra_specs, instance_info)"},{"line_number":489,"context_line":""},{"line_number":490,"context_line":"  * device_rp: The nested RP selected by the scheduler is"},{"line_number":491,"context_line":"      needed for Cyborg to correlate it to a deployable. Read-only."},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"  * extra_specs: the extra_specs field in the flavor, including"},{"line_number":494,"context_line":"      fields interpreted by Cyborg but not Nova. Read-only."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_160d158e","line":491,"range":{"start_line":490,"start_character":15,"end_line":491,"end_character":55},"updated":"2018-06-25 15:07:46.000000000","message":"1) What if the flavor contains resources1: {CUSTOM_ACCELERATOR_FOO \u003d 2}. Do we expect a single getVAN() call from nova to os-acc or two separate calls?\n\n2) What if the flavor contains resources1: {CUSTOM_ACCELERATOR_FOO\u003d1}, resources2: {CUSTOM_ACCELERATOR_FOO1}} As in this case the two accelerators can be allocated from two different RPs (depending on the isolation policy in the extra_spec set to isolate) a single getVAN() call will not be good as the signature only accepts a single RP. \n\nBecause of this I assume that the nova needs to call os-acc twice both in 1) and 2).\n\nWhat if the flavor contains resources:{CUSTOM_ACCELERATOR_FOO\u003d1, CUSTOM_ACCELERATOR_BAR\u003d1} (or resources1:{CUSTOM_ACCELERATOR_FOO\u003d1}, resources2:{CUSTOM_ACCELERATOR_BAR\u003d1} and isolation policy is set to None). This allows the placement to fulfill the request from a single RP but nova still has to do two different getVAN() calls to os-acc. Both getVAN() calls will have the same actual paramters as both the device_rp and the extra_spec and the instance_info is the same for both calls. How cyborg will know which device is the FOO and which device is the BAR? This will be especially important when the user want to detach the BAR device from the instance.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":487,"context_line":""},{"line_number":488,"context_line":"* getVAN(self, device_rp, extra_specs, instance_info)"},{"line_number":489,"context_line":""},{"line_number":490,"context_line":"  * device_rp: The nested RP selected by the scheduler is"},{"line_number":491,"context_line":"      needed for Cyborg to correlate it to a deployable. Read-only."},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"  * extra_specs: the extra_specs field in the flavor, including"},{"line_number":494,"context_line":"      fields interpreted by Cyborg but not Nova. Read-only."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_68db194d","line":491,"range":{"start_line":490,"start_character":15,"end_line":491,"end_character":55},"in_reply_to":"5f7c97a3_160d158e","updated":"2018-06-27 03:54:51.000000000","message":"Yes, Nova will make separate getVAN() calls for each accelerator RC requested. \n\nFor the FOO/BAR scenario, it is difficult to see how a single RP provides a CUSTOM_ACCELERATOR_GPU and a CUSTOM_ACCELERATOR_FPGA for example. The FOO and BAR are device types. Please LMK if we are not on the same page.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":490,"context_line":"  * device_rp: The nested RP selected by the scheduler is"},{"line_number":491,"context_line":"      needed for Cyborg to correlate it to a deployable. Read-only."},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"  * extra_specs: the extra_specs field in the flavor, including"},{"line_number":494,"context_line":"      fields interpreted by Cyborg but not Nova. Read-only."},{"line_number":495,"context_line":""},{"line_number":496,"context_line":"  * instance_info: Nova\u0027s per-instance dictionary. Read-only."},{"line_number":497,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_d6ac7d36","line":494,"range":{"start_line":493,"start_character":54,"end_line":494,"end_character":47},"updated":"2018-06-25 15:07:46.000000000","message":"The flavor extra_spec already could contains a list of hard to understand keys. I don\u0027t feel good about allowing another set of keys there that is only understood by Cyborg.\n\nIf Cyborg needs extra configuration for a certain accelerator instance then I suggest to introduce a Cyborg API resource that represents that accelerator instances (similarly how neutron has the port) and let the user set the extra configuration there instead of pushing it through that the nova flavor extra_spec (like the neutron allows setting the vnic_type in the port).","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":490,"context_line":"  * device_rp: The nested RP selected by the scheduler is"},{"line_number":491,"context_line":"      needed for Cyborg to correlate it to a deployable. Read-only."},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"  * extra_specs: the extra_specs field in the flavor, including"},{"line_number":494,"context_line":"      fields interpreted by Cyborg but not Nova. Read-only."},{"line_number":495,"context_line":""},{"line_number":496,"context_line":"  * instance_info: Nova\u0027s per-instance dictionary. Read-only."},{"line_number":497,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_683f9985","line":494,"range":{"start_line":493,"start_character":54,"end_line":494,"end_character":47},"in_reply_to":"5f7c97a3_d6ac7d36","updated":"2018-06-27 03:54:51.000000000","message":"While I understand the concern, there are only 3 such fields: bitstream_id, function_id and function_name. \n\nWhereas a port requires a new MAC/IP to be allocated for every instance, a function_id requirement (for example) does not require something to be allocated -- if that function is already present, nothing needs to be done. It is more like \u0027ensure this property\u0027 than \u0027make this change\u0027. \n\nAlso, unlike the port case, unless we know which accelerator/device is involved, we wouldn\u0027t know which device to program.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":497,"context_line":""},{"line_number":498,"context_line":"    * The VM image UUID may be used to query Glance for metadata about"},{"line_number":499,"context_line":"      accelerator requirements that may be stored with the VM image"},{"line_number":500,"context_line":"      (i.e., traits in images instead of flavors)."},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"    * The project ID may be used for policy-based decisions in the"},{"line_number":503,"context_line":"      future if more than one bitstream matches the need."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_7931daec","line":500,"updated":"2018-06-25 15:07:46.000000000","message":"Can we just create an accelerator API resource in Cyborg and have a field there describing the requirements of the accelerator. When the user creates an accelerator in Cyborg it can define what she needs and then pass the uuid of the created accelerator to the nova boot command (similarly neutron does not allow any port parameter overrides throguh flavor or image and still can be used to express instance specific connectivity configuration via the port API resource.)","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":497,"context_line":""},{"line_number":498,"context_line":"    * The VM image UUID may be used to query Glance for metadata about"},{"line_number":499,"context_line":"      accelerator requirements that may be stored with the VM image"},{"line_number":500,"context_line":"      (i.e., traits in images instead of flavors)."},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"    * The project ID may be used for policy-based decisions in the"},{"line_number":503,"context_line":"      future if more than one bitstream matches the need."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_08738581","line":500,"in_reply_to":"5f7c97a3_7931daec","updated":"2018-06-27 03:54:51.000000000","message":"With image traits, there is nothing in the flavor. So, could you explain why you advocate a change here?\n\nAlso, please see my reply to your previous comment.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"90f9e0e2c599f5a4e36f8cf239dd03a93fabc69d","unresolved":false,"context_lines":[{"line_number":649,"context_line":""},{"line_number":650,"context_line":"No changes are required in Glance to support Cyborg."},{"line_number":651,"context_line":""},{"line_number":652,"context_line":"In the future, when Glance images can have traits, the accelerator"},{"line_number":653,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":654,"context_line":"traits in the flavor."},{"line_number":655,"context_line":""},{"line_number":656,"context_line":"Alternatives"},{"line_number":657,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_b9977290","line":654,"range":{"start_line":652,"start_character":0,"end_line":654,"end_character":21},"updated":"2018-06-25 15:07:46.000000000","message":"If Cyborg would provide a accelerator API resource then these traits (or more like config parameters) can be expressed in that accelerator API resource without needing to touch glance  image traits or flavor extra_specs","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"6fdf3c1d7c57b54e1ee30af80ad043b99d653de7","unresolved":false,"context_lines":[{"line_number":649,"context_line":""},{"line_number":650,"context_line":"No changes are required in Glance to support Cyborg."},{"line_number":651,"context_line":""},{"line_number":652,"context_line":"In the future, when Glance images can have traits, the accelerator"},{"line_number":653,"context_line":"requirements for a VM image may be expressed as image traits instead of"},{"line_number":654,"context_line":"traits in the flavor."},{"line_number":655,"context_line":""},{"line_number":656,"context_line":"Alternatives"},{"line_number":657,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f7c97a3_c8682d89","line":654,"range":{"start_line":652,"start_character":0,"end_line":654,"end_character":21},"in_reply_to":"5f7c97a3_b9977290","updated":"2018-06-27 03:54:51.000000000","message":"Please see my previous response.","commit_id":"347e4b2014692945acdea9d0e99ed543122dd77f"}]}