)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"20c8645ec1fce38aeca7fb5bf22230f04acaaf69","unresolved":false,"context_lines":[{"line_number":9,"context_line":"Addresses issues of representing accelerators in Cyborg for"},{"line_number":10,"context_line":"alignment with Nova, and integrating Cyborg into Nova"},{"line_number":11,"context_line":"scheduler flow."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Updated to include post-PTG discussions, including CUSTOM_"},{"line_number":14,"context_line":"PROGRAMMABLE trait. Focuses on Rocky and single-function"},{"line_number":15,"context_line":"devices."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"5f7c97a3_1098c219","line":12,"updated":"2018-05-14 13:40:46.000000000","message":"Note: You don\u0027t need to document changes like those below. The commit message should just be an overview of the rest of the commit.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"1a44fa51f3ae439330babed6384cf9bd852bfa8e","unresolved":false,"context_lines":[{"line_number":9,"context_line":"Addresses issues of representing accelerators in Cyborg for"},{"line_number":10,"context_line":"alignment with Nova, and integrating Cyborg into Nova"},{"line_number":11,"context_line":"scheduler flow."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Updated to include post-PTG discussions, including CUSTOM_"},{"line_number":14,"context_line":"PROGRAMMABLE trait. Focuses on Rocky and single-function"},{"line_number":15,"context_line":"devices."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"5f7c97a3_0d6e2ac8","line":12,"in_reply_to":"5f7c97a3_1098c219","updated":"2018-05-16 17:32:37.000000000","message":"Done","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Updated to fix typos and comments from previous review."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Updated to address comments."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Updated to note that Cyborg weigher is optional, and to fix"},{"line_number":22,"context_line":"the keyword \u0027resources\u0027 in flavor syntax."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"5f7c97a3_deb9ebcb","line":19,"range":{"start_line":19,"start_character":0,"end_line":19,"end_character":28},"updated":"2018-05-31 07:22:57.000000000","message":"In my option, commit message is somewhere we give a brief conclusion to other reviewer, but no need to record the commit history. : )","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"}],"doc/specs/rocky/cyborg-nova-sched.rst":[{"author":{"_account_id":11878,"name":"Rushil Chugh","email":"rushil.chugh@gmail.com","username":"rushil"},"change_message_id":"d92ed095932960919ac6976c3b6acd7182531f29","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Cyborg is a service for managing accelerators, such as FPGAs, GPUs, etc. For"},{"line_number":14,"context_line":"scheduling an instance that needs accelerators, Cyborg needs to work with Nova"},{"line_number":15,"context_line":"at three levels:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators as"},{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates resource"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_7e1cd776","line":15,"range":{"start_line":15,"start_character":0,"end_line":15,"end_character":2},"updated":"2018-03-21 05:11:31.000000000","message":"on","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Cyborg is a service for managing accelerators, such as FPGAs, GPUs, etc. For"},{"line_number":14,"context_line":"scheduling an instance that needs accelerators, Cyborg needs to work with Nova"},{"line_number":15,"context_line":"at three levels:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators as"},{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates resource"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_2e6b0685","line":15,"range":{"start_line":15,"start_character":0,"end_line":15,"end_character":2},"in_reply_to":"df7087c5_7e1cd776","updated":"2018-04-13 18:41:58.000000000","message":"Both usages seem acceptable, IMHO.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":15,"context_line":"at three levels:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators as"},{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates resource"},{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_d76b411b","line":18,"range":{"start_line":18,"start_character":2,"end_line":18,"end_character":11},"updated":"2018-03-29 16:36:18.000000000","message":"resource *providers*","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":15,"context_line":"at three levels:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators as"},{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates resource"},{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_ee066ece","line":18,"range":{"start_line":18,"start_character":2,"end_line":18,"end_character":11},"in_reply_to":"bf659307_d76b411b","updated":"2018-04-13 18:41:58.000000000","message":"The model itself is revised in subsequent versions of this spec.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates resource"},{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_b7dc8de4","line":21,"range":{"start_line":21,"start_character":56,"end_line":21,"end_character":70},"updated":"2018-03-29 16:36:18.000000000","message":"not a custom filter. only a customer weigher is needed. The filtering should use the already-existing methodology of asking placement (via the flavor extra specs or image properties) for an amount of a particular resource class.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates resource"},{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_2e00e6aa","line":21,"range":{"start_line":21,"start_character":56,"end_line":21,"end_character":70},"in_reply_to":"bf659307_b7dc8de4","updated":"2018-04-13 18:41:58.000000000","message":"Yes, the revised spec uses only a custom weigher, that too only for one use case, working only off Placement inventory/traits (no Cyborg DB look-ups).","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_4e9ee563","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"updated":"2018-03-22 03:00:18.000000000","message":"accelerator resources\u0027 availability, locality, and topology, etc.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"8aae1aee9e70b7de2da07837fb767e38ae10d775","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9f6a8fd7_a2f8d148","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"in_reply_to":"9f6a8fd7_2f4a9d7f","updated":"2018-04-20 19:55:59.000000000","message":"If it is a transient state (i.e. an admin downed the device or put it in some maintenance mode), then Cyborg should not delete the resource provider. Instead, Cyborg should set the reserved count equal to the total count for its inventories. This will ensure no further workloads get placed on the device, but will ensure that existing allocations will be OK.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"283ceab0e0295c4d2633e75bd378478ac4657302","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9f6a8fd7_2f4a9d7f","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"in_reply_to":"9f6a8fd7_7e39733f","updated":"2018-04-19 17:11:59.000000000","message":"Hi Jay, AFAIK there is no other way to handle a bad device (GPU, QAT, whatever) than by taking it out of the inventory. If you are saying that Cyborg should not keep taking a device out and putting it back in based on repeated retries, I agree. Please LMK if there are other options.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"43b1c25ffd43892327cd78c77f151912b9ff9929","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9f6a8fd7_c4a3c89d","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"in_reply_to":"9f6a8fd7_7e39733f","updated":"2018-04-19 04:58:05.000000000","message":"Sounds like placement already has all of those tracks info.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"7d3fce52a1e8d8ce481c7b857c6b0d2b8b88a8a7","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_f3b72b20","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"in_reply_to":"bf659307_37f2bd71","updated":"2018-04-14 05:07:29.000000000","message":"Hi Jay, availability tracks the states of each accelerator to indicate whether they are being claimed/allocated\nlocality tracks where these accelerators are physically at(e.g. accelerator A is located at Host B).\nTopology tracks how the accelerators are linked/colocated to each other. For instance, FPGA C has PF D and PF E etc.\n\nHope that answers your question","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"7c186adc209e563fb1b7a8476791662296c06d85","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9f6a8fd7_7e39733f","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"in_reply_to":"bf659307_f3b72b20","updated":"2018-04-16 15:38:22.000000000","message":"Thanks, Liu, but no that doesn\u0027t quite answer my question. One thing I want to make sure is that Cyborg isn\u0027t going to be dynamically \"turning on and off\" resource providers in the placement service based on some temporal status value.\n\nIn other words, if an FPGA goes into some hardware maintenance state, Cyborg should not be removing that FPGA\u0027s inventory and/or allocations from placement just because it goes into an error state of some kind.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_37f2bd71","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"in_reply_to":"df7087c5_4e9ee563","updated":"2018-03-29 16:36:18.000000000","message":"Liu, would you mind elaborating on what you mean by the above please?","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":19,"context_line":"  inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg shall provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_3f95ab32","line":22,"range":{"start_line":22,"start_character":42,"end_line":22,"end_character":73},"in_reply_to":"df7087c5_4e9ee563","updated":"2018-04-13 18:41:58.000000000","message":"Sure. There could even be operator policies for deciding availability or priority.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":25,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"},{"line_number":26,"context_line":"  library (like os-vif and os-brick)."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"This spec addresses the first two aspects. There needs to be another spec to"},{"line_number":29,"context_line":"address the attachment of accelerators to instances. Cyborg also needs to"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_f7e765ab","line":26,"updated":"2018-03-29 16:36:18.000000000","message":"++","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":28,"context_line":"This spec addresses the first two aspects. There needs to be another spec to"},{"line_number":29,"context_line":"address the attachment of accelerators to instances. Cyborg also needs to"},{"line_number":30,"context_line":"handle some aspects for FPGAs without involving Nova, specifically FPGA"},{"line_number":31,"context_line":"programming and bitstream management. They will be covered in that spec too."},{"line_number":32,"context_line":"This spec is independent of the other spec."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"This spec is common to all accelerators. Since FPGAs have more aspects to be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_ceb195e3","line":31,"range":{"start_line":31,"start_character":61,"end_line":31,"end_character":71},"updated":"2018-03-22 03:00:18.000000000","message":"couple other specs too","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":28,"context_line":"This spec addresses the first two aspects. There needs to be another spec to"},{"line_number":29,"context_line":"address the attachment of accelerators to instances. Cyborg also needs to"},{"line_number":30,"context_line":"handle some aspects for FPGAs without involving Nova, specifically FPGA"},{"line_number":31,"context_line":"programming and bitstream management. They will be covered in that spec too."},{"line_number":32,"context_line":"This spec is independent of the other spec."},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"This spec is common to all accelerators. Since FPGAs have more aspects to be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_4ef67a5f","line":31,"range":{"start_line":31,"start_character":61,"end_line":31,"end_character":71},"in_reply_to":"df7087c5_ceb195e3","updated":"2018-04-13 18:41:58.000000000","message":"Done","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":11878,"name":"Rushil Chugh","email":"rushil.chugh@gmail.com","username":"rushil"},"change_message_id":"d92ed095932960919ac6976c3b6acd7182531f29","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Here is an example diagram for an FPGA with multiple regions, and multiple"},{"line_number":67,"context_line":"functions in a region::"},{"line_number":68,"context_line":"   "},{"line_number":69,"context_line":"         PCI A     PCI B"},{"line_number":70,"context_line":"          |        |"},{"line_number":71,"context_line":"  +-------|--------|-------------------+"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_c9168b4a","line":68,"range":{"start_line":68,"start_character":0,"end_line":68,"end_character":3},"updated":"2018-03-21 05:11:31.000000000","message":"remove whitespace","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Here is an example diagram for an FPGA with multiple regions, and multiple"},{"line_number":67,"context_line":"functions in a region::"},{"line_number":68,"context_line":"   "},{"line_number":69,"context_line":"         PCI A     PCI B"},{"line_number":70,"context_line":"          |        |"},{"line_number":71,"context_line":"  +-------|--------|-------------------+"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_0e9ac212","line":68,"range":{"start_line":68,"start_character":0,"end_line":68,"end_character":3},"in_reply_to":"df7087c5_c9168b4a","updated":"2018-04-13 18:41:58.000000000","message":"Done","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":77,"context_line":"  |  +-----------------+   +--------+  |"},{"line_number":78,"context_line":"  |  Region 1              Region 2    |"},{"line_number":79,"context_line":"  |                                    |"},{"line_number":80,"context_line":"  +------------------------------------+"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Problem description"},{"line_number":83,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_777ef5af","line":80,"updated":"2018-03-29 16:36:18.000000000","message":"So what of the above things are accelerators? Or are they *all* accelerators?","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":77,"context_line":"  |  +-----------------+   +--------+  |"},{"line_number":78,"context_line":"  |  Region 1              Region 2    |"},{"line_number":79,"context_line":"  |                                    |"},{"line_number":80,"context_line":"  +------------------------------------+"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Problem description"},{"line_number":83,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_8e9fb24e","line":80,"in_reply_to":"bf659307_777ef5af","updated":"2018-04-13 18:41:58.000000000","message":"In the revised spec, an accelerator is either a function (Fn A/B) or a region. This example is a bit complex, but consider a simpler one with only Fn A in Region 1. Then, Cyborg will advertise Region 1 as an RP, with one unit of CUSTOM_ACCELERATOR RC, and with 2 traits: CUSTOM_FPGA_REGION_\u003cuuid\u003e and CUSTOM_FPGA_FUNCTION_\u003cfnA-uuid\u003e. A flavor could ask for the region with:\n    resource:CUSTOM_ACCELERATOR\u003d1\n    trait:CUSTOM_FPGA_REGION_\u003cuuid\u003e\u003drequired\nOR it could ask for a function with:\n    resource:CUSTOM_ACCELERATOR\u003d1\n    trait:CUSTOM_FPGA_FUNCTION_\u003cfnAuuid\u003e\u003drequired\n    (other traits explained in revised spec)\n\nWhen we have multiple function types in the same region, as in this diagram, I hope we can create nested RPs within Region 1 RP, and apply the RCs and traits there. But that is another confusion for another day. :)","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":98,"context_line":"---------"},{"line_number":99,"context_line":"We need to satisfy the following use cases for the tenant role:"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"* Device as a Service (DaaS): The flavor asks for a device."},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"  * FPGA variation: The flavor asks for a device to which specific"},{"line_number":104,"context_line":"    bitstream(s) can be applied. There are two variations for secure"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_7a481c5b","line":101,"range":{"start_line":101,"start_character":2,"end_line":101,"end_character":28},"updated":"2018-03-29 16:36:18.000000000","message":"I would just call this \"passthrough\", not device as a service. What you are indicating here is that the full device will be provided as-is to the guest for it to do with as it pleases.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":98,"context_line":"---------"},{"line_number":99,"context_line":"We need to satisfy the following use cases for the tenant role:"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"* Device as a Service (DaaS): The flavor asks for a device."},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"  * FPGA variation: The flavor asks for a device to which specific"},{"line_number":104,"context_line":"    bitstream(s) can be applied. There are two variations for secure"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_4e419aa5","line":101,"range":{"start_line":101,"start_character":2,"end_line":101,"end_character":28},"in_reply_to":"bf659307_7a481c5b","updated":"2018-04-13 18:41:58.000000000","message":"Short answer: yes. To elaborate, for a GPU, you would pass the entire device (or the render node). For an FPGA, a region may be passed to the instance (via a VF for example).\n\nIt would indeed be passthrough if we are sticking to SR-IOV. If we get mediated devices or something else in the future, I am not sure that is called a \u0027passthrough\u0027.  \n\nThe term \u0027FPGA as a Service\u0027 is getting bandied about, and I used an analogous term to make it non-FPGA-centric.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":111,"context_line":"      (Cyborg receives the request and does the programming.)"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"* Accelerated Function as a Service (AFaaS): The flavor asks for a"},{"line_number":114,"context_line":"  function (e.g. ipsec) attached with my instance."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"An operator must be able to provide both Device as a Service and Accelerated"},{"line_number":117,"context_line":"Function as a Service in the same cluster, to serve all"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_ff2765b5","line":114,"range":{"start_line":114,"start_character":23,"end_line":114,"end_character":24},"updated":"2018-03-22 03:00:18.000000000","message":"and/or with minimum kpi/capability requirements","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":111,"context_line":"      (Cyborg receives the request and does the programming.)"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"* Accelerated Function as a Service (AFaaS): The flavor asks for a"},{"line_number":114,"context_line":"  function (e.g. ipsec) attached with my instance."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"An operator must be able to provide both Device as a Service and Accelerated"},{"line_number":117,"context_line":"Function as a Service in the same cluster, to serve all"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_1f2b3982","line":114,"range":{"start_line":114,"start_character":33,"end_line":114,"end_character":37},"updated":"2018-03-22 03:00:18.000000000","message":"to","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":111,"context_line":"      (Cyborg receives the request and does the programming.)"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"* Accelerated Function as a Service (AFaaS): The flavor asks for a"},{"line_number":114,"context_line":"  function (e.g. ipsec) attached with my instance."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"An operator must be able to provide both Device as a Service and Accelerated"},{"line_number":117,"context_line":"Function as a Service in the same cluster, to serve all"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_4e6f5a27","line":114,"range":{"start_line":114,"start_character":23,"end_line":114,"end_character":24},"in_reply_to":"df7087c5_ff2765b5","updated":"2018-04-13 18:41:58.000000000","message":"\u0027to\u0027: Will fix\n\u0027Minimum KPI\u0027: Yes. The function can be characterized in many ways.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":113,"context_line":"* Accelerated Function as a Service (AFaaS): The flavor asks for a"},{"line_number":114,"context_line":"  function (e.g. ipsec) attached with my instance."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"An operator must be able to provide both Device as a Service and Accelerated"},{"line_number":117,"context_line":"Function as a Service in the same cluster, to serve all"},{"line_number":118,"context_line":"kinds of users: those who are device-agnostic, those using 3rd party"},{"line_number":119,"context_line":"bitstreams, and those using their own bitstreams (incl. developers)."}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_bf56cd05","line":116,"range":{"start_line":116,"start_character":12,"end_line":116,"end_character":16},"updated":"2018-03-22 03:00:18.000000000","message":"Do they have to be in the same cluster? I think this should just be a choice upon use cases.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":113,"context_line":"* Accelerated Function as a Service (AFaaS): The flavor asks for a"},{"line_number":114,"context_line":"  function (e.g. ipsec) attached with my instance."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"An operator must be able to provide both Device as a Service and Accelerated"},{"line_number":117,"context_line":"Function as a Service in the same cluster, to serve all"},{"line_number":118,"context_line":"kinds of users: those who are device-agnostic, those using 3rd party"},{"line_number":119,"context_line":"bitstreams, and those using their own bitstreams (incl. developers)."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_6edc9ee3","line":116,"range":{"start_line":116,"start_character":12,"end_line":116,"end_character":16},"in_reply_to":"df7087c5_bf56cd05","updated":"2018-04-13 18:41:58.000000000","message":"It is up to the operator. The point here is that Cyborg should not force operators to segregate devices or split the cluster based on use cases.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":128,"context_line":"    * Operational simplicity."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"    * Assure tenants of programming security, by doing all programming offline"},{"line_number":131,"context_line":"    * through some audited process."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"  * For FPGAs, allow orchestration to program as needed, to maximize"},{"line_number":134,"context_line":"    flexibility and availability of resources."}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_9f3bc9c6","line":131,"range":{"start_line":131,"start_character":4,"end_line":131,"end_character":5},"updated":"2018-03-22 03:00:18.000000000","message":"remove this","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"488c0537a010d0f61a7d52f502110707b5972895","unresolved":false,"context_lines":[{"line_number":128,"context_line":"    * Operational simplicity."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"    * Assure tenants of programming security, by doing all programming offline"},{"line_number":131,"context_line":"    * through some audited process."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"  * For FPGAs, allow orchestration to program as needed, to maximize"},{"line_number":134,"context_line":"    flexibility and availability of resources."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_2cd1cf23","line":131,"range":{"start_line":131,"start_character":4,"end_line":131,"end_character":5},"in_reply_to":"df7087c5_9f3bc9c6","updated":"2018-04-13 18:46:44.000000000","message":"Done","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":133,"context_line":"  * For FPGAs, allow orchestration to program as needed, to maximize"},{"line_number":134,"context_line":"    flexibility and availability of resources."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"It is useful to consider the cross-product of above combinations as 4 discrete"},{"line_number":137,"context_line":"use cases:"},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"* Flavor requests AFaaS, Operator offers Pre-programmed model."},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_daada8e9","line":137,"range":{"start_line":136,"start_character":68,"end_line":137,"end_character":10},"updated":"2018-03-29 16:36:18.000000000","message":"personally, I don\u0027t think the \"orchestration-driven programming\" is something Nova should support. Have some external agent that clears regions and re-flashes them with a \"hot\" function if system/operator notices a bunch of regions or full FPGAs that are no longer used and have been flashed with a particular function already.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":133,"context_line":"  * For FPGAs, allow orchestration to program as needed, to maximize"},{"line_number":134,"context_line":"    flexibility and availability of resources."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"It is useful to consider the cross-product of above combinations as 4 discrete"},{"line_number":137,"context_line":"use cases:"},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"* Flavor requests AFaaS, Operator offers Pre-programmed model."},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_ee642e24","line":137,"range":{"start_line":136,"start_character":68,"end_line":137,"end_character":10},"in_reply_to":"bf659307_daada8e9","updated":"2018-04-13 18:41:58.000000000","message":"Nova need not support any form of programming by itself, or anything special for this use case. With the appropriate use of traits and other extra specs, we can probably get Nova to pick devices and Cyborg to do the programming, as explained in the revised spec. Hope you won\u0027t have reservations about that. :)","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":146,"context_line":"* Flavor requests DaaS, and the instance will request specific bitstreams."},{"line_number":147,"context_line":"  (Cyborg infrastructure receives the request and programs the bitstreams.)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Note that the operator decides both the Service model (AFaaS or DaaS) and"},{"line_number":150,"context_line":"programming model (allowed or not). The goal for Cyborg is to provide the"},{"line_number":151,"context_line":"mechanisms to enable all these use cases."},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_ff752599","line":149,"range":{"start_line":149,"start_character":40,"end_line":149,"end_character":47},"updated":"2018-03-22 03:00:18.000000000","message":"service","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"488c0537a010d0f61a7d52f502110707b5972895","unresolved":false,"context_lines":[{"line_number":146,"context_line":"* Flavor requests DaaS, and the instance will request specific bitstreams."},{"line_number":147,"context_line":"  (Cyborg infrastructure receives the request and programs the bitstreams.)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Note that the operator decides both the Service model (AFaaS or DaaS) and"},{"line_number":150,"context_line":"programming model (allowed or not). The goal for Cyborg is to provide the"},{"line_number":151,"context_line":"mechanisms to enable all these use cases."},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_ecc67768","line":149,"range":{"start_line":149,"start_character":40,"end_line":149,"end_character":47},"in_reply_to":"df7087c5_ff752599","updated":"2018-04-13 18:46:44.000000000","message":"Done","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":187,"context_line":""},{"line_number":188,"context_line":"  * A flavor always asks for the RC of the specific device/region. e.g."},{"line_number":189,"context_line":"    CUSTOM_FPGA_INTEL_COMPUTE_ARRIA10. (Today, flavors are based on PCI white"},{"line_number":190,"context_line":"    lists.)"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"  * In addition, optionally, the flavor extra specs may specify a bitstream ID"},{"line_number":193,"context_line":"    (for DaaS static use case)."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_badeb46a","line":190,"updated":"2018-03-29 16:36:18.000000000","message":"Why not have a single resource class called ACCELERATED_FUNCTION that represents (generically) a resource context provided to a VM to a particular function that has been programmed on an FPGA?","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":187,"context_line":""},{"line_number":188,"context_line":"  * A flavor always asks for the RC of the specific device/region. e.g."},{"line_number":189,"context_line":"    CUSTOM_FPGA_INTEL_COMPUTE_ARRIA10. (Today, flavors are based on PCI white"},{"line_number":190,"context_line":"    lists.)"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"  * In addition, optionally, the flavor extra specs may specify a bitstream ID"},{"line_number":193,"context_line":"    (for DaaS static use case)."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_ce460a7c","line":190,"in_reply_to":"bf659307_badeb46a","updated":"2018-04-13 18:41:58.000000000","message":"Yes, the revised proposal does it that way.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"5d93d7ea0232ce74fdd82d3242c2c3e54dc29756","unresolved":false,"context_lines":[{"line_number":193,"context_line":"    (for DaaS static use case)."},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"  * In addition, optionally, for requesting functions instead of devices, the"},{"line_number":196,"context_line":"    flavor extra specs will specify the function UUID."},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"  * A flavor may ask for other RCs, such as local memory."},{"line_number":199,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_9abad007","line":196,"updated":"2018-03-29 16:36:18.000000000","message":"Why should the flavor specify the function UUID? Why wouldn\u0027t the image properties specify the accelerated function that the guest image is built to utilize?","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":193,"context_line":"    (for DaaS static use case)."},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"  * In addition, optionally, for requesting functions instead of devices, the"},{"line_number":196,"context_line":"    flavor extra specs will specify the function UUID."},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"  * A flavor may ask for other RCs, such as local memory."},{"line_number":199,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_6e0fbe36","line":196,"in_reply_to":"bf659307_9abad007","updated":"2018-04-13 18:41:58.000000000","message":"It can be done that way too. In the revised spec, the function UUID is a trait. By including it in the flavor, it becomes easier for the scheduler to filter on that, or for a weigher to rank RPs with that trait higher. If it were in the image metadata, wouldn\u0027t one have to query Glance as part of the scheduling flow?","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":14131,"name":"shaohef","email":"shaohe.feng@intel.com","username":"shaohefeng"},"change_message_id":"ffb71e013699cf95fb88043ddaf240cc650d97bb","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  * Placement API returns the list of RPs (compute nodes + device/region) which"},{"line_number":213,"context_line":"    contain the requested FPGA device types."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"  * Cyborg will provide a custom filter which queries Cyborg DB. This will"},{"line_number":216,"context_line":"    eliminate all candidate nodes and fail the request if:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"    * If the use case is AFaaS + Preprogrammed, and the requested function"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_841cfa77","line":215,"range":{"start_line":215,"start_character":4,"end_line":215,"end_character":39},"updated":"2018-03-21 07:38:48.000000000","message":"Who will call this custom filter? And how to call it, cyborg API?\n\nPlease elaborate it.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  * Placement API returns the list of RPs (compute nodes + device/region) which"},{"line_number":213,"context_line":"    contain the requested FPGA device types."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"  * Cyborg will provide a custom filter which queries Cyborg DB. This will"},{"line_number":216,"context_line":"    eliminate all candidate nodes and fail the request if:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"    * If the use case is AFaaS + Preprogrammed, and the requested function"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_cc513310","line":215,"range":{"start_line":215,"start_character":4,"end_line":215,"end_character":39},"in_reply_to":"bf659307_919c0e0f","updated":"2018-04-13 18:41:58.000000000","message":"There are no filters in the revised proposal.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"8ad19eaacc00af7b09dabb55672e7fa00d59bc79","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  * Placement API returns the list of RPs (compute nodes + device/region) which"},{"line_number":213,"context_line":"    contain the requested FPGA device types."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"  * Cyborg will provide a custom filter which queries Cyborg DB. This will"},{"line_number":216,"context_line":"    eliminate all candidate nodes and fail the request if:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"    * If the use case is AFaaS + Preprogrammed, and the requested function"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_919c0e0f","line":215,"range":{"start_line":215,"start_character":4,"end_line":215,"end_character":39},"in_reply_to":"df7087c5_841cfa77","updated":"2018-03-28 20:51:50.000000000","message":"If you need an additional filter besides placement, my feeling is that you aren\u0027t modeling your resources correctly. If you have configured your deployment with AFaaS + Preprogrammed, then your inventory should be the functions. If you are concerned about bitstream licenses, then your inventory should be the licenses.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":14131,"name":"shaohef","email":"shaohe.feng@intel.com","username":"shaohefeng"},"change_message_id":"ffb71e013699cf95fb88043ddaf240cc650d97bb","unresolved":false,"context_lines":[{"line_number":221,"context_line":"    * If the use case is DaaS, and the flavor-requested bitstream, if any,"},{"line_number":222,"context_line":"      is not available. (E.g. the bitstream is out of licenses.)"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"  * FPGA-specific: For AFaaS, Cyborg will provide a custom weigher which"},{"line_number":225,"context_line":"    queries Cyborg DB to prioritize nodes where the requested function (if"},{"line_number":226,"context_line":"    any) is already available. This enables the orchestrator to choose the"},{"line_number":227,"context_line":"    function if it already exists or else set the stage for reprogramming"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_587d666c","line":224,"range":{"start_line":224,"start_character":52,"end_line":224,"end_character":66},"updated":"2018-03-21 07:38:48.000000000","message":"How to present this weigher, A new API?\n\nNot sure this is a good idea to provide a custom weigher in Cyborg. \n\nFor there maybe different strategy for weighing, depends on the cloud operators.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":221,"context_line":"    * If the use case is DaaS, and the flavor-requested bitstream, if any,"},{"line_number":222,"context_line":"      is not available. (E.g. the bitstream is out of licenses.)"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"  * FPGA-specific: For AFaaS, Cyborg will provide a custom weigher which"},{"line_number":225,"context_line":"    queries Cyborg DB to prioritize nodes where the requested function (if"},{"line_number":226,"context_line":"    any) is already available. This enables the orchestrator to choose the"},{"line_number":227,"context_line":"    function if it already exists or else set the stage for reprogramming"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_0c43ab40","line":224,"range":{"start_line":224,"start_character":52,"end_line":224,"end_character":66},"in_reply_to":"bf659307_b1d352d5","updated":"2018-04-13 18:41:58.000000000","message":"@Shaohe: This is the standard weigher mechanism in Nova. No new APIs. \n\n@ edleafe: Yes, you are right.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"8ad19eaacc00af7b09dabb55672e7fa00d59bc79","unresolved":false,"context_lines":[{"line_number":221,"context_line":"    * If the use case is DaaS, and the flavor-requested bitstream, if any,"},{"line_number":222,"context_line":"      is not available. (E.g. the bitstream is out of licenses.)"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"  * FPGA-specific: For AFaaS, Cyborg will provide a custom weigher which"},{"line_number":225,"context_line":"    queries Cyborg DB to prioritize nodes where the requested function (if"},{"line_number":226,"context_line":"    any) is already available. This enables the orchestrator to choose the"},{"line_number":227,"context_line":"    function if it already exists or else set the stage for reprogramming"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_b1d352d5","line":224,"range":{"start_line":224,"start_character":52,"end_line":224,"end_character":66},"in_reply_to":"df7087c5_587d666c","updated":"2018-03-28 20:51:50.000000000","message":"I think that there really is only one use case for weighing, and that\u0027s whether the requested function is on the FPGA or not.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":14091,"name":"Yih Leong Sun","email":"yihleong@gmail.com","username":"yihleongsun"},"change_message_id":"3dca446ab33ddd11ee522c8c7909e1045978c75e","unresolved":false,"context_lines":[{"line_number":252,"context_line":"    to an instance (such as a PCI function). NOTE: This entire workflow is"},{"line_number":253,"context_line":"    PCI-agnostic; we can use PCI or something else to attach instances to"},{"line_number":254,"context_line":"    devices."},{"line_number":255,"context_line":""},{"line_number":256,"context_line":"Representation"},{"line_number":257,"context_line":"--------------"},{"line_number":258,"context_line":"* Cyborg needs to decide which devices fall in the same RC."}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_f4ae7ae5","line":255,"updated":"2018-03-23 14:03:22.000000000","message":"what about Termination workflow after VM/task is completed and acc device is not needed? What\u0027s the option for cleaning/non-cleaning the acc device for the case of FPGA?","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":252,"context_line":"    to an instance (such as a PCI function). NOTE: This entire workflow is"},{"line_number":253,"context_line":"    PCI-agnostic; we can use PCI or something else to attach instances to"},{"line_number":254,"context_line":"    devices."},{"line_number":255,"context_line":""},{"line_number":256,"context_line":"Representation"},{"line_number":257,"context_line":"--------------"},{"line_number":258,"context_line":"* Cyborg needs to decide which devices fall in the same RC."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_2cc78f98","line":255,"in_reply_to":"df7087c5_f4ae7ae5","updated":"2018-04-13 18:41:58.000000000","message":"Since this spec is mostly focused on scheduling, we didn;t get into that. However, when an instance terminates, Nova compute will inform Cyborg via os-acc, which will help Cyborg track #VFs etc. \nCleaning up the accelerator/device after instance termination is partly FPGA vendor-dependent and partly operator-dependent. The vendor driver stack may do some automatic cleanups. In more security-conscious environments, one could do more radical things like reprogramming with a null bitstream, but that will prevent AFaaS. IMHO, Cyborg should not enforce a policy, but leave it to vendors/operators.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"8ed3b8ac63770f6ac6a0a539ad153fceed73f971","unresolved":false,"context_lines":[{"line_number":266,"context_line":"    images have the drivers to handle any device of that RC."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"  * NOTE: This implies that either Cyborg pre-decides what constitutes a RC, or"},{"line_number":269,"context_line":"    provides knobs for operators to configure it."},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"* Cyborg currently ties PCI functions to RCs. Instead, it should identify the"},{"line_number":272,"context_line":"  RCs from the driver, and associate attach handles (such as PCI functions,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"df7087c5_3ab88b28","line":269,"updated":"2018-03-22 03:00:18.000000000","message":"The thing I am worried is this mapping (device \u003c--\u003e RC) might be vendor specific, which means Cloud Operator might not even have enough knowledge to configure it. Suggestion: Leave the mapping to the vendor driver during resource reporting.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":266,"context_line":"    images have the drivers to handle any device of that RC."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"  * NOTE: This implies that either Cyborg pre-decides what constitutes a RC, or"},{"line_number":269,"context_line":"    provides knobs for operators to configure it."},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"* Cyborg currently ties PCI functions to RCs. Instead, it should identify the"},{"line_number":272,"context_line":"  RCs from the driver, and associate attach handles (such as PCI functions,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_2c4c6f40","line":269,"in_reply_to":"bf659307_51619604","updated":"2018-04-13 18:41:58.000000000","message":"@Li_Liu: Yes, exactly. The driver should be able to declare the topology to Cyborg agent.\n\n@edleafe: OK","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"8ad19eaacc00af7b09dabb55672e7fa00d59bc79","unresolved":false,"context_lines":[{"line_number":266,"context_line":"    images have the drivers to handle any device of that RC."},{"line_number":267,"context_line":""},{"line_number":268,"context_line":"  * NOTE: This implies that either Cyborg pre-decides what constitutes a RC, or"},{"line_number":269,"context_line":"    provides knobs for operators to configure it."},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"* Cyborg currently ties PCI functions to RCs. Instead, it should identify the"},{"line_number":272,"context_line":"  RCs from the driver, and associate attach handles (such as PCI functions,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_51619604","line":269,"in_reply_to":"df7087c5_3ab88b28","updated":"2018-03-28 20:51:50.000000000","message":"This is similar to the issues surrounding NUMA configuration. Libvirt reports the full NUMA hierarchy if it is configured to do so, or, if the operator wishes, treats the VCPU, RAM, etc., and all coming from the compute node instead of separate NUMA nodes.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"8ad19eaacc00af7b09dabb55672e7fa00d59bc79","unresolved":false,"context_lines":[{"line_number":279,"context_line":""},{"line_number":280,"context_line":"* Cyborg should associate RPs/RCs with Deployables in its internal DB."},{"line_number":281,"context_line":""},{"line_number":282,"context_line":"Here is the mapping from FPGA components to Nova Placement components::"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"  FPGA +--------+------+ Resource Provider"},{"line_number":285,"context_line":"                |"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_71237abe","line":282,"range":{"start_line":282,"start_character":0,"end_line":282,"end_character":19},"updated":"2018-03-28 20:51:50.000000000","message":"I don\u0027t think it will be this simple. As noted above, if you are mapping pre-programmed functions, the function will be the resource class","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":279,"context_line":""},{"line_number":280,"context_line":"* Cyborg should associate RPs/RCs with Deployables in its internal DB."},{"line_number":281,"context_line":""},{"line_number":282,"context_line":"Here is the mapping from FPGA components to Nova Placement components::"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"  FPGA +--------+------+ Resource Provider"},{"line_number":285,"context_line":"                |"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_ac615fab","line":282,"range":{"start_line":282,"start_character":0,"end_line":282,"end_character":19},"in_reply_to":"bf659307_71237abe","updated":"2018-04-13 18:41:58.000000000","message":"This is addressed in the revised proposal.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"8ad19eaacc00af7b09dabb55672e7fa00d59bc79","unresolved":false,"context_lines":[{"line_number":360,"context_line":"  than go through Cyborg REST API. This will be not unlike other weighers."},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"For AFaaS, Placement returns all compute node RPs which contain the requested"},{"line_number":363,"context_line":"device type (but not necessarily the function). That can be a large list. To"},{"line_number":364,"context_line":"prune it as much as possible, here is a possible optimization for FPGAs."},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* Every device/region RP must be annotated with traits that indicate what"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_710cda40","line":363,"range":{"start_line":363,"start_character":48,"end_line":363,"end_character":73},"updated":"2018-03-28 20:51:50.000000000","message":"Not if you model your resources correctly. The resource class in this case should be the specific function, not the region.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"e52b3a3467f943f2da42d3e88e296a08832aecd5","unresolved":false,"context_lines":[{"line_number":360,"context_line":"  than go through Cyborg REST API. This will be not unlike other weighers."},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"For AFaaS, Placement returns all compute node RPs which contain the requested"},{"line_number":363,"context_line":"device type (but not necessarily the function). That can be a large list. To"},{"line_number":364,"context_line":"prune it as much as possible, here is a possible optimization for FPGAs."},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* Every device/region RP must be annotated with traits that indicate what"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf659307_cc64d3bb","line":363,"range":{"start_line":363,"start_character":48,"end_line":363,"end_character":73},"in_reply_to":"bf659307_710cda40","updated":"2018-04-13 18:41:58.000000000","message":"This is addressed in the revised proposal.","commit_id":"3737ee16df6440cd05f16050f8243436776eedc5"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9f9c81fc7b0770e51e864d4cb8af93eb7b6d0d19","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Function: A specific functionality, such as matrix multiplication or video"},{"line_number":58,"context_line":"  transcoding, usually represented as a string or UUID. This term may be used"},{"line_number":59,"context_line":"  with multi-function devices, incl. FPGAs and other fixed function hardware"},{"line_number":60,"context_line":"  like Intel QuickAssist."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_4e090adb","line":59,"range":{"start_line":59,"start_character":31,"end_line":59,"end_character":35},"updated":"2018-04-19 13:31:51.000000000","message":"nit: abbreviations like this should be avoided in specs.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"10e30c2bfacbfa8771af6e391d84f905c3e26280","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Function: A specific functionality, such as matrix multiplication or video"},{"line_number":58,"context_line":"  transcoding, usually represented as a string or UUID. This term may be used"},{"line_number":59,"context_line":"  with multi-function devices, incl. FPGAs and other fixed function hardware"},{"line_number":60,"context_line":"  like Intel QuickAssist."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_e5dd8f14","line":59,"range":{"start_line":59,"start_character":31,"end_line":59,"end_character":35},"in_reply_to":"9f6a8fd7_4e090adb","updated":"2018-04-20 03:16:08.000000000","message":"Will fix","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"341583924833693055f4f6bec122e9ec60ce624e","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Function: A specific functionality, such as matrix multiplication or video"},{"line_number":58,"context_line":"  transcoding, usually represented as a string or UUID. This term may be used"},{"line_number":59,"context_line":"  with multi-function devices, incl. FPGAs and other fixed function hardware"},{"line_number":60,"context_line":"  like Intel QuickAssist."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_38a8618a","line":59,"range":{"start_line":59,"start_character":31,"end_line":59,"end_character":35},"in_reply_to":"9f6a8fd7_e5dd8f14","updated":"2018-04-23 00:56:22.000000000","message":"Done","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9f9c81fc7b0770e51e864d4cb8af93eb7b6d0d19","unresolved":false,"context_lines":[{"line_number":59,"context_line":"  with multi-function devices, incl. FPGAs and other fixed function hardware"},{"line_number":60,"context_line":"  like Intel QuickAssist."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"},{"line_number":63,"context_line":"  the rest of the fabric. If an FPGA does not support Partial Reconfiguration,"},{"line_number":64,"context_line":"  the entire device constitutes one region. A region may implement one or more"},{"line_number":65,"context_line":"  functions. NOTE: In this document, we use the term \u0027region\u0027 instead of"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_aef9c6c8","line":62,"range":{"start_line":62,"start_character":29,"end_line":62,"end_character":35},"updated":"2018-04-19 13:31:51.000000000","message":"You should probably define what a \u0027fabric\u0027 is, too.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"341583924833693055f4f6bec122e9ec60ce624e","unresolved":false,"context_lines":[{"line_number":59,"context_line":"  with multi-function devices, incl. FPGAs and other fixed function hardware"},{"line_number":60,"context_line":"  like Intel QuickAssist."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"},{"line_number":63,"context_line":"  the rest of the fabric. If an FPGA does not support Partial Reconfiguration,"},{"line_number":64,"context_line":"  the entire device constitutes one region. A region may implement one or more"},{"line_number":65,"context_line":"  functions. NOTE: In this document, we use the term \u0027region\u0027 instead of"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_58ad1599","line":62,"range":{"start_line":62,"start_character":29,"end_line":62,"end_character":35},"in_reply_to":"9f6a8fd7_05e1a3d0","updated":"2018-04-23 00:56:22.000000000","message":"Done","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"10e30c2bfacbfa8771af6e391d84f905c3e26280","unresolved":false,"context_lines":[{"line_number":59,"context_line":"  with multi-function devices, incl. FPGAs and other fixed function hardware"},{"line_number":60,"context_line":"  like Intel QuickAssist."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"},{"line_number":63,"context_line":"  the rest of the fabric. If an FPGA does not support Partial Reconfiguration,"},{"line_number":64,"context_line":"  the entire device constitutes one region. A region may implement one or more"},{"line_number":65,"context_line":"  functions. NOTE: In this document, we use the term \u0027region\u0027 instead of"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_05e1a3d0","line":62,"range":{"start_line":62,"start_character":29,"end_line":62,"end_character":35},"in_reply_to":"9f6a8fd7_aef9c6c8","updated":"2018-04-20 03:16:08.000000000","message":"Will fix.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9f9c81fc7b0770e51e864d4cb8af93eb7b6d0d19","unresolved":false,"context_lines":[{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"},{"line_number":63,"context_line":"  the rest of the fabric. If an FPGA does not support Partial Reconfiguration,"},{"line_number":64,"context_line":"  the entire device constitutes one region. A region may implement one or more"},{"line_number":65,"context_line":"  functions. NOTE: In this document, we use the term \u0027region\u0027 instead of"},{"line_number":66,"context_line":"  device/region when referring to FPGAs."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"Here is an example diagram for an FPGA with multiple regions, and multiple"},{"line_number":69,"context_line":"functions in a region::"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_aed0e620","line":66,"range":{"start_line":65,"start_character":13,"end_line":66,"end_character":39},"updated":"2018-04-19 13:31:51.000000000","message":"You say this here, but then refer to \u0027device/region\u0027 on lines 174, 266, 276, and 314. Either remove this note, or update those other lines.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"341583924833693055f4f6bec122e9ec60ce624e","unresolved":false,"context_lines":[{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"},{"line_number":63,"context_line":"  the rest of the fabric. If an FPGA does not support Partial Reconfiguration,"},{"line_number":64,"context_line":"  the entire device constitutes one region. A region may implement one or more"},{"line_number":65,"context_line":"  functions. NOTE: In this document, we use the term \u0027region\u0027 instead of"},{"line_number":66,"context_line":"  device/region when referring to FPGAs."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"Here is an example diagram for an FPGA with multiple regions, and multiple"},{"line_number":69,"context_line":"functions in a region::"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_18931d5d","line":66,"range":{"start_line":65,"start_character":13,"end_line":66,"end_character":39},"in_reply_to":"9f6a8fd7_a5e797e4","updated":"2018-04-23 00:56:22.000000000","message":"Done","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"10e30c2bfacbfa8771af6e391d84f905c3e26280","unresolved":false,"context_lines":[{"line_number":62,"context_line":"* Region: A part of the FPGA fabric which can be programmed without disrupting"},{"line_number":63,"context_line":"  the rest of the fabric. If an FPGA does not support Partial Reconfiguration,"},{"line_number":64,"context_line":"  the entire device constitutes one region. A region may implement one or more"},{"line_number":65,"context_line":"  functions. NOTE: In this document, we use the term \u0027region\u0027 instead of"},{"line_number":66,"context_line":"  device/region when referring to FPGAs."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"Here is an example diagram for an FPGA with multiple regions, and multiple"},{"line_number":69,"context_line":"functions in a region::"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_a5e797e4","line":66,"range":{"start_line":65,"start_character":13,"end_line":66,"end_character":39},"in_reply_to":"9f6a8fd7_aed0e620","updated":"2018-04-20 03:16:08.000000000","message":"Will review and address. Thanks.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":7900,"name":"Tomasz Paszkowski","email":"ss7pro@gmail.com","username":"tpaszkowski"},"change_message_id":"2110405f1efa8fb929a600a991eae76d61d3bd6f","unresolved":false,"context_lines":[{"line_number":180,"context_line":"      support, Cyborg will configure aforementioned RCs and traits on"},{"line_number":181,"context_line":"      the compute node RP in which the device resides."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"  * Cyborg will associate a device type trait with each device. For"},{"line_number":184,"context_line":"    example, CUSTOM_GPU_\u003cvendor\u003e_\u003cproduct\u003e, CUSTOM_HPTS_ZTE_\u003cmodel\u003e or"},{"line_number":185,"context_line":"    CUSTOM_FPGA_INTEL_PAC_ARRIA10. This trait is intended to help match"},{"line_number":186,"context_line":"    the software drivers/libraries in the instance image. (Cyborg does"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_02b41df1","line":183,"updated":"2018-04-20 19:51:24.000000000","message":"What about situation if we would have different type of devices in single machine? How we\u0027re going to track if this specific device (ie: ARIA10) was not already consumed?","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"e988b07e387c8f24e3b64f34606fbf41de12d1d6","unresolved":false,"context_lines":[{"line_number":180,"context_line":"      support, Cyborg will configure aforementioned RCs and traits on"},{"line_number":181,"context_line":"      the compute node RP in which the device resides."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"  * Cyborg will associate a device type trait with each device. For"},{"line_number":184,"context_line":"    example, CUSTOM_GPU_\u003cvendor\u003e_\u003cproduct\u003e, CUSTOM_HPTS_ZTE_\u003cmodel\u003e or"},{"line_number":185,"context_line":"    CUSTOM_FPGA_INTEL_PAC_ARRIA10. This trait is intended to help match"},{"line_number":186,"context_line":"    the software drivers/libraries in the instance image. (Cyborg does"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_d1d90df3","line":183,"in_reply_to":"9f6a8fd7_02b41df1","updated":"2018-04-21 01:43:06.000000000","message":"Hi Tomasz, Cyborg itself has a db to track the usage of specific devices on each machine.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9b08161db9e2944c96d949ef437de595c8a777c4","unresolved":false,"context_lines":[{"line_number":180,"context_line":"      support, Cyborg will configure aforementioned RCs and traits on"},{"line_number":181,"context_line":"      the compute node RP in which the device resides."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"  * Cyborg will associate a device type trait with each device. For"},{"line_number":184,"context_line":"    example, CUSTOM_GPU_\u003cvendor\u003e_\u003cproduct\u003e, CUSTOM_HPTS_ZTE_\u003cmodel\u003e or"},{"line_number":185,"context_line":"    CUSTOM_FPGA_INTEL_PAC_ARRIA10. This trait is intended to help match"},{"line_number":186,"context_line":"    the software drivers/libraries in the instance image. (Cyborg does"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_58df5539","line":183,"in_reply_to":"9f6a8fd7_d1d90df3","updated":"2018-04-23 00:44:42.000000000","message":"Hi Tomasz, each device is represented as a separate nested resource provider (nRP) within the compute node RP, and traits can be applied to individual RPs. For Rocky release, such nRP support may not be available. So, we will have to apply the traits to the compute node RP itself.  \n\n\"track if this specific device was not already consumed\" Traits are properties rather than consumable resources. The resources we expose are generic accelerators, and they are tracked by Placement.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":7900,"name":"Tomasz Paszkowski","email":"ss7pro@gmail.com","username":"tpaszkowski"},"change_message_id":"2110405f1efa8fb929a600a991eae76d61d3bd6f","unresolved":false,"context_lines":[{"line_number":186,"context_line":"    the software drivers/libraries in the instance image. (Cyborg does"},{"line_number":187,"context_line":"    not handle this today.)"},{"line_number":188,"context_line":""},{"line_number":189,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":190,"context_line":"    (or with the FPGA itself if there is no PR support). E.g."},{"line_number":191,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":192,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_e2cec980","line":189,"updated":"2018-04-20 19:51:24.000000000","message":"What\u0027s the usecase for this kind of a trait?","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":27458,"name":"Li Liu","email":"liliueecg@gmail.com","username":"liliu"},"change_message_id":"e988b07e387c8f24e3b64f34606fbf41de12d1d6","unresolved":false,"context_lines":[{"line_number":186,"context_line":"    the software drivers/libraries in the instance image. (Cyborg does"},{"line_number":187,"context_line":"    not handle this today.)"},{"line_number":188,"context_line":""},{"line_number":189,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":190,"context_line":"    (or with the FPGA itself if there is no PR support). E.g."},{"line_number":191,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":192,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_71fb6146","line":189,"in_reply_to":"9f6a8fd7_e2cec980","updated":"2018-04-21 01:43:06.000000000","message":"Each FPGA can be divided into regions. For instance, region A and region B. These regions A and B are not equivalent. Nova needs this information during scheduling.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":7900,"name":"Tomasz Paszkowski","email":"ss7pro@gmail.com","username":"tpaszkowski"},"change_message_id":"2110405f1efa8fb929a600a991eae76d61d3bd6f","unresolved":false,"context_lines":[{"line_number":191,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":192,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":195,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":196,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":197,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_4291b546","line":194,"range":{"start_line":194,"start_character":4,"end_line":194,"end_character":35},"updated":"2018-04-20 19:51:24.000000000","message":"Why you want to add uuid to the trait name?","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9b08161db9e2944c96d949ef437de595c8a777c4","unresolved":false,"context_lines":[{"line_number":191,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":192,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":195,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":196,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":197,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_b8ed51f1","line":194,"range":{"start_line":194,"start_character":4,"end_line":194,"end_character":35},"in_reply_to":"9f6a8fd7_4291b546","updated":"2018-04-23 00:44:42.000000000","message":"The user/flavor should be able to ask for a specific function at a granular level, e.g. ipsec-uuid. Having it as a name is usually not sufficient: for example, two implementations of ipsec may differ in functionality, performance, bug fixes, etc.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9f9c81fc7b0770e51e864d4cb8af93eb7b6d0d19","unresolved":false,"context_lines":[{"line_number":191,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":192,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":195,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":196,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"  * For FPGAs, Cyborg should associate a CUSTOM_PROGRAMMABLE trait with "},{"line_number":199,"context_line":"    every region. This is needed to lay the groundwork for"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_6eadee86","line":196,"range":{"start_line":194,"start_character":4,"end_line":196,"end_character":74},"updated":"2018-04-19 13:31:51.000000000","message":"I realize that there is nothing preventing this, but it is not the way that placement is designed to work. Traits signify qualities of an RP, not its current state. Maintaining state by adding/removing traits is not a good design.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"10e30c2bfacbfa8771af6e391d84f905c3e26280","unresolved":false,"context_lines":[{"line_number":191,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":192,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":193,"context_line":""},{"line_number":194,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":195,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":196,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"  * For FPGAs, Cyborg should associate a CUSTOM_PROGRAMMABLE trait with "},{"line_number":199,"context_line":"    every region. This is needed to lay the groundwork for"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_c5872b0f","line":196,"range":{"start_line":194,"start_character":4,"end_line":196,"end_character":74},"in_reply_to":"9f6a8fd7_6eadee86","updated":"2018-04-20 03:16:08.000000000","message":"I thought this specific aspect has been discussed and approved  in openstack-dev. http://lists.openstack.org/pipermail/openstack-dev/2018-March/128888.html says: \"it\u0027s acceptable for the thing\ndoing the programming to set a trait indicating that that function is in play.\" We are applying and removing traits on re-programming, not necessarily when an instance is spawned/terminated.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9f9c81fc7b0770e51e864d4cb8af93eb7b6d0d19","unresolved":false,"context_lines":[{"line_number":195,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":196,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"  * For FPGAs, Cyborg should associate a CUSTOM_PROGRAMMABLE trait with "},{"line_number":199,"context_line":"    every region. This is needed to lay the groundwork for"},{"line_number":200,"context_line":"    multi-function accelerators in the future. Flavors should ask for"},{"line_number":201,"context_line":"    this trait, except in the pre-programmed case."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"  * The Cyborg agent needs to get enough information from the Cyborg driver"},{"line_number":204,"context_line":"    to create the RPs, RCs and traits. In particular, it needs to get the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_cecc3a63","line":201,"range":{"start_line":198,"start_character":4,"end_line":201,"end_character":50},"updated":"2018-04-19 13:31:51.000000000","message":"While all FPGA devices from different vendors could be CUSTOM_PROGRAMMABLE, they wouldn\u0027t all be able to handle all available bitstreams, so you should probably note that traits should be set that show what bitstreams a particular device is capable of running for the reprogrammable use cases. For the pre-programmed use case, the CUSTOM_PROGRAMMABLE should not be set, as the pre-programmed functions are simple resources.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"10e30c2bfacbfa8771af6e391d84f905c3e26280","unresolved":false,"context_lines":[{"line_number":195,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":196,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"  * For FPGAs, Cyborg should associate a CUSTOM_PROGRAMMABLE trait with "},{"line_number":199,"context_line":"    every region. This is needed to lay the groundwork for"},{"line_number":200,"context_line":"    multi-function accelerators in the future. Flavors should ask for"},{"line_number":201,"context_line":"    this trait, except in the pre-programmed case."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"  * The Cyborg agent needs to get enough information from the Cyborg driver"},{"line_number":204,"context_line":"    to create the RPs, RCs and traits. In particular, it needs to get the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_45731b4e","line":201,"range":{"start_line":198,"start_character":4,"end_line":201,"end_character":50},"in_reply_to":"9f6a8fd7_cecc3a63","updated":"2018-04-20 03:16:08.000000000","message":"The region type trait indicates which bitstreams are compatible. You are right that CUSTOM_PROGRAMMABLE should not be set for the pre-programmed use case.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9f9c81fc7b0770e51e864d4cb8af93eb7b6d0d19","unresolved":false,"context_lines":[{"line_number":248,"context_line":""},{"line_number":249,"context_line":"    | ``resourcesN: CUSTOM_ACCELERATOR\u003d1``"},{"line_number":250,"context_line":"    | ``traitsN: CUSTOM_FPGA_INTEL_PAC_ARRIA10\u003drequired``"},{"line_number":251,"context_line":"    | ``othersN: function:CUSTOM_FPGA_INTEL_FUNCTION_\u003cgzip-uuid\u003e\u003drequired``"},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * If the flavor asks for multiple devices, and also bitstreams or"},{"line_number":254,"context_line":"    functions in extra specs, the correlation of devices to extra specs"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_2e329611","line":251,"range":{"start_line":251,"start_character":8,"end_line":251,"end_character":15},"updated":"2018-04-19 13:31:51.000000000","message":"I don\u0027t know what this is supposed to be. If you include it, it will be up to Cyborg to parse the extra_specs to interpret it. You should probably define the syntax that this \u0027others\u0027 key will use.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"10e30c2bfacbfa8771af6e391d84f905c3e26280","unresolved":false,"context_lines":[{"line_number":248,"context_line":""},{"line_number":249,"context_line":"    | ``resourcesN: CUSTOM_ACCELERATOR\u003d1``"},{"line_number":250,"context_line":"    | ``traitsN: CUSTOM_FPGA_INTEL_PAC_ARRIA10\u003drequired``"},{"line_number":251,"context_line":"    | ``othersN: function:CUSTOM_FPGA_INTEL_FUNCTION_\u003cgzip-uuid\u003e\u003drequired``"},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * If the flavor asks for multiple devices, and also bitstreams or"},{"line_number":254,"context_line":"    functions in extra specs, the correlation of devices to extra specs"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_859e1352","line":251,"range":{"start_line":251,"start_character":8,"end_line":251,"end_character":15},"in_reply_to":"9f6a8fd7_2e329611","updated":"2018-04-20 03:16:08.000000000","message":"Yes, Ed. This will be interpreted by Cyborg agent, no Nova. The syntax is what I wrote above. Is it unclear?","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9f9c81fc7b0770e51e864d4cb8af93eb7b6d0d19","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * If the flavor asks for multiple devices, and also bitstreams or"},{"line_number":254,"context_line":"    functions in extra specs, the correlation of devices to extra specs"},{"line_number":255,"context_line":"    needs to be worked out."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"Scheduling workflow"},{"line_number":258,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_aeb22693","line":255,"updated":"2018-04-19 13:31:51.000000000","message":"Perhaps add a comment that this is not in scope for the Rocky cycle.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"10e30c2bfacbfa8771af6e391d84f905c3e26280","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * If the flavor asks for multiple devices, and also bitstreams or"},{"line_number":254,"context_line":"    functions in extra specs, the correlation of devices to extra specs"},{"line_number":255,"context_line":"    needs to be worked out."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"Scheduling workflow"},{"line_number":258,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9f6a8fd7_25c6274a","line":255,"in_reply_to":"9f6a8fd7_aeb22693","updated":"2018-04-20 03:16:08.000000000","message":"The othersN syntax is expected to address this. This point should be removed. Thanks for catching this.","commit_id":"52a2bb6644749dc8537be079c8e9c54d194773f3"},{"author":{"_account_id":25547,"name":"jiapei","email":"jiapeish@gmail.com","username":"jeremy.jia"},"change_message_id":"e8a94edab08c5290f8a56d93e30f9739da52cd28","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Terminology"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":49,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"},{"line_number":50,"context_line":"  device itself or a virtualized version of it (e.g. vGPUs). For FPGAs, an"},{"line_number":51,"context_line":"  accelerator is either the entire device, a region within the device or a"},{"line_number":52,"context_line":"  function."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_1ad12bce","line":49,"range":{"start_line":48,"start_character":15,"end_line":49,"end_character":35},"updated":"2018-05-08 15:24:00.000000000","message":"I think we can expand this definition. Some accelerator resource may not reside in the local host, but in the remote server/pool/rack, and it\u0027s remote mounted to the local host through network (like Fabric or sth redfish) or something else.","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"1c0baacf294a796bf3e58ed8ad8538ce39dd570b","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Terminology"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":49,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"},{"line_number":50,"context_line":"  device itself or a virtualized version of it (e.g. vGPUs). For FPGAs, an"},{"line_number":51,"context_line":"  accelerator is either the entire device, a region within the device or a"},{"line_number":52,"context_line":"  function."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_cadafc15","line":49,"range":{"start_line":48,"start_character":15,"end_line":49,"end_character":35},"in_reply_to":"5f7c97a3_1ad12bce","updated":"2018-05-08 18:57:12.000000000","message":"If the remote FPGA is attached to a compute node before Cyborg discovery, as a PCI device, that is the same as today. Are you looking at other usage models?","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":25547,"name":"jiapei","email":"jiapeish@gmail.com","username":"jeremy.jia"},"change_message_id":"c225c6b2a15509717a0a3ea23416e7bd8fbae68c","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Terminology"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":49,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"},{"line_number":50,"context_line":"  device itself or a virtualized version of it (e.g. vGPUs). For FPGAs, an"},{"line_number":51,"context_line":"  accelerator is either the entire device, a region within the device or a"},{"line_number":52,"context_line":"  function."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_d6a37e9d","line":49,"range":{"start_line":48,"start_character":15,"end_line":49,"end_character":35},"in_reply_to":"5f7c97a3_cadafc15","updated":"2018-05-09 09:16:31.000000000","message":"Yes, if we haven\u0027t attached GPU/NVMe devices to a compute node, can Cyborg auto discover it and auto attach it?","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"5d2073b4791076100b8ed875c224d14894c86cfc","unresolved":false,"context_lines":[{"line_number":45,"context_line":""},{"line_number":46,"context_line":"Terminology"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":"* Accelerator: The unit that can be assigned to an instance for"},{"line_number":49,"context_line":"  offloading specific functionality. For non-FPGA devices, it is either the"},{"line_number":50,"context_line":"  device itself or a virtualized version of it (e.g. vGPUs). For FPGAs, an"},{"line_number":51,"context_line":"  accelerator is either the entire device, a region within the device or a"},{"line_number":52,"context_line":"  function."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_a25e8e41","line":49,"range":{"start_line":48,"start_character":15,"end_line":49,"end_character":35},"in_reply_to":"5f7c97a3_d6a37e9d","updated":"2018-05-10 23:19:31.000000000","message":"Please see my reply to your next comment.","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":25547,"name":"jiapei","email":"jiapeish@gmail.com","username":"jeremy.jia"},"change_message_id":"e8a94edab08c5290f8a56d93e30f9739da52cd28","unresolved":false,"context_lines":[{"line_number":99,"context_line":"---------"},{"line_number":100,"context_line":"We need to satisfy the following use cases for the tenant role:"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Device as a Service (DaaS): The flavor asks for a device."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"  * FPGA variation: The flavor asks for a device to which specific"},{"line_number":105,"context_line":"    bitstream(s) can be applied. There are two variations for secure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_fa1f8f6d","line":102,"range":{"start_line":102,"start_character":2,"end_line":102,"end_character":21},"updated":"2018-05-08 15:24:00.000000000","message":"Why don\u0027t we add \"Accelerator as a Service\" too?  For example, the accelerator is in another rack, and we connect it through Fabric.","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":25547,"name":"jiapei","email":"jiapeish@gmail.com","username":"jeremy.jia"},"change_message_id":"c225c6b2a15509717a0a3ea23416e7bd8fbae68c","unresolved":false,"context_lines":[{"line_number":99,"context_line":"---------"},{"line_number":100,"context_line":"We need to satisfy the following use cases for the tenant role:"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Device as a Service (DaaS): The flavor asks for a device."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"  * FPGA variation: The flavor asks for a device to which specific"},{"line_number":105,"context_line":"    bitstream(s) can be applied. There are two variations for secure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_b644c2b8","line":102,"range":{"start_line":102,"start_character":2,"end_line":102,"end_character":21},"in_reply_to":"5f7c97a3_0adb14ed","updated":"2018-05-09 09:16:31.000000000","message":"Yes, if the user issues a request for a accelerated device, can the accelerator/device be attached to the compute node? \n\nAs a user, I may not want to do the attachment by myself. Why not I ask for a flavor with a particular device and the Cyborg does that attachment?","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"5d2073b4791076100b8ed875c224d14894c86cfc","unresolved":false,"context_lines":[{"line_number":99,"context_line":"---------"},{"line_number":100,"context_line":"We need to satisfy the following use cases for the tenant role:"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Device as a Service (DaaS): The flavor asks for a device."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"  * FPGA variation: The flavor asks for a device to which specific"},{"line_number":105,"context_line":"    bitstream(s) can be applied. There are two variations for secure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_82611208","line":102,"range":{"start_line":102,"start_character":2,"end_line":102,"end_character":21},"in_reply_to":"5f7c97a3_b644c2b8","updated":"2018-05-10 23:19:31.000000000","message":"I think I understand it. We can treat this similar to shared storage, with shared resource providers. So, we define an aggregate of compute nodes and associate a shared RP with that. But I need to work out the details and the catches. Is this something you want in Rocky?","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"1c0baacf294a796bf3e58ed8ad8538ce39dd570b","unresolved":false,"context_lines":[{"line_number":99,"context_line":"---------"},{"line_number":100,"context_line":"We need to satisfy the following use cases for the tenant role:"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Device as a Service (DaaS): The flavor asks for a device."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"  * FPGA variation: The flavor asks for a device to which specific"},{"line_number":105,"context_line":"    bitstream(s) can be applied. There are two variations for secure"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5f7c97a3_0adb14ed","line":102,"range":{"start_line":102,"start_character":2,"end_line":102,"end_character":21},"in_reply_to":"5f7c97a3_fa1f8f6d","updated":"2018-05-08 18:57:12.000000000","message":"Trying to understand: are you saying a request is issued with a particular flavor, and that causes an accelerator (device) to be attached to a compute node?","commit_id":"867999ac0e631135f6a3d78f1add8106dcf050a1"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"20c8645ec1fce38aeca7fb5bf22230f04acaaf69","unresolved":false,"context_lines":[{"line_number":183,"context_line":""},{"line_number":184,"context_line":"  * Cyborg will associate a device type trait with each device. For"},{"line_number":185,"context_line":"    example, CUSTOM_GPU_\u003cvendor\u003e_\u003cproduct\u003e, CUSTOM_HPTS_ZTE_\u003cmodel\u003e or"},{"line_number":186,"context_line":"    CUSTOM_FPGA_INTEL_PAC_ARRIA10. This trait is intended to help match"},{"line_number":187,"context_line":"    the software drivers/libraries in the instance image. (Cyborg does"},{"line_number":188,"context_line":"    not handle this today.)"},{"line_number":189,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_33a708c0","line":186,"range":{"start_line":186,"start_character":4,"end_line":186,"end_character":33},"updated":"2018-05-14 13:40:46.000000000","message":"This is not ideal. Traits should represent what a resource provider is capable of. I understand that the vendor/product represents what sorts of bitstreams will run on it, but you get into the case where, say, Intel releases a new version that can run all of the old as well as something new. It would be better in that case to have CUSTOM_CAN_RUN_A and then add CUSTOM_CAN_RUN_B. The older device will only have the first trait, while the newer device will have both.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"1a44fa51f3ae439330babed6384cf9bd852bfa8e","unresolved":false,"context_lines":[{"line_number":183,"context_line":""},{"line_number":184,"context_line":"  * Cyborg will associate a device type trait with each device. For"},{"line_number":185,"context_line":"    example, CUSTOM_GPU_\u003cvendor\u003e_\u003cproduct\u003e, CUSTOM_HPTS_ZTE_\u003cmodel\u003e or"},{"line_number":186,"context_line":"    CUSTOM_FPGA_INTEL_PAC_ARRIA10. This trait is intended to help match"},{"line_number":187,"context_line":"    the software drivers/libraries in the instance image. (Cyborg does"},{"line_number":188,"context_line":"    not handle this today.)"},{"line_number":189,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_8d9c7a6a","line":186,"range":{"start_line":186,"start_character":4,"end_line":186,"end_character":33},"in_reply_to":"5f7c97a3_33a708c0","updated":"2018-05-16 17:32:37.000000000","message":"A GPU can run anything that can be compiled from Cuda. An FPGA can run any bitstream synthesized for that type. Identifying all functions that _can_ be supported on a device type has two issues:\n* Scale: There can be thousands of functions/bitstreams/... which can be supported on a device.\n* Usability: Typically, functions/bitstreams are identified by UUIDs. Not sure about Cuda functionality. Having thousands of traits based on UUIDs may not be well-received by operators.\n* Maintainability: Since the Cuda program or the FPGA bitstream may be created after the product has shipped, the operator has to update traits o existing devices when he uploads new bitstreams. Here too, I am not sure how it works for GPUs.\n\nThe traits like CUSTOM_FPGA_INTEL_PAC_ARRIA10 and CUSTOM_FPGA_..._REGION_\u003cuuid\u003e effectively indicate what funcaionality can be supported in that device type. Further, metadata in the bitstream can be used to identify which device types it is compatible with. So, we can have generic logic to match them up, while also conveying useful info to operators.\n\nHTH.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"20c8645ec1fce38aeca7fb5bf22230f04acaaf69","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":191,"context_line":"    (or with the FPGA itself if there is no PR support). E.g."},{"line_number":192,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":193,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_b3a0f891","line":192,"range":{"start_line":192,"start_character":4,"end_line":192,"end_character":35},"updated":"2018-05-14 13:40:46.000000000","message":"The comment above applies here too.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"1a44fa51f3ae439330babed6384cf9bd852bfa8e","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":191,"context_line":"    (or with the FPGA itself if there is no PR support). E.g."},{"line_number":192,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":193,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_ada5b6a0","line":192,"range":{"start_line":192,"start_character":4,"end_line":192,"end_character":35},"in_reply_to":"5f7c97a3_b3a0f891","updated":"2018-05-16 17:32:37.000000000","message":"Could I ask if the objection is against having vendor names in the trait? In principle, a UUID should be unique across vendors. However, the vendor name here can help the operator in creating flavors, rather than rely on UUIDs alone.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"20c8645ec1fce38aeca7fb5bf22230f04acaaf69","unresolved":false,"context_lines":[{"line_number":193,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":196,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":197,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"  * For FPGAs, Cyborg should associate a CUSTOM_PROGRAMMABLE trait with"}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_93ce1c61","line":196,"range":{"start_line":196,"start_character":42,"end_line":196,"end_character":71},"updated":"2018-05-14 13:40:46.000000000","message":"Again, having specific vendors in the trait is not ideal. It would be better to reference the program.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"1a44fa51f3ae439330babed6384cf9bd852bfa8e","unresolved":false,"context_lines":[{"line_number":193,"context_line":"    Service with FPGAs. (Cyborg does not handle this today.)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":196,"context_line":"    when the region gets programmed. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":197,"context_line":"    This is needed for AFaaS use case. (Cyborg does not handle this today.)"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"  * For FPGAs, Cyborg should associate a CUSTOM_PROGRAMMABLE trait with"}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_8dd8fa1d","line":196,"range":{"start_line":196,"start_character":42,"end_line":196,"end_character":71},"in_reply_to":"5f7c97a3_93ce1c61","updated":"2018-05-16 17:32:37.000000000","message":"As with region type UUIDs, we could say that function UUIDs also ought to be unique across vendors, and so we don\u0027t really need the vendor name here. However, for operator\u0027s ease of use in creating flavors, being more explicit will be helpful. We can pose this question to operators during the Vancouver forum.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"20c8645ec1fce38aeca7fb5bf22230f04acaaf69","unresolved":false,"context_lines":[{"line_number":256,"context_line":"      * NOTE: This assumes the operator has configured the function name"},{"line_number":257,"context_line":"        as a property of the bitstream image in Glance. The FPGA"},{"line_number":258,"context_line":"        hardware is not expected to expose function names, and so"},{"line_number":259,"context_line":"        Cyborg will not represent function names as traits. "},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"  * A flavor may ask for other RCs, such as local memory."},{"line_number":262,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_d3fed469","line":259,"updated":"2018-05-14 13:40:46.000000000","message":"Horrors! Trailing whitespace!!!\n:)","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"1a44fa51f3ae439330babed6384cf9bd852bfa8e","unresolved":false,"context_lines":[{"line_number":256,"context_line":"      * NOTE: This assumes the operator has configured the function name"},{"line_number":257,"context_line":"        as a property of the bitstream image in Glance. The FPGA"},{"line_number":258,"context_line":"        hardware is not expected to expose function names, and so"},{"line_number":259,"context_line":"        Cyborg will not represent function names as traits. "},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"  * A flavor may ask for other RCs, such as local memory."},{"line_number":262,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"5f7c97a3_8d46babf","line":259,"in_reply_to":"5f7c97a3_d3fed469","updated":"2018-05-16 17:32:37.000000000","message":"Aargh :) Will fix.","commit_id":"b2ca7ec0001eb72d705e7881b6656e01c2cc7944"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"edca8e8ba9e061fa85301d42713cdd11290efb63","unresolved":false,"context_lines":[{"line_number":87,"context_line":"Resource Providers (RPs), Resource Classes (RCs) and Traits."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Though PCI Express is entrenched in the data center, some accelerators and NICs"},{"line_number":90,"context_line":"may be exposed to the host via some other protocol. Even with PCI, the"},{"line_number":91,"context_line":"connections between accelerator components and PCI functions"},{"line_number":92,"context_line":"may vary across devices. Accordingly, Cyborg should not represent"},{"line_number":93,"context_line":"accelerators as PCI functions."},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"For instances that need accelerators, we need to define a way for Cyborg to be"},{"line_number":96,"context_line":"included seamlessly in the Nova scheduling workflow."}],"source_content_type":"text/x-rst","patch_set":10,"id":"5f7c97a3_b273324e","line":93,"range":{"start_line":90,"start_character":52,"end_line":93,"end_character":30},"updated":"2018-05-29 08:14:39.000000000","message":"what does PCI Function here mean? Does it mean the function you mentioned above like matrix multiplication? Why it will vary across devi ces. Please let me know if i missed something...","commit_id":"e91ae854ee427edc11f92ee876f24b6fa2549f92"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"9c5aaa76c9b768a2044f6199e6f688436736dbe3","unresolved":false,"context_lines":[{"line_number":87,"context_line":"Resource Providers (RPs), Resource Classes (RCs) and Traits."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Though PCI Express is entrenched in the data center, some accelerators and NICs"},{"line_number":90,"context_line":"may be exposed to the host via some other protocol. Even with PCI, the"},{"line_number":91,"context_line":"connections between accelerator components and PCI functions"},{"line_number":92,"context_line":"may vary across devices. Accordingly, Cyborg should not represent"},{"line_number":93,"context_line":"accelerators as PCI functions."},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"For instances that need accelerators, we need to define a way for Cyborg to be"},{"line_number":96,"context_line":"included seamlessly in the Nova scheduling workflow."}],"source_content_type":"text/x-rst","patch_set":10,"id":"5f7c97a3_71582689","line":93,"range":{"start_line":90,"start_character":52,"end_line":93,"end_character":30},"in_reply_to":"5f7c97a3_7d0a3d79","updated":"2018-05-30 02:48:29.000000000","message":"okay, I understood. Thanks for your explanation :)","commit_id":"e91ae854ee427edc11f92ee876f24b6fa2549f92"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"225b2a9590b218d0ece4c02fc2170fc0925c8156","unresolved":false,"context_lines":[{"line_number":87,"context_line":"Resource Providers (RPs), Resource Classes (RCs) and Traits."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Though PCI Express is entrenched in the data center, some accelerators and NICs"},{"line_number":90,"context_line":"may be exposed to the host via some other protocol. Even with PCI, the"},{"line_number":91,"context_line":"connections between accelerator components and PCI functions"},{"line_number":92,"context_line":"may vary across devices. Accordingly, Cyborg should not represent"},{"line_number":93,"context_line":"accelerators as PCI functions."},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"For instances that need accelerators, we need to define a way for Cyborg to be"},{"line_number":96,"context_line":"included seamlessly in the Nova scheduling workflow."}],"source_content_type":"text/x-rst","patch_set":10,"id":"5f7c97a3_7d0a3d79","line":93,"range":{"start_line":90,"start_character":52,"end_line":93,"end_character":30},"in_reply_to":"5f7c97a3_b273324e","updated":"2018-05-29 21:23:26.000000000","message":"No, PCI function is a PCI-centric term, independent of FPGAs. It is a BDF (bus-device-function). The accelerators inside an FPGA are tied to PCI functions in some way. For example, an FPGA may implement SR-IOV, and accelerators are tied to VFs. Or, an FPGA may not implement SR-IOV, but just several physical functions, which are tied to accelerators. The main point is that, implementations may vary.","commit_id":"e91ae854ee427edc11f92ee876f24b6fa2549f92"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"edca8e8ba9e061fa85301d42713cdd11290efb63","unresolved":false,"context_lines":[{"line_number":166,"context_line":"  * Cyborg represents a generic accelerator for a device type as a custom"},{"line_number":167,"context_line":"    Resource Class (RC) for that type, such as CUSTOM_ACCELERATOR_GPU,"},{"line_number":168,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":169,"context_line":"    different device types. (Today, Cyborg treats PCI PFs and"},{"line_number":170,"context_line":"    VFs as resources.)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"  * Device-local memory can also be represented as a RC. (Cyborg does"},{"line_number":173,"context_line":"    not handle this today.)"}],"source_content_type":"text/x-rst","patch_set":10,"id":"5f7c97a3_32cde22f","line":170,"range":{"start_line":169,"start_character":29,"end_line":170,"end_character":21},"updated":"2018-05-29 08:14:39.000000000","message":"does it mean if one user allocate a VF, we should -1 for his FPGA quota?  Or there are CUSTOM_ACCELERATOR_FPGA_VF and CUSTOM_ACCELERATOR_FPGA_PF these two resource classes. I\u0027m a little confused about it... Could you please give me some more explanation?","commit_id":"e91ae854ee427edc11f92ee876f24b6fa2549f92"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c713ca6d16c471464f0af778943e143af7fa0c2f","unresolved":false,"context_lines":[{"line_number":166,"context_line":"  * Cyborg represents a generic accelerator for a device type as a custom"},{"line_number":167,"context_line":"    Resource Class (RC) for that type, such as CUSTOM_ACCELERATOR_GPU,"},{"line_number":168,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":169,"context_line":"    different device types. (Today, Cyborg treats PCI PFs and"},{"line_number":170,"context_line":"    VFs as resources.)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"  * Device-local memory can also be represented as a RC. (Cyborg does"},{"line_number":173,"context_line":"    not handle this today.)"}],"source_content_type":"text/x-rst","patch_set":10,"id":"5f7c97a3_055875f7","line":170,"range":{"start_line":169,"start_character":29,"end_line":170,"end_character":21},"in_reply_to":"5f7c97a3_114212ac","updated":"2018-05-31 01:13:01.000000000","message":"No, Cyborg will make changes, as proposed in this spec. In particular, the resource classes are CUSTOM_ACCELERATOR_FPGA, etc., not PCI-focused.","commit_id":"e91ae854ee427edc11f92ee876f24b6fa2549f92"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"225b2a9590b218d0ece4c02fc2170fc0925c8156","unresolved":false,"context_lines":[{"line_number":166,"context_line":"  * Cyborg represents a generic accelerator for a device type as a custom"},{"line_number":167,"context_line":"    Resource Class (RC) for that type, such as CUSTOM_ACCELERATOR_GPU,"},{"line_number":168,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":169,"context_line":"    different device types. (Today, Cyborg treats PCI PFs and"},{"line_number":170,"context_line":"    VFs as resources.)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"  * Device-local memory can also be represented as a RC. (Cyborg does"},{"line_number":173,"context_line":"    not handle this today.)"}],"source_content_type":"text/x-rst","patch_set":10,"id":"5f7c97a3_7d7f9dde","line":170,"range":{"start_line":169,"start_character":29,"end_line":170,"end_character":21},"in_reply_to":"5f7c97a3_32cde22f","updated":"2018-05-29 21:23:26.000000000","message":"We do not want to represent PCI PFs and VFs as resources in Nova, because we do not want this to be PCI-centric. We will represent accelerators as resource classes e.g. CUSTOM_ACCELERATOR_GPU, CUSTOM_ACCELERATOR_FPGA. Quotas will apply to such RCs.","commit_id":"e91ae854ee427edc11f92ee876f24b6fa2549f92"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"9c5aaa76c9b768a2044f6199e6f688436736dbe3","unresolved":false,"context_lines":[{"line_number":166,"context_line":"  * Cyborg represents a generic accelerator for a device type as a custom"},{"line_number":167,"context_line":"    Resource Class (RC) for that type, such as CUSTOM_ACCELERATOR_GPU,"},{"line_number":168,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":169,"context_line":"    different device types. (Today, Cyborg treats PCI PFs and"},{"line_number":170,"context_line":"    VFs as resources.)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"  * Device-local memory can also be represented as a RC. (Cyborg does"},{"line_number":173,"context_line":"    not handle this today.)"}],"source_content_type":"text/x-rst","patch_set":10,"id":"5f7c97a3_114212ac","line":170,"range":{"start_line":169,"start_character":29,"end_line":170,"end_character":21},"in_reply_to":"5f7c97a3_7d7f9dde","updated":"2018-05-30 02:48:29.000000000","message":"Yes, I got the point. So Cyborg will always treats PCI PFs and VFs as resources or it will make some changes?","commit_id":"e91ae854ee427edc11f92ee876f24b6fa2549f92"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/cyborg/+spec/cyborg-nova-interaction"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Cyborg is a service for managing accelerators, such as FPGAs, GPUs, etc. For"},{"line_number":14,"context_line":"scheduling an instance that needs accelerators, Cyborg needs to work with Nova"},{"line_number":15,"context_line":"at three levels:"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_80d8f31d","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":72},"updated":"2018-05-31 07:22:57.000000000","message":"\"Cyborg provides a general management framework for accelerators such as FPGA, GPU, etc.\"\n\nLet\u0027s copy it from cyborg project README[1]. :)\n\n[1] https://github.com/openstack/cyborg/blob/master/README.rst","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/cyborg/+spec/cyborg-nova-interaction"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Cyborg is a service for managing accelerators, such as FPGAs, GPUs, etc. For"},{"line_number":14,"context_line":"scheduling an instance that needs accelerators, Cyborg needs to work with Nova"},{"line_number":15,"context_line":"at three levels:"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5b8fe02d","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":72},"in_reply_to":"5f7c97a3_80d8f31d","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"0d4683c6d1308e3826743917453a7b6636d22c4d","unresolved":false,"context_lines":[{"line_number":86,"context_line":"with Nova\u0027s Placement API. Specifically, they must be modeled in terms of"},{"line_number":87,"context_line":"Resource Providers (RPs), Resource Classes (RCs) and Traits."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Though PCI Express is entrenched in the data center, some accelerators and"},{"line_number":90,"context_line":"may be exposed to the host via some other protocol. Even with PCI, the"},{"line_number":91,"context_line":"connections between accelerator components and PCI functions"},{"line_number":92,"context_line":"may vary across devices. Accordingly, Cyborg should not represent"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_60891773","line":89,"range":{"start_line":89,"start_character":71,"end_line":89,"end_character":74},"updated":"2018-05-31 02:15:52.000000000","message":"remove and","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":86,"context_line":"with Nova\u0027s Placement API. Specifically, they must be modeled in terms of"},{"line_number":87,"context_line":"Resource Providers (RPs), Resource Classes (RCs) and Traits."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"Though PCI Express is entrenched in the data center, some accelerators and"},{"line_number":90,"context_line":"may be exposed to the host via some other protocol. Even with PCI, the"},{"line_number":91,"context_line":"connections between accelerator components and PCI functions"},{"line_number":92,"context_line":"may vary across devices. Accordingly, Cyborg should not represent"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_dba3f0b0","line":89,"range":{"start_line":89,"start_character":71,"end_line":89,"end_character":74},"in_reply_to":"5f7c97a3_60891773","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"0d4683c6d1308e3826743917453a7b6636d22c4d","unresolved":false,"context_lines":[{"line_number":116,"context_line":""},{"line_number":117,"context_line":"    * Direct Programming: The instance directly programs the FPGA"},{"line_number":118,"context_line":"      region assigned to it, without delegating it to Cyborg. The"},{"line_number":119,"context_line":"      security questions that this raises needs to be addressed in"},{"line_number":120,"context_line":"      the future."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"* Accelerated Function as a Service (AFaaS): The flavor asks for a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_e094478d","line":119,"range":{"start_line":119,"start_character":42,"end_line":119,"end_character":47},"updated":"2018-05-31 02:15:52.000000000","message":"s/needs/need","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":116,"context_line":""},{"line_number":117,"context_line":"    * Direct Programming: The instance directly programs the FPGA"},{"line_number":118,"context_line":"      region assigned to it, without delegating it to Cyborg. The"},{"line_number":119,"context_line":"      security questions that this raises needs to be addressed in"},{"line_number":120,"context_line":"      the future."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"* Accelerated Function as a Service (AFaaS): The flavor asks for a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_9bad789d","line":119,"range":{"start_line":119,"start_character":42,"end_line":119,"end_character":47},"in_reply_to":"5f7c97a3_e094478d","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":152,"context_line":"Proposed change"},{"line_number":153,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"Representation"},{"line_number":156,"context_line":"--------------"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a custom"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_fe7dc7e6","line":155,"range":{"start_line":155,"start_character":0,"end_line":155,"end_character":14},"updated":"2018-05-31 07:22:57.000000000","message":"How to report these to placement? and how to discovery the device?","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":152,"context_line":"Proposed change"},{"line_number":153,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"Representation"},{"line_number":156,"context_line":"--------------"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a custom"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5b9da055","line":155,"range":{"start_line":155,"start_character":0,"end_line":155,"end_character":14},"in_reply_to":"5f7c97a3_fe7dc7e6","updated":"2018-06-05 04:29:47.000000000","message":"Cyborg will call into placement API (https://developer.openstack.org/api-ref/placement/).\n\nDiscovery is done by Cyborg drivers/agent in the compute node. That is outside the scope of this spec.","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":156,"context_line":"--------------"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a custom"},{"line_number":159,"context_line":"    Resource Class (RC) for that type, such as CUSTOM_ACCELERATOR_GPU,"},{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":161,"context_line":"    different device types."},{"line_number":162,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_7b48ea5a","line":159,"range":{"start_line":159,"start_character":39,"end_line":159,"end_character":70},"updated":"2018-05-31 07:22:57.000000000","message":"Could we first give a generalized name format before here？Just a suggestion, I think it would be more clear.\n\n(1) Show a generalized format resource class name, like:\nCUSTOM_ACCELERATOR_{DEVICE_TYPE}\n(2) Introduce the meaning of this generalized class name, and also explain the \"device_type\" can be GPU, FPGA.\n(3) Give some examples.","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":156,"context_line":"--------------"},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a custom"},{"line_number":159,"context_line":"    Resource Class (RC) for that type, such as CUSTOM_ACCELERATOR_GPU,"},{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":161,"context_line":"    different device types."},{"line_number":162,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_fba90c7c","line":159,"range":{"start_line":159,"start_character":39,"end_line":159,"end_character":70},"in_reply_to":"5f7c97a3_7b48ea5a","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":161,"context_line":"    different device types."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"  * Device-local memory can also be represented as a RC."},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":166,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_205d5f78","line":163,"range":{"start_line":163,"start_character":4,"end_line":163,"end_character":23},"updated":"2018-05-31 07:22:57.000000000","message":"Maybe we need add some brief description about the meaning of Device-local memory, and give a resource class name in here, like CUSTOM_XXX","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas for"},{"line_number":161,"context_line":"    different device types."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"  * Device-local memory can also be represented as a RC."},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":166,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_1ba728aa","line":163,"range":{"start_line":163,"start_character":4,"end_line":163,"end_character":23},"in_reply_to":"5f7c97a3_205d5f78","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":165,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":166,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"},{"line_number":167,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":168,"context_line":"    of that device type\u0027s RC. (This depends on nested RP support in Nova.)"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"  * Cyborg will associate a Vendor/Category trait with each device."},{"line_number":171,"context_line":"    E.g. CUSTOM_GPU_AMD or CUSTOM_FPGA_XILINX."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_80fb1345","line":168,"range":{"start_line":168,"start_character":31,"end_line":168,"end_character":73},"updated":"2018-05-31 07:22:57.000000000","message":"It is better to add N-R-P ref link [1] in here.\n\n[1] https://specs.openstack.org/openstack/nova-specs/specs/ocata/approved/nested-resource-providers.html","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":165,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":166,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"},{"line_number":167,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":168,"context_line":"    of that device type\u0027s RC. (This depends on nested RP support in Nova.)"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"  * Cyborg will associate a Vendor/Category trait with each device."},{"line_number":171,"context_line":"    E.g. CUSTOM_GPU_AMD or CUSTOM_FPGA_XILINX."}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_dbba10bf","line":168,"range":{"start_line":168,"start_character":31,"end_line":168,"end_character":73},"in_reply_to":"5f7c97a3_80fb1345","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":167,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":168,"context_line":"    of that device type\u0027s RC. (This depends on nested RP support in Nova.)"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"  * Cyborg will associate a Vendor/Category trait with each device."},{"line_number":171,"context_line":"    E.g. CUSTOM_GPU_AMD or CUSTOM_FPGA_XILINX."},{"line_number":172,"context_line":"    This trait is intended to help match the software drivers/libraries"},{"line_number":173,"context_line":"    in the instance image. This is meant to be used in a flavor when a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_80579326","line":170,"range":{"start_line":170,"start_character":28,"end_line":170,"end_character":43},"updated":"2018-05-31 07:22:57.000000000","message":"vendor/category","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":167,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":168,"context_line":"    of that device type\u0027s RC. (This depends on nested RP support in Nova.)"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"  * Cyborg will associate a Vendor/Category trait with each device."},{"line_number":171,"context_line":"    E.g. CUSTOM_GPU_AMD or CUSTOM_FPGA_XILINX."},{"line_number":172,"context_line":"    This trait is intended to help match the software drivers/libraries"},{"line_number":173,"context_line":"    in the instance image. This is meant to be used in a flavor when a"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5bc6004c","line":170,"range":{"start_line":170,"start_character":28,"end_line":170,"end_character":43},"in_reply_to":"5f7c97a3_80579326","updated":"2018-06-05 04:29:47.000000000","message":"Fixed, switched to a more appropriate terminology","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":178,"context_line":"    E.g. CUSTOM_FPGA_INTEL_ARRIA10."},{"line_number":179,"context_line":"    This is not a product name, but the name of a device family, used to"},{"line_number":180,"context_line":"    match software in the instance image with the device family. This is"},{"line_number":181,"context_line":"    a refinement of the Vendor/Category Trait. It is meant to be used in"},{"line_number":182,"context_line":"    a flavor when there are different drivers/libraries for different"},{"line_number":183,"context_line":"    device families. Since it may be tough to forecast whether a new"},{"line_number":184,"context_line":"    device family will need a new driver/library, it may make sense to"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_802c73ac","line":181,"range":{"start_line":181,"start_character":24,"end_line":181,"end_character":39},"updated":"2018-05-31 07:22:57.000000000","message":"ditto","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":178,"context_line":"    E.g. CUSTOM_FPGA_INTEL_ARRIA10."},{"line_number":179,"context_line":"    This is not a product name, but the name of a device family, used to"},{"line_number":180,"context_line":"    match software in the instance image with the device family. This is"},{"line_number":181,"context_line":"    a refinement of the Vendor/Category Trait. It is meant to be used in"},{"line_number":182,"context_line":"    a flavor when there are different drivers/libraries for different"},{"line_number":183,"context_line":"    device families. Since it may be tough to forecast whether a new"},{"line_number":184,"context_line":"    device family will need a new driver/library, it may make sense to"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_fbc26c34","line":181,"range":{"start_line":181,"start_character":24,"end_line":181,"end_character":39},"in_reply_to":"5f7c97a3_802c73ac","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":208,"context_line":"    device type string, region IDs and function IDs from the driver. This"},{"line_number":209,"context_line":"    requires the driver/agent interface to be enhanced."},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"Flavors"},{"line_number":212,"context_line":"-------"},{"line_number":213,"context_line":"  For the sake of illustrating how the device representation in Nova"},{"line_number":214,"context_line":"  can be used, and for completeness, we now show how to define flavors"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_3e149fe8","line":211,"range":{"start_line":211,"start_character":0,"end_line":211,"end_character":7},"updated":"2018-05-31 07:22:57.000000000","message":"This \"flavor\" means the Nova flavor, right? So, would you mind give a brief introduction to how to define flavor, or give a ref link[1] at least.\n\n[1] Allow custom resource classes in flavor extra specs https://specs.openstack.org/openstack/nova-specs/specs/pike/implemented/custom-resource-classes-in-flavors.html","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":208,"context_line":"    device type string, region IDs and function IDs from the driver. This"},{"line_number":209,"context_line":"    requires the driver/agent interface to be enhanced."},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"Flavors"},{"line_number":212,"context_line":"-------"},{"line_number":213,"context_line":"  For the sake of illustrating how the device representation in Nova"},{"line_number":214,"context_line":"  can be used, and for completeness, we now show how to define flavors"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5bcb6049","line":211,"range":{"start_line":211,"start_character":0,"end_line":211,"end_character":7},"in_reply_to":"5f7c97a3_3e149fe8","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":283,"context_line":"  * A request spec with a flavor comes to Nova conductor/scheduler."},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"  * Placement API returns the list of RPs which contain the requested"},{"line_number":286,"context_line":"    resources with matching traits. (With nested RP support, the RPs"},{"line_number":287,"context_line":"    returned are device/region RPs. Without it, they are compute node RPs.)"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"  * FPGA-specific: For AFaaS orchestration-programmed use case, Placement"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_5b896e72","line":286,"range":{"start_line":286,"start_character":65,"end_line":286,"end_character":68},"updated":"2018-05-31 07:22:57.000000000","message":"s/RPs returned/returned RPs/ ?","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":283,"context_line":"  * A request spec with a flavor comes to Nova conductor/scheduler."},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"  * Placement API returns the list of RPs which contain the requested"},{"line_number":286,"context_line":"    resources with matching traits. (With nested RP support, the RPs"},{"line_number":287,"context_line":"    returned are device/region RPs. Without it, they are compute node RPs.)"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"  * FPGA-specific: For AFaaS orchestration-programmed use case, Placement"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_7bfdfc6c","line":286,"range":{"start_line":286,"start_character":65,"end_line":286,"end_character":68},"in_reply_to":"5f7c97a3_5b896e72","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"9a7019ee6df8f4567285a5036a0773b7ac2390a5","unresolved":false,"context_lines":[{"line_number":300,"context_line":"    This can be addressed later and is not a priority for Rocky release."},{"line_number":301,"context_line":"    See References."},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"  * Nova compute calls os-acc/Cyborg."},{"line_number":304,"context_line":""},{"line_number":305,"context_line":"  * FPGA-specific: If the request spec asks for a function X in extra specs,"},{"line_number":306,"context_line":"    but X is not present in the selected region RP, Cyborg should program"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_de380ba1","line":303,"range":{"start_line":303,"start_character":17,"end_line":303,"end_character":37},"updated":"2018-05-31 07:22:57.000000000","message":"It\u0027s better to describe clearly. such as what happened in here?  Which REST API or library would be called? What the effect after calling?","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":300,"context_line":"    This can be addressed later and is not a priority for Rocky release."},{"line_number":301,"context_line":"    See References."},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"  * Nova compute calls os-acc/Cyborg."},{"line_number":304,"context_line":""},{"line_number":305,"context_line":"  * FPGA-specific: If the request spec asks for a function X in extra specs,"},{"line_number":306,"context_line":"    but X is not present in the selected region RP, Cyborg should program"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_1bee48b4","line":303,"range":{"start_line":303,"start_character":17,"end_line":303,"end_character":37},"in_reply_to":"5f7c97a3_de380ba1","updated":"2018-06-05 04:29:47.000000000","message":"That is addressed in another spec. Will give a link.","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":25738,"name":"Xinran WANG","email":"xin-ran.wang@intel.com","username":"Xinran"},"change_message_id":"0d4683c6d1308e3826743917453a7b6636d22c4d","unresolved":false,"context_lines":[{"line_number":430,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":431,"context_line":""},{"line_number":432,"context_line":"Optional section intended to be used each time the spec is updated to describe"},{"line_number":433,"context_line":"new design, API or any database schema updated. Useful to let reader"},{"line_number":434,"context_line":"what\u0027s happened along the time."},{"line_number":435,"context_line":""},{"line_number":436,"context_line":".. list-table:: Revisions"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_20231fa9","line":433,"range":{"start_line":433,"start_character":62,"end_line":433,"end_character":68},"updated":"2018-05-31 02:15:52.000000000","message":"maybe you lost the word understand","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9008510062429c0dbdf17afbedb158b5fc7a5d9f","unresolved":false,"context_lines":[{"line_number":430,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":431,"context_line":""},{"line_number":432,"context_line":"Optional section intended to be used each time the spec is updated to describe"},{"line_number":433,"context_line":"new design, API or any database schema updated. Useful to let reader"},{"line_number":434,"context_line":"what\u0027s happened along the time."},{"line_number":435,"context_line":""},{"line_number":436,"context_line":".. list-table:: Revisions"}],"source_content_type":"text/x-rst","patch_set":11,"id":"5f7c97a3_7b32dc60","line":433,"range":{"start_line":433,"start_character":62,"end_line":433,"end_character":68},"in_reply_to":"5f7c97a3_20231fa9","updated":"2018-06-05 04:29:47.000000000","message":"Done","commit_id":"47afed22d96f7f686290424337849cd82ffc4683"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"41f1fa64f834620fe57ad1ac66f1ccfdc5a59e53","unresolved":false,"context_lines":[{"line_number":157,"context_line":""},{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a"},{"line_number":159,"context_line":"    custom Resource Class (RC) for that type, of the form"},{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_\u003cdevice-type\u003e. E.g. CUSTOM_ACCELERATOR_GPU,"},{"line_number":161,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas"},{"line_number":162,"context_line":"    for different device types."},{"line_number":163,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_9aac72ef","line":160,"range":{"start_line":160,"start_character":43,"end_line":160,"end_character":65},"updated":"2018-06-06 07:37:41.000000000","message":"Since nova already have GPU resource class, can we use it directly? And should we remove the prefix \u0027ACCELERATOR\u0027 to match same style with GPU.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"2db4839d75d9beaa3df3102b6fdae73384c3ea67","unresolved":false,"context_lines":[{"line_number":157,"context_line":""},{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a"},{"line_number":159,"context_line":"    custom Resource Class (RC) for that type, of the form"},{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_\u003cdevice-type\u003e. E.g. CUSTOM_ACCELERATOR_GPU,"},{"line_number":161,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas"},{"line_number":162,"context_line":"    for different device types."},{"line_number":163,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_197050da","line":160,"range":{"start_line":160,"start_character":43,"end_line":160,"end_character":65},"in_reply_to":"5f7c97a3_9aac72ef","updated":"2018-06-07 10:37:41.000000000","message":"Yes, VGPU and VGPU_DISPLAY_HEAD are defined in https://github.com/openstack/nova/blob/d741f624c81baf89fc8b6b94a2bc20eb5355a818/nova/rc_fields.py. \n\nHowever, that seems to be for graphics? For compute offload, the terminology we have agreed is to use the generic term ACCELERATOR, in conversations, emails, etc.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"41f1fa64f834620fe57ad1ac66f1ccfdc5a59e53","unresolved":false,"context_lines":[{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a"},{"line_number":159,"context_line":"    custom Resource Class (RC) for that type, of the form"},{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_\u003cdevice-type\u003e. E.g. CUSTOM_ACCELERATOR_GPU,"},{"line_number":161,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas"},{"line_number":162,"context_line":"    for different device types."},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"  * Device-local memory is the memory available to the device alone,"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_7f8dc8d1","line":161,"range":{"start_line":161,"start_character":4,"end_line":161,"end_character":27},"updated":"2018-06-06 07:37:41.000000000","message":"This means a FPGA function?","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"2db4839d75d9beaa3df3102b6fdae73384c3ea67","unresolved":false,"context_lines":[{"line_number":158,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a"},{"line_number":159,"context_line":"    custom Resource Class (RC) for that type, of the form"},{"line_number":160,"context_line":"    CUSTOM_ACCELERATOR_\u003cdevice-type\u003e. E.g. CUSTOM_ACCELERATOR_GPU,"},{"line_number":161,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas"},{"line_number":162,"context_line":"    for different device types."},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"  * Device-local memory is the memory available to the device alone,"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_d9ddf8dd","line":161,"range":{"start_line":161,"start_character":4,"end_line":161,"end_character":27},"in_reply_to":"5f7c97a3_7f8dc8d1","updated":"2018-06-07 10:37:41.000000000","message":"We have defined an accelerator as a device, region or function -- a unit of offload that can be assigned to a VM via a PCI BDF. So, a CUSTOM_ACCELERATOR_FPGA is a generic virtual resource. In specific use cases, it could be a function or a region.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"41f1fa64f834620fe57ad1ac66f1ccfdc5a59e53","unresolved":false,"context_lines":[{"line_number":168,"context_line":"    CUSTOM_ACCELERATOR_MEMORY_DDR. A single PCIe board may have more"},{"line_number":169,"context_line":"    than one type of memory."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":172,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"},{"line_number":173,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":174,"context_line":"    of that device type\u0027s RC. This depends on nested RP support in"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_ff6f98f9","line":171,"range":{"start_line":171,"start_character":22,"end_line":171,"end_character":36},"updated":"2018-06-06 07:37:41.000000000","message":"This resource provider should provide the resource class said in line 161.\n\nSo if that resource class is function, then this should be region as resource provider.\n\nIf that resource is region, then this should be device as resource provider.\n\nI\u0027m thinking of the first case.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ff1679ff065ce46d1740ca9b3badcae219e43340","unresolved":false,"context_lines":[{"line_number":168,"context_line":"    CUSTOM_ACCELERATOR_MEMORY_DDR. A single PCIe board may have more"},{"line_number":169,"context_line":"    than one type of memory."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":172,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"},{"line_number":173,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":174,"context_line":"    of that device type\u0027s RC. This depends on nested RP support in"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_904e6ebf","line":171,"range":{"start_line":171,"start_character":22,"end_line":171,"end_character":36},"in_reply_to":"5f7c97a3_59a00843","updated":"2018-06-07 14:09:01.000000000","message":"*FPGA","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"2db4839d75d9beaa3df3102b6fdae73384c3ea67","unresolved":false,"context_lines":[{"line_number":168,"context_line":"    CUSTOM_ACCELERATOR_MEMORY_DDR. A single PCIe board may have more"},{"line_number":169,"context_line":"    than one type of memory."},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":172,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"},{"line_number":173,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":174,"context_line":"    of that device type\u0027s RC. This depends on nested RP support in"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_59a00843","line":171,"range":{"start_line":171,"start_character":22,"end_line":171,"end_character":36},"in_reply_to":"5f7c97a3_ff6f98f9","updated":"2018-06-07 10:37:41.000000000","message":"Both the F{GA device and the regions within them can be RPs. Even when there are regions, we would need to represent the device itself as a RP for some things. For example, if we have local memory that is not associated with regions but with device as a whole, that is an RC associated with device RP.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"41f1fa64f834620fe57ad1ac66f1ccfdc5a59e53","unresolved":false,"context_lines":[{"line_number":196,"context_line":"    be used in a flavor when a single driver/library in the instance"},{"line_number":197,"context_line":"    image can handle most or all of device types from a vendor."},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"  * Cyborg will associate a Device Family trait with each device as"},{"line_number":200,"context_line":"    needed, of the form CUSTOM_\u003cdevice-type\u003e_\u003cvemdor\u003e_\u003cfamily\u003e."},{"line_number":201,"context_line":"    E.g. CUSTOM_FPGA_INTEL_ARRIA10."},{"line_number":202,"context_line":"    This is not a product name, but the name of a device family, used to"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_9f5d444a","line":199,"range":{"start_line":199,"start_character":53,"end_line":199,"end_character":64},"updated":"2018-06-06 07:37:41.000000000","message":"It also means associate a device family trait with each resource provider which provides CUSTOM_ACCELERATOR_FPGA.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"2db4839d75d9beaa3df3102b6fdae73384c3ea67","unresolved":false,"context_lines":[{"line_number":196,"context_line":"    be used in a flavor when a single driver/library in the instance"},{"line_number":197,"context_line":"    image can handle most or all of device types from a vendor."},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"  * Cyborg will associate a Device Family trait with each device as"},{"line_number":200,"context_line":"    needed, of the form CUSTOM_\u003cdevice-type\u003e_\u003cvemdor\u003e_\u003cfamily\u003e."},{"line_number":201,"context_line":"    E.g. CUSTOM_FPGA_INTEL_ARRIA10."},{"line_number":202,"context_line":"    This is not a product name, but the name of a device family, used to"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_79bce46b","line":199,"range":{"start_line":199,"start_character":53,"end_line":199,"end_character":64},"in_reply_to":"5f7c97a3_9f5d444a","updated":"2018-06-07 10:37:41.000000000","message":"Yes. This is a point worth calling out explicitly. I will add a note.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"41f1fa64f834620fe57ad1ac66f1ccfdc5a59e53","unresolved":false,"context_lines":[{"line_number":207,"context_line":"    device family will need a new driver/library, it may make sense to"},{"line_number":208,"context_line":"    associate both these traits with the same device RP."},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":211,"context_line":"    (or with the FPGA itself if there is no PR support), of the form"},{"line_number":212,"context_line":"    CUSTOM_FPGA_\u003cvendor\u003e_REGION_\u003cuuid\u003e. E.g."},{"line_number":213,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_7f54e821","line":210,"range":{"start_line":210,"start_character":62,"end_line":210,"end_character":73},"updated":"2018-06-06 07:37:41.000000000","message":"same as here, now, we said region, so it confuses me at here, the resource provider is device or region? the resource class is region or function?","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"2db4839d75d9beaa3df3102b6fdae73384c3ea67","unresolved":false,"context_lines":[{"line_number":207,"context_line":"    device family will need a new driver/library, it may make sense to"},{"line_number":208,"context_line":"    associate both these traits with the same device RP."},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":211,"context_line":"    (or with the FPGA itself if there is no PR support), of the form"},{"line_number":212,"context_line":"    CUSTOM_FPGA_\u003cvendor\u003e_REGION_\u003cuuid\u003e. E.g."},{"line_number":213,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_f97474d2","line":210,"range":{"start_line":210,"start_character":62,"end_line":210,"end_character":73},"in_reply_to":"5f7c97a3_7f54e821","updated":"2018-06-07 10:37:41.000000000","message":"Both device and the regions within are RPs. The region itself is not a RC. Only CUSTOM_ACCELERATOR_FPGA is a RC. The function is a trait associated with the region RP.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"fe0ab01cbede264e94ad9251c16adca9669c8225","unresolved":false,"context_lines":[{"line_number":248,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_HPTS\u003d1``"},{"line_number":249,"context_line":"    | ``trait:CUSTOM_HPTS_ZTE\u003drequired``"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    NOTE: For FPGAs, the flavor should also include CUSTOM_PROGRAMMABLE trait."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * Example flavor for AFaaS Pre-programed:"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_6a2abf7e","line":251,"range":{"start_line":251,"start_character":4,"end_line":251,"end_character":78},"updated":"2018-06-07 12:43:28.000000000","message":"So, for the FPGA case, the flavor will be as below:\n\n| ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``\n| ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``\n| ``trait:CUSTOM_PROGRAMMABLE\u003drequired``\n\nSo compare to the AFaaS case, at line 261. The AfaaS case has another extra spec ``function:CUSTOM_FPGA_INTEL_FUNCTION_\u003cgzip-uuid\u003e\u003drequired``, but this won\u0027t be passed to the placement. So how the placement to distingush the AFaaS and DaaS case, since they have totally same input.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"c527c9217e6d3f3bc56a29760e46d9f273c9d685","unresolved":false,"context_lines":[{"line_number":248,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_HPTS\u003d1``"},{"line_number":249,"context_line":"    | ``trait:CUSTOM_HPTS_ZTE\u003drequired``"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    NOTE: For FPGAs, the flavor should also include CUSTOM_PROGRAMMABLE trait."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * Example flavor for AFaaS Pre-programed:"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_76d4ea47","line":251,"range":{"start_line":251,"start_character":4,"end_line":251,"end_character":78},"in_reply_to":"5f7c97a3_3b597b34","updated":"2018-06-07 14:40:40.000000000","message":"I mean for the DaaS case, the user wants a whole FPGA device, and program by himself in the VM.  \nBut the request:\n| ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``\n | ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``\n | ``trait:CUSTOM_PROGRAMMABLE\u003drequired``\n may return a programmable function by the placement.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"910bd0ffd46c8983a3c54f342070036874a3e7a3","unresolved":false,"context_lines":[{"line_number":248,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_HPTS\u003d1``"},{"line_number":249,"context_line":"    | ``trait:CUSTOM_HPTS_ZTE\u003drequired``"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    NOTE: For FPGAs, the flavor should also include CUSTOM_PROGRAMMABLE trait."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * Example flavor for AFaaS Pre-programed:"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_557f5aaa","line":251,"range":{"start_line":251,"start_character":4,"end_line":251,"end_character":78},"in_reply_to":"5f7c97a3_4008ff8f","updated":"2018-06-07 23:22:36.000000000","message":"I thought the different between Daas and AFaaS is the DaaS will passthrough the PF to the instance. The AFaaS on pass-through the VF to the instance. I may misunderstand this. If this is true, I don\u0027t understand cyborg how to distinguish this two from the same input.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"c8bb0c4927727d89d628eecb7213f1437365e7f3","unresolved":false,"context_lines":[{"line_number":248,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_HPTS\u003d1``"},{"line_number":249,"context_line":"    | ``trait:CUSTOM_HPTS_ZTE\u003drequired``"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    NOTE: For FPGAs, the flavor should also include CUSTOM_PROGRAMMABLE trait."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * Example flavor for AFaaS Pre-programed:"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_c9dbcf4c","line":251,"range":{"start_line":251,"start_character":4,"end_line":251,"end_character":78},"in_reply_to":"5f7c97a3_557f5aaa","updated":"2018-06-08 04:20:34.000000000","message":"Even with DaaS, we pass only VFs today. I have clarified that in the next update.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ff1679ff065ce46d1740ca9b3badcae219e43340","unresolved":false,"context_lines":[{"line_number":248,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_HPTS\u003d1``"},{"line_number":249,"context_line":"    | ``trait:CUSTOM_HPTS_ZTE\u003drequired``"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    NOTE: For FPGAs, the flavor should also include CUSTOM_PROGRAMMABLE trait."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * Example flavor for AFaaS Pre-programed:"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_3b597b34","line":251,"range":{"start_line":251,"start_character":4,"end_line":251,"end_character":78},"in_reply_to":"5f7c97a3_6a2abf7e","updated":"2018-06-07 14:09:01.000000000","message":"TL;DR Placement does not know the difference; Cyborg will handle it.\n\nPlacement will pick up all RPs that match the traits and resource class, but without regard to whether they have the function. So, Cyborg _may_ provide a weigher, as stated in line 312, to prioritize those with the function. Irrespective of the weigher, if the scheduler picks a RP without the function, Cyborg agent in the host will program the needed bitstream (see https://git.openstack.org/cgit/openstack/cyborg/tree/doc/specs/rocky/compute-node.rst?h\u003drefs/changes/98/566798/5#n284).","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"b316b3bb7bd68191bcc35166d1f6c6a82fa5dfd2","unresolved":false,"context_lines":[{"line_number":248,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_HPTS\u003d1``"},{"line_number":249,"context_line":"    | ``trait:CUSTOM_HPTS_ZTE\u003drequired``"},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    NOTE: For FPGAs, the flavor should also include CUSTOM_PROGRAMMABLE trait."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"  * Example flavor for AFaaS Pre-programed:"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"5f7c97a3_4008ff8f","line":251,"range":{"start_line":251,"start_character":4,"end_line":251,"end_character":78},"in_reply_to":"5f7c97a3_76d4ea47","updated":"2018-06-07 16:56:41.000000000","message":"The fact that a function may already be present is not relevant in this use case. The user asks for a device, and gets a device. The instance may then proceed to reprogram the device as it pleases. The previously existing function is ignored.\nFor placement, DaaS and AFaaS with programming look the same -- get all devices of a certain type. Only Cyborg distinguishes the two.","commit_id":"9c799b2562ad816063ca68ebf7e5bc717d8b2c1c"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators as"},{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates"},{"line_number":19,"context_line":"  resource inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg may provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources,"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_8eccea89","line":19,"range":{"start_line":19,"start_character":11,"end_line":19,"end_character":22},"updated":"2018-06-08 22:22:36.000000000","message":"just inventories, or also providers?","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* Representation and Discovery: Cyborg shall represent accelerators as"},{"line_number":18,"context_line":"  resources in Placement. When a device is discovered, Cyborg updates"},{"line_number":19,"context_line":"  resource inventories in Placement."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg may provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources,"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_989f4c4f","line":19,"range":{"start_line":19,"start_character":11,"end_line":19,"end_character":22},"in_reply_to":"5f7c97a3_8eccea89","updated":"2018-06-09 10:56:51.000000000","message":"Providers and traits as well, as explained further below. This section is an overview and doesn\u0027t get into all the details. Will update anyway.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg may provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources,"},{"line_number":23,"context_line":"  but is expected that Placement itself can handle most requirements."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":26,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_4ec25255","line":23,"range":{"start_line":23,"start_character":6,"end_line":23,"end_character":8},"updated":"2018-06-08 22:22:36.000000000","message":"it is","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":20,"context_line":""},{"line_number":21,"context_line":"* Instance placement/scheduling: Cyborg may provide a filter and/or weigher"},{"line_number":22,"context_line":"  that limit or prioritize hosts based on available accelerator resources,"},{"line_number":23,"context_line":"  but is expected that Placement itself can handle most requirements."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Attaching accelerators to instances. In the compute node, Cyborg shall"},{"line_number":26,"context_line":"  define a workflow based on interacting with Nova through a new os-acc"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_58be94ae","line":23,"range":{"start_line":23,"start_character":6,"end_line":23,"end_character":8},"in_reply_to":"5f7c97a3_4ec25255","updated":"2018-06-09 10:56:51.000000000","message":"Done","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":90,"context_line":"Though PCI Express is entrenched in the data center, some accelerators"},{"line_number":91,"context_line":"may be exposed to the host via some other protocol. Even with PCI, the"},{"line_number":92,"context_line":"connections between accelerator components and PCI functions"},{"line_number":93,"context_line":"may vary across devices. Accordingly, Cyborg should not represent"},{"line_number":94,"context_line":"accelerators as PCI functions."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"For instances that need accelerators, we need to define a way for Cyborg"},{"line_number":97,"context_line":"to be included seamlessly in the Nova scheduling workflow."}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_ee055e7b","line":94,"range":{"start_line":93,"start_character":38,"end_line":94,"end_character":30},"updated":"2018-06-08 22:22:36.000000000","message":"Whee!  ++","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":157,"context_line":"Representation"},{"line_number":158,"context_line":"--------------"},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a"},{"line_number":161,"context_line":"    custom Resource Class (RC) for that type, of the form"},{"line_number":162,"context_line":"    CUSTOM_ACCELERATOR_\u003cdevice-type\u003e. E.g. CUSTOM_ACCELERATOR_GPU,"},{"line_number":163,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas"},{"line_number":164,"context_line":"    for different device types."},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"  * Device-local memory is the memory available to the device alone,"},{"line_number":167,"context_line":"    usually in the form of DDR, QDR or High Bandwidth Memory in the"},{"line_number":168,"context_line":"    PCIe board along with the device. It can also be represented as an"},{"line_number":169,"context_line":"    RC of the form CUSTOM_ACCELERATOR_MEMORY_\u003cmemory-type\u003e. E.g."},{"line_number":170,"context_line":"    CUSTOM_ACCELERATOR_MEMORY_DDR. A single PCIe board may have more"},{"line_number":171,"context_line":"    than one type of memory."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":174,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_4e41f2a5","line":171,"range":{"start_line":160,"start_character":0,"end_line":171,"end_character":28},"updated":"2018-06-08 22:22:36.000000000","message":"Is there any reason not to make these standard resource classes?","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":157,"context_line":"Representation"},{"line_number":158,"context_line":"--------------"},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"  * Cyborg will represent a generic accelerator for a device type as a"},{"line_number":161,"context_line":"    custom Resource Class (RC) for that type, of the form"},{"line_number":162,"context_line":"    CUSTOM_ACCELERATOR_\u003cdevice-type\u003e. E.g. CUSTOM_ACCELERATOR_GPU,"},{"line_number":163,"context_line":"    CUSTOM_ACCELERATOR_FPGA, etc. This helps in defining separate quotas"},{"line_number":164,"context_line":"    for different device types."},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"  * Device-local memory is the memory available to the device alone,"},{"line_number":167,"context_line":"    usually in the form of DDR, QDR or High Bandwidth Memory in the"},{"line_number":168,"context_line":"    PCIe board along with the device. It can also be represented as an"},{"line_number":169,"context_line":"    RC of the form CUSTOM_ACCELERATOR_MEMORY_\u003cmemory-type\u003e. E.g."},{"line_number":170,"context_line":"    CUSTOM_ACCELERATOR_MEMORY_DDR. A single PCIe board may have more"},{"line_number":171,"context_line":"    than one type of memory."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":174,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_38a87876","line":171,"range":{"start_line":160,"start_character":0,"end_line":171,"end_character":28},"in_reply_to":"5f7c97a3_4e41f2a5","updated":"2018-06-09 10:56:51.000000000","message":"The accelerator classes can be standardized. I believe Alex has taken that up. There is much variation among memory types, so we may want to get enough experience and usage before standardizing them.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":173,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":174,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"},{"line_number":175,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":176,"context_line":"    of that device type\u0027s RC. This depends on nested RP support in"},{"line_number":177,"context_line":"    Nova [#nRP]_."},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"       * For FPGAs, both the device and the regions within it will be"},{"line_number":180,"context_line":"         represented as RPs. This allows the hierarchy within an FPGA"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_ce6d0204","line":177,"range":{"start_line":176,"start_character":35,"end_line":177,"end_character":17},"updated":"2018-06-08 22:22:36.000000000","message":"There are actually two aspects of nrp that you care about.\n\nOne is the ability to request allocation candidates against a provider tree.  That\u0027s the last thing we need to do to close the loop on nrp functionality in nova.  And it\u0027s getting close to being done [1].\n\nThe other is how we handle upgrades, which is still in the early spec stages [2].\n\nThe key point to note is that [2] is only a prerequisite if cyborg needs to MOVE inventories that ALREADY EXIST.  For example, if cyborg plans to take over the management of VGPUs that\u0027s already implemented for libvirt/xen.  But AFAIK nobody has FPGA or HPTS resources inventoried on their compute node providers at this time, so you can start doing that, tree-wise, as soon as [1] lands.\n\n[1] https://review.openstack.org/#/q/topic:bp/nested-resource-providers-allocation-candidates+status:open\n[2] https://review.openstack.org/#/c/572583/","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":173,"context_line":"  * In addition, each device/region is represented as a Resource Provider"},{"line_number":174,"context_line":"    (RP). This enables traits to be applied to it and other RPs/RCs to"},{"line_number":175,"context_line":"    be contained within it. So, a device RP provides one or more instances"},{"line_number":176,"context_line":"    of that device type\u0027s RC. This depends on nested RP support in"},{"line_number":177,"context_line":"    Nova [#nRP]_."},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"       * For FPGAs, both the device and the regions within it will be"},{"line_number":180,"context_line":"         represented as RPs. This allows the hierarchy within an FPGA"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_f8b760d7","line":177,"range":{"start_line":176,"start_character":35,"end_line":177,"end_character":17},"in_reply_to":"5f7c97a3_ce6d0204","updated":"2018-06-09 10:56:51.000000000","message":"From #6 in the list in Jay Pipes\u0027 email [3], it seems that enhancing virt drivers to handle NRPs is gated on the Nova upgrade considerations. Will investigate if virt driver change is a hard prerequisite for Cyborg.\n\n[3] http://lists.openstack.org/pipermail/openstack-dev/2018-June/131157.html","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":178,"context_line":""},{"line_number":179,"context_line":"       * For FPGAs, both the device and the regions within it will be"},{"line_number":180,"context_line":"         represented as RPs. This allows the hierarchy within an FPGA"},{"line_number":181,"context_line":"         to be naturally modelled as an RP hierarchy."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"       * Using Nested RPs is the preferred way. But, until Nova"},{"line_number":184,"context_line":"         supports nested RPs, Cyborg shall associate the"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_ce0e628b","line":181,"updated":"2018-06-08 22:22:36.000000000","message":"Specifically, you\u0027re going to model the device as an (empty) RP under the compute node, and each region as a child RP thereof?","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":178,"context_line":""},{"line_number":179,"context_line":"       * For FPGAs, both the device and the regions within it will be"},{"line_number":180,"context_line":"         represented as RPs. This allows the hierarchy within an FPGA"},{"line_number":181,"context_line":"         to be naturally modelled as an RP hierarchy."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"       * Using Nested RPs is the preferred way. But, until Nova"},{"line_number":184,"context_line":"         supports nested RPs, Cyborg shall associate the"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_7876f0da","line":181,"in_reply_to":"5f7c97a3_ce0e628b","updated":"2018-06-09 10:56:51.000000000","message":"Yes, except it may not be empty if local memory is part of the device as a whole (as opposed to specific regions within the device).","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":185,"context_line":"         RCs and traits (described below) with the compute node RPs. This"},{"line_number":186,"context_line":"         requires that all devices on a single host must share the same"},{"line_number":187,"context_line":"         traits. If nested RP support becomes usable after Rocky release,"},{"line_number":188,"context_line":"         the operator needs to handle the upgrade as below:"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"         * Terminate all instances using accelerators."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"         * Remove all Cyborg traits and inventory on all compute node RPs,"},{"line_number":193,"context_line":"           perhaps by running a script."},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"         * Perform the Cyborg upgrade. Post-upgrade, the new agent/driver(s)"},{"line_number":196,"context_line":"           will create RPs for the devices and publish the traits"},{"line_number":197,"context_line":"           and inventory."},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"  * Cyborg will associate a Device Type trait with each device, of the"},{"line_number":200,"context_line":"    form CUSTOM_\u003cdevice-type\u003e-\u003cvendor\u003e. E.g. CUSTOM_GPU_AMD or"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_4e1a52c6","line":197,"range":{"start_line":188,"start_character":31,"end_line":197,"end_character":25},"updated":"2018-06-08 22:22:36.000000000","message":"Not sure about this.  See bp reshape-provider-tree.  We haven\u0027t really considered what the upgrade flow needs to look like when nova and cyborg (and potentially cinder, neutron, etc.) are \"sharing\" responsibility for providers/inventories/allocations in a single tree.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":185,"context_line":"         RCs and traits (described below) with the compute node RPs. This"},{"line_number":186,"context_line":"         requires that all devices on a single host must share the same"},{"line_number":187,"context_line":"         traits. If nested RP support becomes usable after Rocky release,"},{"line_number":188,"context_line":"         the operator needs to handle the upgrade as below:"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"         * Terminate all instances using accelerators."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"         * Remove all Cyborg traits and inventory on all compute node RPs,"},{"line_number":193,"context_line":"           perhaps by running a script."},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"         * Perform the Cyborg upgrade. Post-upgrade, the new agent/driver(s)"},{"line_number":196,"context_line":"           will create RPs for the devices and publish the traits"},{"line_number":197,"context_line":"           and inventory."},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"  * Cyborg will associate a Device Type trait with each device, of the"},{"line_number":200,"context_line":"    form CUSTOM_\u003cdevice-type\u003e-\u003cvendor\u003e. E.g. CUSTOM_GPU_AMD or"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_1819fcd5","line":197,"range":{"start_line":188,"start_character":31,"end_line":197,"end_character":25},"in_reply_to":"5f7c97a3_4e1a52c6","updated":"2018-06-09 10:56:51.000000000","message":"Since Cyborg installations will be fresh ones, rather than legacy ones, and Cyborg RPs will be distinct from existing RPs, we can decouple cleaning of Cyborg RPs from other considerations, right? \n\nIOW, in a Rocky deployment, we clean out Cyborg RPs before upgrading. So, this should be independent of Nova upgrade considerations.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":207,"context_line":"         RPs which are children of the device RPs as well."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"  * Cyborg will associate a Device Family trait with each device as"},{"line_number":210,"context_line":"    needed, of the form CUSTOM_\u003cdevice-type\u003e_\u003cvemdor\u003e_\u003cfamily\u003e."},{"line_number":211,"context_line":"    E.g. CUSTOM_FPGA_INTEL_ARRIA10."},{"line_number":212,"context_line":"    This is not a product name, but the name of a device family, used to"},{"line_number":213,"context_line":"    match software in the instance image with the device family. This is"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_ce142250","line":210,"range":{"start_line":210,"start_character":46,"end_line":210,"end_character":52},"updated":"2018-06-08 22:22:36.000000000","message":"vendor","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":207,"context_line":"         RPs which are children of the device RPs as well."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"  * Cyborg will associate a Device Family trait with each device as"},{"line_number":210,"context_line":"    needed, of the form CUSTOM_\u003cdevice-type\u003e_\u003cvemdor\u003e_\u003cfamily\u003e."},{"line_number":211,"context_line":"    E.g. CUSTOM_FPGA_INTEL_ARRIA10."},{"line_number":212,"context_line":"    This is not a product name, but the name of a device family, used to"},{"line_number":213,"context_line":"    match software in the instance image with the device family. This is"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_f8dc4019","line":210,"range":{"start_line":210,"start_character":46,"end_line":210,"end_character":52},"in_reply_to":"5f7c97a3_ce142250","updated":"2018-06-09 10:56:51.000000000","message":"Done","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":215,"context_line":"    a flavor when there are different drivers/libraries for different"},{"line_number":216,"context_line":"    device families. Since it may be tough to forecast whether a new"},{"line_number":217,"context_line":"    device family will need a new driver/library, it may make sense to"},{"line_number":218,"context_line":"    associate both these traits with the same device RP."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":221,"context_line":"    (or with the FPGA itself if there is no PR support), of the form"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_2e2a7690","line":218,"updated":"2018-06-08 22:22:36.000000000","message":"Seems like there was a ML thread on this, where Jay asked you to do this with a massive hash table of vendor/family to capability traits.  I\u0027m not weighing in, just noting that this may not be a done deal.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":215,"context_line":"    a flavor when there are different drivers/libraries for different"},{"line_number":216,"context_line":"    device families. Since it may be tough to forecast whether a new"},{"line_number":217,"context_line":"    device family will need a new driver/library, it may make sense to"},{"line_number":218,"context_line":"    associate both these traits with the same device RP."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":221,"context_line":"    (or with the FPGA itself if there is no PR support), of the form"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_f805207b","line":218,"in_reply_to":"5f7c97a3_2e2a7690","updated":"2018-06-09 10:56:51.000000000","message":"The primary need here is driver selection. Other capabilities are represented as specific traits listed in this doc. \n   Can you list what other capabilities are of interest, apart from those listed here as traits?","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":218,"context_line":"    associate both these traits with the same device RP."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":221,"context_line":"    (or with the FPGA itself if there is no PR support), of the form"},{"line_number":222,"context_line":"    CUSTOM_FPGA_\u003cvendor\u003e_REGION_\u003cuuid\u003e. E.g."},{"line_number":223,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":224,"context_line":"    Service with FPGAs."}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_4e27b2c8","line":221,"range":{"start_line":221,"start_character":44,"end_line":221,"end_character":46},"updated":"2018-06-08 22:22:36.000000000","message":"whazzat?","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":218,"context_line":"    associate both these traits with the same device RP."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"  * For FPGAs, Cyborg will associate a region type trait with each region"},{"line_number":221,"context_line":"    (or with the FPGA itself if there is no PR support), of the form"},{"line_number":222,"context_line":"    CUSTOM_FPGA_\u003cvendor\u003e_REGION_\u003cuuid\u003e. E.g."},{"line_number":223,"context_line":"    CUSTOM_FPGA_INTEL_REGION_\u003cuuid\u003e. This is needed for Device as a"},{"line_number":224,"context_line":"    Service with FPGAs."}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_98f66c8f","line":221,"range":{"start_line":221,"start_character":44,"end_line":221,"end_character":46},"in_reply_to":"5f7c97a3_4e27b2c8","updated":"2018-06-09 10:56:51.000000000","message":"Partial Reconfiguration :). Will spell out in full.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":225,"context_line":""},{"line_number":226,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":227,"context_line":"    when the region gets programmed, of the form"},{"line_number":228,"context_line":"    CUSTOM_FPGA_\u003cvendor\u003e_\u003cuuid\u003e. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":229,"context_line":"    This is needed for AFaaS use case. This is updated when Cyborg"},{"line_number":230,"context_line":"    reprograms a region as part of AFaaS request."},{"line_number":231,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_ae0f6637","line":228,"range":{"start_line":228,"start_character":24,"end_line":228,"end_character":25},"updated":"2018-06-08 22:22:36.000000000","message":"..._FUNCTION_...","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":225,"context_line":""},{"line_number":226,"context_line":"  * For FPGAs, Cyborg may associate a function type trait with a region"},{"line_number":227,"context_line":"    when the region gets programmed, of the form"},{"line_number":228,"context_line":"    CUSTOM_FPGA_\u003cvendor\u003e_\u003cuuid\u003e. E.g. CUSTOM_FPGA_INTEL_\u003cgzip-uuid\u003e."},{"line_number":229,"context_line":"    This is needed for AFaaS use case. This is updated when Cyborg"},{"line_number":230,"context_line":"    reprograms a region as part of AFaaS request."},{"line_number":231,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_58ec54ba","line":228,"range":{"start_line":228,"start_character":24,"end_line":228,"end_character":25},"in_reply_to":"5f7c97a3_ae0f6637","updated":"2018-06-09 10:56:51.000000000","message":"Fixed. Also, moved _FUNCTION_ and _REGION_ before \u003cvendor\u003e to keep the generic parts first and specific parts later.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":246,"context_line":"  * The modeling in Placement represents generic virtual accelerators as"},{"line_number":247,"context_line":"    resource classes, and devices/regions as RPs. This is PCI-agnostic."},{"line_number":248,"context_line":"    However, many FPGA implementations use PCI Express in general, and"},{"line_number":249,"context_line":"    SR-IOV in particular. In those cases, it is expected that Cyborg will"},{"line_number":250,"context_line":"    pass PCI VFs to instances via PCI Passthrough, and retain the PCI PF"},{"line_number":251,"context_line":"    in the host for management."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"Flavors"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_6e0dce3c","line":250,"range":{"start_line":249,"start_character":63,"end_line":250,"end_character":49},"updated":"2018-06-08 22:22:36.000000000","message":"Why?  Let the os-acc plugin handle it.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":246,"context_line":"  * The modeling in Placement represents generic virtual accelerators as"},{"line_number":247,"context_line":"    resource classes, and devices/regions as RPs. This is PCI-agnostic."},{"line_number":248,"context_line":"    However, many FPGA implementations use PCI Express in general, and"},{"line_number":249,"context_line":"    SR-IOV in particular. In those cases, it is expected that Cyborg will"},{"line_number":250,"context_line":"    pass PCI VFs to instances via PCI Passthrough, and retain the PCI PF"},{"line_number":251,"context_line":"    in the host for management."},{"line_number":252,"context_line":""},{"line_number":253,"context_line":"Flavors"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_3844f8b7","line":250,"range":{"start_line":249,"start_character":63,"end_line":250,"end_character":49},"in_reply_to":"5f7c97a3_6e0dce3c","updated":"2018-06-09 10:56:51.000000000","message":"I am counting os-acc backend (implementation of the plug/unplug APIs) as part of the Cyborg implementation. Besides, that is implementation-specific, right? The implementation may involve consulting Cyborg DB, and could be in the agent possibly.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":292,"context_line":"    refers to a function by name instead of UUID for ease of use:"},{"line_number":293,"context_line":""},{"line_number":294,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":295,"context_line":"    | ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``"},{"line_number":296,"context_line":"    | ``trait:CUSTOM_PROGRAMMABLE\u003drequired``"},{"line_number":297,"context_line":"    | ``function_name:\u003cstring\u003e\u003drequired``"},{"line_number":298,"context_line":"      (Not interpreted by Nova.)"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_31317bee","line":295,"range":{"start_line":295,"start_character":40,"end_line":295,"end_character":41},"updated":"2018-06-08 22:22:36.000000000","message":"nix","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":292,"context_line":"    refers to a function by name instead of UUID for ease of use:"},{"line_number":293,"context_line":""},{"line_number":294,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":295,"context_line":"    | ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``"},{"line_number":296,"context_line":"    | ``trait:CUSTOM_PROGRAMMABLE\u003drequired``"},{"line_number":297,"context_line":"    | ``function_name:\u003cstring\u003e\u003drequired``"},{"line_number":298,"context_line":"      (Not interpreted by Nova.)"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_5841b4a5","line":295,"range":{"start_line":295,"start_character":40,"end_line":295,"end_character":41},"in_reply_to":"5f7c97a3_31317bee","updated":"2018-06-09 10:56:51.000000000","message":"Done","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"82b43b4b76845fb4a4aa9c2e574b31d935ce0c31","unresolved":false,"context_lines":[{"line_number":294,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":295,"context_line":"    | ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``"},{"line_number":296,"context_line":"    | ``trait:CUSTOM_PROGRAMMABLE\u003drequired``"},{"line_number":297,"context_line":"    | ``function_name:\u003cstring\u003e\u003drequired``"},{"line_number":298,"context_line":"      (Not interpreted by Nova.)"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"    * NOTE: This assumes the operator has configured the function name"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_c46bb6ee","line":297,"range":{"start_line":297,"start_character":8,"end_line":297,"end_character":21},"updated":"2018-06-08 06:35:23.000000000","message":"s/function_name/function/","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":294,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":295,"context_line":"    | ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``"},{"line_number":296,"context_line":"    | ``trait:CUSTOM_PROGRAMMABLE\u003drequired``"},{"line_number":297,"context_line":"    | ``function_name:\u003cstring\u003e\u003drequired``"},{"line_number":298,"context_line":"      (Not interpreted by Nova.)"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"    * NOTE: This assumes the operator has configured the function name"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_b809e884","line":297,"range":{"start_line":297,"start_character":8,"end_line":297,"end_character":21},"in_reply_to":"5f7c97a3_71405336","updated":"2018-06-09 10:56:51.000000000","message":"Yes, we would use numbered request groups (Granular Resource Request syntax) _if_ there are accelerators of different types. If the flavor needs N identical accelerators, we don\u0027t necessarily need that.\n\nThe namespacing for functions is addressed below, as you noted too.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":294,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":295,"context_line":"    | ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``"},{"line_number":296,"context_line":"    | ``trait:CUSTOM_PROGRAMMABLE\u003drequired``"},{"line_number":297,"context_line":"    | ``function_name:\u003cstring\u003e\u003drequired``"},{"line_number":298,"context_line":"      (Not interpreted by Nova.)"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"    * NOTE: This assumes the operator has configured the function name"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_71405336","line":297,"range":{"start_line":297,"start_character":8,"end_line":297,"end_character":21},"in_reply_to":"5f7c97a3_a2cf8527","updated":"2018-06-08 22:22:36.000000000","message":"We need some way to disambiguate here.  For the resources and traits, I\u0027ve been assuming that, if you were doing more than one at a time, you would put each into its own numbered request group.  Are you going to support function_name\u003cN\u003e namespacing in a similar fashion?  (If that\u0027s the plan, I\u0027m not sure how I feel about that :)","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9251001827ef5921153ff291d924c36ba7aca71b","unresolved":false,"context_lines":[{"line_number":294,"context_line":"    | ``resources:CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":295,"context_line":"    | ``trait:CUSTOM_FPGA_INTEL_ARRIA10\u003d required``"},{"line_number":296,"context_line":"    | ``trait:CUSTOM_PROGRAMMABLE\u003drequired``"},{"line_number":297,"context_line":"    | ``function_name:\u003cstring\u003e\u003drequired``"},{"line_number":298,"context_line":"      (Not interpreted by Nova.)"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"    * NOTE: This assumes the operator has configured the function name"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_a2cf8527","line":297,"range":{"start_line":297,"start_character":8,"end_line":297,"end_character":21},"in_reply_to":"5f7c97a3_c46bb6ee","updated":"2018-06-08 12:21:38.000000000","message":"To distinguish from the previous case (line 281) where the function UUID is given, this is explicitly called \u0027function_name\u0027.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":5754,"name":"Alex Xu","email":"hejie.xu@intel.com","username":"xuhj"},"change_message_id":"82b43b4b76845fb4a4aa9c2e574b31d935ce0c31","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"    | ``resourcesN: CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":313,"context_line":"    | ``traitsN: CUSTOM_FPGA_INTEL_ARRIA10\u003drequired``"},{"line_number":314,"context_line":"    | ``othersN: function:CUSTOM_FPGA_INTEL_FUNCTION_\u003cgzip-uuid\u003e\u003drequired``"},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"Scheduling workflow"},{"line_number":317,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_64a50247","line":314,"range":{"start_line":314,"start_character":8,"end_line":314,"end_character":73},"updated":"2018-06-08 06:35:23.000000000","message":"This is strange. \n\nI guess it should be:\nfunctionN:[traits]\u003drequired","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"9251001827ef5921153ff291d924c36ba7aca71b","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"    | ``resourcesN: CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":313,"context_line":"    | ``traitsN: CUSTOM_FPGA_INTEL_ARRIA10\u003drequired``"},{"line_number":314,"context_line":"    | ``othersN: function:CUSTOM_FPGA_INTEL_FUNCTION_\u003cgzip-uuid\u003e\u003drequired``"},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"Scheduling workflow"},{"line_number":317,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_a29d450b","line":314,"range":{"start_line":314,"start_character":8,"end_line":314,"end_character":73},"in_reply_to":"5f7c97a3_64a50247","updated":"2018-06-08 12:21:38.000000000","message":"I am open to any suggestions on syntax. Since there could be keywords other than \u0027functions\u0027 that Placment won\u0027t interpret, I added \u0027othersN\u0027. What do other Nova folks think?","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"    | ``resourcesN: CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":313,"context_line":"    | ``traitsN: CUSTOM_FPGA_INTEL_ARRIA10\u003drequired``"},{"line_number":314,"context_line":"    | ``othersN: function:CUSTOM_FPGA_INTEL_FUNCTION_\u003cgzip-uuid\u003e\u003drequired``"},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"Scheduling workflow"},{"line_number":317,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_98242c0f","line":314,"range":{"start_line":314,"start_character":8,"end_line":314,"end_character":73},"in_reply_to":"5f7c97a3_91664fed","updated":"2018-06-09 10:56:51.000000000","message":"I do _not_ expect Nova/placement to handle othersN, as noted in line 310 above. But, as we build experience and get feedback, maybe we can work together to do this in a standard way. :)","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"    | ``resourcesN: CUSTOM_ACCELERATOR_FPGA\u003d1``"},{"line_number":313,"context_line":"    | ``traitsN: CUSTOM_FPGA_INTEL_ARRIA10\u003drequired``"},{"line_number":314,"context_line":"    | ``othersN: function:CUSTOM_FPGA_INTEL_FUNCTION_\u003cgzip-uuid\u003e\u003drequired``"},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"Scheduling workflow"},{"line_number":317,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_91664fed","line":314,"range":{"start_line":314,"start_character":8,"end_line":314,"end_character":73},"in_reply_to":"5f7c97a3_a29d450b","updated":"2018-06-08 22:22:36.000000000","message":"Oh, this starts to address the above.\n\nIf you\u0027re going to piggyback on the numbered request group syntax, I\u0027m not sure why \u0027othersN\u0027 is better than just \u0027functionN\u0027.\n\nThat still makes me a bit uncomfortable, because you\u0027re sort of implying that your new thingy is getting processed by placement when it\u0027s not.  But I think I\u0027ll get over it, because in reality, it\u0027s the admin/operator/user who need to create/understand these things, and they don\u0027t (and shouldn\u0027t) have any concept of what\u0027s handled by which component.\n\n^ /me talks self into this being a reasonable thing to do.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":327,"context_line":"  * FPGA-specific: For AFaaS orchestration-programmed use case, Placement"},{"line_number":328,"context_line":"    will return matching devices but they may not have the requested"},{"line_number":329,"context_line":"    function. So, Cyborg may provide a weigher which checks the"},{"line_number":330,"context_line":"    allocation candidates to see which ones have the required function trait,"},{"line_number":331,"context_line":"    and ranks them higher. This requires no change to Cyborg DB."},{"line_number":332,"context_line":""},{"line_number":333,"context_line":"  * The request_spec goes to compute node (ignoring Cells for now)."}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_315f3b10","line":330,"range":{"start_line":330,"start_character":53,"end_line":330,"end_character":76},"updated":"2018-06-08 22:22:36.000000000","message":"Well, if you specified it as a required trait, placement will omit the ones that don\u0027t have it.  Are you talking about when the flavor/RequestSpec specified one of these functionN thingies?","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":327,"context_line":"  * FPGA-specific: For AFaaS orchestration-programmed use case, Placement"},{"line_number":328,"context_line":"    will return matching devices but they may not have the requested"},{"line_number":329,"context_line":"    function. So, Cyborg may provide a weigher which checks the"},{"line_number":330,"context_line":"    allocation candidates to see which ones have the required function trait,"},{"line_number":331,"context_line":"    and ranks them higher. This requires no change to Cyborg DB."},{"line_number":332,"context_line":""},{"line_number":333,"context_line":"  * The request_spec goes to compute node (ignoring Cells for now)."}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_187edce3","line":330,"range":{"start_line":330,"start_character":53,"end_line":330,"end_character":76},"in_reply_to":"5f7c97a3_315f3b10","updated":"2018-06-09 10:56:51.000000000","message":"Yes. I will clarify.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":347,"context_line":"  * Cyborg should associate RPs/RCs and PFs/VFs with Deployables in its"},{"line_number":348,"context_line":"    internal DB. It can use such mappings associating the requested resource"},{"line_number":349,"context_line":"    (device/function) with some attach handle that can be used to"},{"line_number":350,"context_line":"    attach the resource to an instance (such as a PCI function)."},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"NOTE : This flow is PCI-agnostic: no PCI whitelists involved."},{"line_number":353,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_31e2db31","line":350,"updated":"2018-06-08 22:22:36.000000000","message":"I expected to learn about how this attach flow works.  After cyborg identifies the \"deployables\" in its database, does it call out to some registered, platform-specific \"plugin\" gizmo to do the actual work?  Or does the flow return back to nova, where the virt driver is invoked to do that attach?  Or what?\n\nThe other big hole is on the discovery side.  At some point cyborg needs to call some platform-specific code to discover devices, including their hierarchy on the system, traits, inventories, etc.  Where and how does that happen?\n\nPerhaps these are topics of some of the other specs on my reading list.  This is the first one I\u0027m getting to.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":347,"context_line":"  * Cyborg should associate RPs/RCs and PFs/VFs with Deployables in its"},{"line_number":348,"context_line":"    internal DB. It can use such mappings associating the requested resource"},{"line_number":349,"context_line":"    (device/function) with some attach handle that can be used to"},{"line_number":350,"context_line":"    attach the resource to an instance (such as a PCI function)."},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"NOTE : This flow is PCI-agnostic: no PCI whitelists involved."},{"line_number":353,"context_line":""}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_3872b813","line":350,"in_reply_to":"5f7c97a3_31e2db31","updated":"2018-06-09 10:56:51.000000000","message":"That flow is listed in the compute node spec https://review.openstack.org/#/c/566798/. You will probably find that to be a bit high-level too :). \n\nShort answer: Cyborg returns PCI BDFs to Nova compute for Rocky, so that the virt driver can do its job. Per your earlier feedback, I tried not make that PCI-centric, but I didn\u0027t have enough clarity for the alternatives. We could revise this post-Rocky.","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":429,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":430,"context_line":""},{"line_number":431,"context_line":"* `Nested Resource Providers"},{"line_number":432,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/nested-resource-providers.html\u003e`_"},{"line_number":433,"context_line":""},{"line_number":434,"context_line":"* `Nova Granular Requests"},{"line_number":435,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/granular-resource-requests.html\u003e`_"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_11655fb5","line":432,"updated":"2018-06-08 22:22:36.000000000","message":"This spec is okay as a reference, but your actual dependency is on [1].  And you\u0027ll be wanting to include a dependency on [2] for the upgrade issues.\n\n[1] http://specs.openstack.org/openstack/nova-specs/specs/rocky/approved/nested-resource-providers-allocation-candidates.html\n[2] https://review.openstack.org/572583","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":429,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":430,"context_line":""},{"line_number":431,"context_line":"* `Nested Resource Providers"},{"line_number":432,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/nested-resource-providers.html\u003e`_"},{"line_number":433,"context_line":""},{"line_number":434,"context_line":"* `Nova Granular Requests"},{"line_number":435,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/granular-resource-requests.html\u003e`_"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_5848d4b1","line":432,"in_reply_to":"5f7c97a3_11655fb5","updated":"2018-06-09 10:56:51.000000000","message":"Done","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3a6218d0ffc8ef56f386b9a26ae35fb6d9af398b","unresolved":false,"context_lines":[{"line_number":432,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/nested-resource-providers.html\u003e`_"},{"line_number":433,"context_line":""},{"line_number":434,"context_line":"* `Nova Granular Requests"},{"line_number":435,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/granular-resource-requests.html\u003e`_"},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"Testing"},{"line_number":438,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_d141270c","line":435,"range":{"start_line":435,"start_character":58,"end_line":435,"end_character":64},"updated":"2018-06-08 22:22:36.000000000","message":"There\u0027s a rocky update to this spec","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"},{"author":{"_account_id":21672,"name":"Sundar Nadathur","email":"sundar.nadathur@intel.com","username":"nsundar"},"change_message_id":"ba973735f324e82a48a7eb939eb1c4584dd33a1b","unresolved":false,"context_lines":[{"line_number":432,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/nested-resource-providers.html\u003e`_"},{"line_number":433,"context_line":""},{"line_number":434,"context_line":"* `Nova Granular Requests"},{"line_number":435,"context_line":"  \u003chttps://specs.openstack.org/openstack/nova-specs/specs/queens/approved/granular-resource-requests.html\u003e`_"},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"Testing"},{"line_number":438,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":17,"id":"5f7c97a3_f8c68016","line":435,"range":{"start_line":435,"start_character":58,"end_line":435,"end_character":64},"in_reply_to":"5f7c97a3_d141270c","updated":"2018-06-09 10:56:51.000000000","message":"Done","commit_id":"ec9b4a88a61241dfc814b1126a1eb4494924bea7"}]}