)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"37a43379_2c7a6a38","updated":"2026-06-04 13:37:55.000000000","message":"over all this si well written and researched\n\ni think this need more work however and we need to have a wider discussion about if nova want to continue to add new images_types or not in general\n\nwe have 2 propped for this release after a long time of not adding any and we know there is technical debt in this area.\n\n-1 because i think there are some open question about version and production ready ness of spdk\n\naslo i think we need to better integarte this with the rbd_direct_download config options and tighten the live migration story","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"}],"specs/2026.2/approved/spdk-based-storage-backend.rst":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":93,"context_line":".. code::"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":" [libvirt]"},{"line_number":96,"context_line":" images_type\u003dspdk_lvol"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"Additionally several options will allow configuring SPDK related options."},{"line_number":99,"context_line":"These include:"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bc608b88_27d52ed1","line":96,"updated":"2026-06-04 13:37:55.000000000","message":"this is an option\n\nthe other option would be to declare this out of scope fo nova and delegater it to cybrog\n\nhttps://github.com/openstack/cyborg/tree/master/cyborg/accelerator/drivers/spdk\n\nthe spdk drifver in cyborg is not really functional today but i have been talking about the idea of boot form cyborg storage on and off with some folks.\n\nso wether its a root disk or a addtiaonl data disk i wonder if it really should be nativly in nova at this point?","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":93,"context_line":".. code::"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":" [libvirt]"},{"line_number":96,"context_line":" images_type\u003dspdk_lvol"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"Additionally several options will allow configuring SPDK related options."},{"line_number":99,"context_line":"These include:"}],"source_content_type":"text/x-rst","patch_set":2,"id":"cb693f5a_78fd3dd8","line":96,"in_reply_to":"bc608b88_27d52ed1","updated":"2026-06-12 10:41:41.000000000","message":"It might be possible to support this through cyborg. However as I mentioned in the mailing list discussion I am not very familiar in that project\nso I would prefer to implement it via nova. If that would not be possible I will try to think how this might be introduced to cyborg.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":119,"context_line":" spdk_rpc_socket\u003d/var/tmp/spdk.sock"},{"line_number":120,"context_line":" spdk_vhost_socket_dir\u003d/var/lib/libvirt/qemu/"},{"line_number":121,"context_line":" spdk_rpc_client_timeout\u003d1000"},{"line_number":122,"context_line":" enable_spdk_rbd_direct_operations\u003dFalse"},{"line_number":123,"context_line":" spdk_rpc_validate_version\u003dTrue"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"dedc01e3_bd808a63","line":122,"updated":"2026-06-04 13:37:55.000000000","message":"the ceph case is defnietly an interesting one. although that would need\nimages_type\u003drbd","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":119,"context_line":" spdk_rpc_socket\u003d/var/tmp/spdk.sock"},{"line_number":120,"context_line":" spdk_vhost_socket_dir\u003d/var/lib/libvirt/qemu/"},{"line_number":121,"context_line":" spdk_rpc_client_timeout\u003d1000"},{"line_number":122,"context_line":" enable_spdk_rbd_direct_operations\u003dFalse"},{"line_number":123,"context_line":" spdk_rpc_validate_version\u003dTrue"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"950ce45d_8f679157","line":122,"in_reply_to":"dedc01e3_bd808a63","updated":"2026-06-12 10:41:41.000000000","message":"As specified in list above, this option enables later mentioned direct snapshot and clone mechanisms that allow for direct pulling and pushing of images from SPDK to CEPH image store.\nThey do not require `images_type\u003drbd`, only that CEPH configuration options are set in nova.conf.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":129,"context_line":"Same as with other storage backends, scheduling and requesting a build of an"},{"line_number":130,"context_line":"instance on a compute with SPDK-based storage backend will result in that"},{"line_number":131,"context_line":"instance using such storage backend for its root and ephemeral disks"},{"line_number":132,"context_line":"(if ``--boot-from-volume`` is not used)."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"One restriction for instances with SPDK-based storage is that they require"},{"line_number":135,"context_line":"shared memory for QEMU guest to share virtqueues and I/O buffers with SPDK"}],"source_content_type":"text/x-rst","patch_set":2,"id":"49bf3e80_a601d46e","line":132,"updated":"2026-06-04 13:37:55.000000000","message":"so to be compatiable if --boot-form-volume is used the ephmeral disik shoudl still use spdk and in all casles if swap is enabeld it should also use spdk for the swap disk.\nonly the root disk gets backed by a cidner volume when you boot form volume we use hte hsot image_type driver for any addtional nova provisioned storage.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":129,"context_line":"Same as with other storage backends, scheduling and requesting a build of an"},{"line_number":130,"context_line":"instance on a compute with SPDK-based storage backend will result in that"},{"line_number":131,"context_line":"instance using such storage backend for its root and ephemeral disks"},{"line_number":132,"context_line":"(if ``--boot-from-volume`` is not used)."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"One restriction for instances with SPDK-based storage is that they require"},{"line_number":135,"context_line":"shared memory for QEMU guest to share virtqueues and I/O buffers with SPDK"}],"source_content_type":"text/x-rst","patch_set":2,"id":"f44b0b85_475b1aa3","line":132,"in_reply_to":"49bf3e80_a601d46e","updated":"2026-06-12 10:41:41.000000000","message":"You are correct. Fixed.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":134,"context_line":"One restriction for instances with SPDK-based storage is that they require"},{"line_number":135,"context_line":"shared memory for QEMU guest to share virtqueues and I/O buffers with SPDK"},{"line_number":136,"context_line":"process. This can be achieved by either configuring the instance with"},{"line_number":137,"context_line":"``hw:mem_page_size\u003dlarge`` extra spec or"},{"line_number":138,"context_line":"``trait:COMPUTE_MEM_BACKING_FILE\u003drequired`` and ensuring compute with"},{"line_number":139,"context_line":"SPDK-based storage either provides huge pages for instances or supports file"},{"line_number":140,"context_line":"backed memory."},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"22291d43_41bf8605","line":138,"range":{"start_line":137,"start_character":0,"end_line":138,"end_character":43},"updated":"2026-06-04 13:37:55.000000000","message":"yes until we add native supprot for memfd\n\nhttps://review.opendev.org/c/openstack/nova-specs/+/990259\n\nthe actuall requiremetns for dpdk and spdk are that the memory is mmap shared\nand there is a open file descriptor that point to the memory that qemu can pass via the vhost-user socket to the relevent backend\n\nso hugepages or file backed memory are the only ways to do that in nova today.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":144,"context_line":" mechanism. In this apporach  memory backing is configured via file descriptors"},{"line_number":145,"context_line":" using Linux\u0027s `memfd` mechanism. This is a preferred approach, however it is"},{"line_number":146,"context_line":" not yet available in nova. See references for proposed spec that introduces"},{"line_number":147,"context_line":" memfd support to nova."},{"line_number":148,"context_line":""},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"When building the instance on the compute nova first creates the disk by:"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9e64e786_6820100c","line":147,"updated":"2026-06-04 13:37:55.000000000","message":"ah you noted that here cool","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":149,"context_line":""},{"line_number":150,"context_line":"When building the instance on the compute nova first creates the disk by:"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"1. Finding appropriate *lvstore* for the *lvol* (one with enough disk space)"},{"line_number":153,"context_line":"   using ``bdev_lvol_get_lvstores`` command. *Lvstores* are SPDK abstraction"},{"line_number":154,"context_line":"   that allows partitioning SPDK block device (called *bdev*, this can be NVMe"},{"line_number":155,"context_line":"   disk, local file, chunk of RAM, another *lvol* and many other things)"},{"line_number":156,"context_line":"   into *lvols* that can be interacted with in many ways."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"2. Creating an *lvol* in chosen *lvstore* with size configured in flavor."},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"3. Importing initial data. This can be either done by pulling the image from"}],"source_content_type":"text/x-rst","patch_set":2,"id":"853df443_4093c316","line":157,"range":{"start_line":152,"start_character":2,"end_line":157,"end_character":1},"updated":"2026-06-04 13:37:55.000000000","message":"this sounds problematic\n\nunless spdk can compesl a lvol form multipel store howe are we going to model this in placement ?\n\nwe report a singel inventory of disk gb for a host.\nso unless we are goign to use nested resouce provider to report each lvstore spereately \nwho are  we oging to model the min/max unit values for each lvstore and properly ensure we dont have a fragmenation problem?\n\nusing multipel nested resouce providers is a valid anwer but if that is the only answer then that need to be capture in the spec.\n\nat some point we do want to be able to move to having more then one images_type backend enabled at a time so evenrtully we would mvoe to modelign storage per backend anyway but i dont belive you have disucssed that in the sepc.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":149,"context_line":""},{"line_number":150,"context_line":"When building the instance on the compute nova first creates the disk by:"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"1. Finding appropriate *lvstore* for the *lvol* (one with enough disk space)"},{"line_number":153,"context_line":"   using ``bdev_lvol_get_lvstores`` command. *Lvstores* are SPDK abstraction"},{"line_number":154,"context_line":"   that allows partitioning SPDK block device (called *bdev*, this can be NVMe"},{"line_number":155,"context_line":"   disk, local file, chunk of RAM, another *lvol* and many other things)"},{"line_number":156,"context_line":"   into *lvols* that can be interacted with in many ways."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"2. Creating an *lvol* in chosen *lvstore* with size configured in flavor."},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"3. Importing initial data. This can be either done by pulling the image from"}],"source_content_type":"text/x-rst","patch_set":2,"id":"054b7807_806cbfd7","line":157,"range":{"start_line":152,"start_character":2,"end_line":157,"end_character":1},"in_reply_to":"853df443_4093c316","updated":"2026-06-12 10:41:41.000000000","message":"You are correct. This might cause some issues. Maybe it is better then to only use a single lvstore specified in the configuration.\nIf user wants to use multiple storage devices for SPDK and have all of their storage available in nova they would have to use \nRAID virtual bdev module (https://spdk.io/doc/bdev.html#bdev_ug_raid) and create lvstore on top of it.\nI combined points 1\u00262 to expect a single lvstore specified in the config.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":168,"context_line":""},{"line_number":169,"context_line":"5. During libvirt domain XML construction: adding to devices section a disk"},{"line_number":170,"context_line":"   with type *vhostuser* and ``source.path`` pointing to the socket exposed in"},{"line_number":171,"context_line":"   the previous step."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"6. Creating the instance by calling libvirt."},{"line_number":174,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"c13e24bc_0ffd9296","line":171,"updated":"2026-06-04 13:37:55.000000000","message":"so vhost-user-blk was added a long time ago as was vhost-user-scsi in qemu and i belvie libvirt.\n\nbut can you find the  exact libvirt and qemu version that added tehm so we know that they are below our min versions otherwise you woudl need to do a min libvirt version check on start up\n\ni belvie these were added in the libvirt 7 era or older and our min libvirt is currenly \n\nMIN_LIBVIRT_VERSION \u003d (8, 0, 0)\nMIN_QEMU_VERSION \u003d (6, 2, 0)\n\nso you should be fine.\n\nwe are also overdue moving the min values to \n\nNEXT_MIN_LIBVIRT_VERSION \u003d (10, 0, 0)\nNEXT_MIN_QEMU_VERSION \u003d (8, 2, 2)\n\nso we likel shoudl have doen that last cycle but we defintly can do it this cycle\nif needed\n\n\ni assmue we will supprot both vhost-user-blk and vhost-user-scsi if we were to proceed with this and just reuse the dising hw_disk_bus\u003dvirtio for block and hw_disk_bus\u003dscsi image properteis so this si transaprent to the end user the same way dpdk networking is transparenet when we use the orginal vhost-user-net\n\n\nalso will there be a depency on \n\nhttps://www.qemu.org/docs/master/system/devices/virtio/vhost-user-contrib.html#vhost-user-scsi-scsi-controller\nor\nhttps://www.qemu.org/docs/master/tools/qemu-storage-daemon.html#storage-daemon\n\nwe delibatlly do not use any of libvirt storage deamon functionaltiy today so this would be a new requireemtn for packages when libivrt is packages as a module deamon if we need the storage deamon\n\nif we dont and we are jsut spdk directly you will need to detail how that will work in the spec\n\nwill https://spdk.io/doc/sma.html be used?\n\nalso nova suprot disk qos via the quota: flavor extra specs and spdk appare sto also supprot qos based on https://spdk.io/doc/sma.html is that in socpe of this spec?\nhttps://github.com/openstack/nova/blob/master/nova/api/validation/extra_specs/quota.py#L149-L164","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":177,"context_line":""},{"line_number":178,"context_line":"Nova will consider all *lvstores* configured in SPDK when creating *lvols*"},{"line_number":179,"context_line":"for disks, so space from all *lvstores* will be reported as available when SPDK"},{"line_number":180,"context_line":"storage backend is used."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"SPDK\u0027s `bdev_lvol_get_lvstores` RPC call provides among other things"},{"line_number":183,"context_line":"information on total and available space on given *lvstore*. Sum of totals for"}],"source_content_type":"text/x-rst","patch_set":2,"id":"34145241_b1f9c43e","line":180,"updated":"2026-06-04 13:37:55.000000000","message":"again see my point before. this is only valid if nova can create an lvol that will span teh stores otherwise we cant jsut aggreate the stroage and present a single view as we will have case where fragmenation prevent us form booting a vm that shoudl otherwize fit.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":177,"context_line":""},{"line_number":178,"context_line":"Nova will consider all *lvstores* configured in SPDK when creating *lvols*"},{"line_number":179,"context_line":"for disks, so space from all *lvstores* will be reported as available when SPDK"},{"line_number":180,"context_line":"storage backend is used."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"SPDK\u0027s `bdev_lvol_get_lvstores` RPC call provides among other things"},{"line_number":183,"context_line":"information on total and available space on given *lvstore*. Sum of totals for"}],"source_content_type":"text/x-rst","patch_set":2,"id":"acda1972_c1c90790","line":180,"in_reply_to":"34145241_b1f9c43e","updated":"2026-06-12 10:41:41.000000000","message":"As you say, there does not seem to be a need to bump min libvirt or qemu version for this feature.\n\nRegarding vhost-user-* support: On our environments we only used vhost-user-blk and virtio bus for instances.\nHowever when introducing it here vhost-user-scsi support can be also added.\n\nThere does not seem to be a need for either of those daemons. In setup proposed by this spec SPDK is always the vhost server\nand QEMU process is always a client. Additional vhost servers (which those are from what I understand) are unnecessary.\nIf there is any need to save data to disk or load from it then NBD is opened and data is accessed through it.\n\nThe spec only specifies usage of SPDK RPC commands. SMA is not used in any way.\nI didn\u0027t write the spec with qos in mind but it seems support for it can be added as\nthere is a rpc_bdev_set_qos_limit (https://spdk.io/doc/jsonrpc.html#rpc_bdev_set_qos_limit) RPC available.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":204,"context_line":""},{"line_number":205,"context_line":".. code:: bash"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":" dd if\u003d\u003csrc_nbd_dev\u003e bs\u003d4M | ssh -o BatchMode\u003dyes \u003cdest_host\u003e dd of\u003d\u003cdst_nbd_dev\u003e bs\u003d4M"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"The *NBDs* that will be used on source and destination hosts have form"},{"line_number":210,"context_line":"`/dev/nbd\u003cx\u003e` where x is a number between 0 and 20."}],"source_content_type":"text/x-rst","patch_set":2,"id":"27709d84_c128fbf6","line":207,"updated":"2026-06-04 13:37:55.000000000","message":"if im not mistaken we do that copy vai a configurbal driver today in the cold migration path.\n\ni may be wrong about tha tbut wwe may need to condier fi we need to extend of build on that.\n\nhttps://github.com/openstack/nova/blob/master/nova/virt/libvirt/volume/remotefs.py#L172-L316","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":204,"context_line":""},{"line_number":205,"context_line":".. code:: bash"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":" dd if\u003d\u003csrc_nbd_dev\u003e bs\u003d4M | ssh -o BatchMode\u003dyes \u003cdest_host\u003e dd of\u003d\u003cdst_nbd_dev\u003e bs\u003d4M"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"The *NBDs* that will be used on source and destination hosts have form"},{"line_number":210,"context_line":"`/dev/nbd\u003cx\u003e` where x is a number between 0 and 20."}],"source_content_type":"text/x-rst","patch_set":2,"id":"79bbf54b_2f801508","line":207,"in_reply_to":"27709d84_c128fbf6","updated":"2026-06-12 10:41:41.000000000","message":"We can put utility that does this transfer there. But we will still need a separate code path\nfor SPDK because:\n1. We do not between copy file paths corresponding to disks but between NBDs for those disks opened on source and destination compute\n2. We do not create a file at the destination but copy data to existing file/block device.\n3. For same host resize we do not run `cp` as the nova code does for filesystem based migrations, but we execute specific SPDK RPC commands.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":210,"context_line":"`/dev/nbd\u003cx\u003e` where x is a number between 0 and 20."},{"line_number":211,"context_line":"When `nbd_start_disk` call is made, *lvol* is exposed by SPDK on first"},{"line_number":212,"context_line":"available *NBD* under `/dev` (on host) and path to that NBD is returned as"},{"line_number":213,"context_line":"response to that call."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"This approach requires additional configuration for nova-ssh container (if"},{"line_number":216,"context_line":"kolla-based deployment is used). Since `dd of\u003d...` command is executed in the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"b72228cd_f3c13acc","line":213,"updated":"2026-06-04 13:37:55.000000000","message":"and to be clear this is neede because qemu-image for exampel does not have the ablity to conenct to the spdk lvol directly so we need the nbd server to rpovde a block deve that we can use form the host right?","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":210,"context_line":"`/dev/nbd\u003cx\u003e` where x is a number between 0 and 20."},{"line_number":211,"context_line":"When `nbd_start_disk` call is made, *lvol* is exposed by SPDK on first"},{"line_number":212,"context_line":"available *NBD* under `/dev` (on host) and path to that NBD is returned as"},{"line_number":213,"context_line":"response to that call."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"This approach requires additional configuration for nova-ssh container (if"},{"line_number":216,"context_line":"kolla-based deployment is used). Since `dd of\u003d...` command is executed in the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"458248b0_4ddb8ea1","line":213,"in_reply_to":"b72228cd_f3c13acc","updated":"2026-06-12 10:41:41.000000000","message":"Yes this is correct. We use these to access lvol contents from the host.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":211,"context_line":"When `nbd_start_disk` call is made, *lvol* is exposed by SPDK on first"},{"line_number":212,"context_line":"available *NBD* under `/dev` (on host) and path to that NBD is returned as"},{"line_number":213,"context_line":"response to that call."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"This approach requires additional configuration for nova-ssh container (if"},{"line_number":216,"context_line":"kolla-based deployment is used). Since `dd of\u003d...` command is executed in the"},{"line_number":217,"context_line":"context of that container, it requires `/dev` directory from source host to be"},{"line_number":218,"context_line":"mounted."},{"line_number":219,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"b76da549_8d499624","line":216,"range":{"start_line":214,"start_character":1,"end_line":216,"end_character":31},"updated":"2026-06-04 13:37:55.000000000","message":"that not really relevent since were alwasy require ssh acccess for the libvirt driver even when useing the rbd images_type backend to copy things like the console logs for live migration so ssh conenctivity is a requirement already today\n\nweither its in a contanier or not is not relevnet to the nova spec","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":211,"context_line":"When `nbd_start_disk` call is made, *lvol* is exposed by SPDK on first"},{"line_number":212,"context_line":"available *NBD* under `/dev` (on host) and path to that NBD is returned as"},{"line_number":213,"context_line":"response to that call."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"This approach requires additional configuration for nova-ssh container (if"},{"line_number":216,"context_line":"kolla-based deployment is used). Since `dd of\u003d...` command is executed in the"},{"line_number":217,"context_line":"context of that container, it requires `/dev` directory from source host to be"},{"line_number":218,"context_line":"mounted."},{"line_number":219,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"cbafe9ce_6d272fcc","line":216,"range":{"start_line":214,"start_character":1,"end_line":216,"end_character":31},"in_reply_to":"b76da549_8d499624","updated":"2026-06-12 10:41:41.000000000","message":"Understood. I have removed this paragraph from the spec. Please note however, that this was more about `/dev` directory access than about `ssh` connectivity. Not sure if that is relevant.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":230,"context_line":""},{"line_number":231,"context_line":"For resizes where host does not change, there is no need to transfer disks"},{"line_number":232,"context_line":"using *NBD*. All operations can be done entirely within SPDK by making"},{"line_number":233,"context_line":"appropriate RPC calls."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"It can be done in a following way:"},{"line_number":236,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"41b83b32_fa8a1db2","line":233,"updated":"2026-06-04 13:37:55.000000000","message":"hum perhaps but we may want to keep a common path but we would have an opertunity to optimize yes.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":234,"context_line":""},{"line_number":235,"context_line":"It can be done in a following way:"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"* Call `bdev_lvol_snapshot` - to create snapshot in case revert is necessary."},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"* Call `bdev_lvol_decouple_parent` - to detach snapshot from the *lvol* (This"},{"line_number":240,"context_line":"  is necessary if *lvol* is to be resized)."}],"source_content_type":"text/x-rst","patch_set":2,"id":"949e8c92_cd98bdd4","line":237,"updated":"2026-06-04 13:37:55.000000000","message":"so we would need to do thei sin the cross host resize case too.\n\n\ni would move the snap showt and revert decrtip up to the previous secion\n\nthe only thing that change here is instead fo doing the replcaiton vai dd we can clone the snapshot and extend it.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":234,"context_line":""},{"line_number":235,"context_line":"It can be done in a following way:"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"* Call `bdev_lvol_snapshot` - to create snapshot in case revert is necessary."},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"* Call `bdev_lvol_decouple_parent` - to detach snapshot from the *lvol* (This"},{"line_number":240,"context_line":"  is necessary if *lvol* is to be resized)."}],"source_content_type":"text/x-rst","patch_set":2,"id":"56d177e7_33a9ff8b","line":237,"in_reply_to":"949e8c92_cd98bdd4","updated":"2026-06-12 10:41:41.000000000","message":"For cross host resize there is no need to do a snapshot. lvol on source host will not be modified over the course of migration/resize\nsince instance on source host is powered off before any data is transferred and is only powered on in case of the revert.\nThe data there will remain intact even without a snapshot. We will only work on new lvol on destination host.\n\nFor same host resize we need a snapshot since we will be modifying the lvol that instance was using.\nWe work on that lvol instead of cloning it and working on a clone because we want to have predicable lvol names\nand the appropriate one is already taken by the original lvol. Using a different name would create unneccessary complexity in code.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":277,"context_line":""},{"line_number":278,"context_line":"Libvirt supports live migration of virtual machines with *vhostuser* disks out"},{"line_number":279,"context_line":"of the box. It requires that *vhost* socket (which the virtual machine uses to"},{"line_number":280,"context_line":"access the *lvol*) exists on destination host under the same path as on the"},{"line_number":281,"context_line":"source host when initiating live migration. To achieve this we ensure that"},{"line_number":282,"context_line":"*vhost* socket paths created by Nova through RPC calls to SPDK always have"},{"line_number":283,"context_line":"following form:"},{"line_number":284,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"932ec987_688b3174","line":281,"range":{"start_line":280,"start_character":46,"end_line":281,"end_character":11},"updated":"2026-06-04 13:37:55.000000000","message":"why?\n\nyou are propsoing making the path configurabl in this spec which mean you cannot assume it will be the same on teh souce and dest host\n\ninstead you need to exten dthe libvirt live mgiration data object with the destination paths in pre live migration\n\nfor dpdk we pass the path in the vif_details_json blob\nhttps://github.com/openstack/nova/blob/master/nova/objects/migrate_data.py#L55\nwhich is derived form the destion host port bindings.\n\nwe do similar things for cpu pinning or other host specific config driven paths.\n\nThen we update the paths in the migraiton domain xml\n\nhttps://github.com/openstack/nova/blob/278c6e305c3da085fd1c1e338e95ce4d63631272/nova/virt/libvirt/migration.py#L133-L152\nhttps://github.com/openstack/nova/blob/278c6e305c3da085fd1c1e338e95ce4d63631272/nova/virt/libvirt/migration.py#L499-L563\n\nwe either need to remove  spdk_vhost_socket_dir as a config option or pass back the value and update the xml with the relevent path.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":277,"context_line":""},{"line_number":278,"context_line":"Libvirt supports live migration of virtual machines with *vhostuser* disks out"},{"line_number":279,"context_line":"of the box. It requires that *vhost* socket (which the virtual machine uses to"},{"line_number":280,"context_line":"access the *lvol*) exists on destination host under the same path as on the"},{"line_number":281,"context_line":"source host when initiating live migration. To achieve this we ensure that"},{"line_number":282,"context_line":"*vhost* socket paths created by Nova through RPC calls to SPDK always have"},{"line_number":283,"context_line":"following form:"},{"line_number":284,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a5108a4_eac5b3a1","line":281,"range":{"start_line":280,"start_character":46,"end_line":281,"end_character":11},"in_reply_to":"932ec987_688b3174","updated":"2026-06-12 10:41:41.000000000","message":"I specified this as a requirement to avoid modifying the XML, but since modifying it is a possibility, I have rewritten this paragraph.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":287,"context_line":" \u003cvhost_base_path_from_conf\u003e/vhost.\u003cinstance_uuid\u003e_\u003cdisk_name\u003e"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":""},{"line_number":290,"context_line":"It is also necessary to perform SPDK snapshot of *lvol* on source compute and"},{"line_number":291,"context_line":"pre-fill *lvol* on destination compute with that data (by copying it over in a"},{"line_number":292,"context_line":"way similar to one used for cold-migrate) before initiating live migration in"},{"line_number":293,"context_line":"libvirt. Otherwise the instance fails to boot."},{"line_number":294,"context_line":""},{"line_number":295,"context_line":"Instead of using system_metadata to transfer necessary information between"},{"line_number":296,"context_line":"computes, LibvirtLiveMigrateData is expanded with new fields."}],"source_content_type":"text/x-rst","patch_set":2,"id":"272a4c05_e8dc8c7c","line":293,"range":{"start_line":290,"start_character":0,"end_line":293,"end_character":46},"updated":"2026-06-04 13:37:55.000000000","message":"that sounds like a qemu bug and not somethign nova shoudl have to do\n\nnative supprot for this would actully mean that the data copy is done by qemu/libvirt as it si for intel persetietn memory or just normal disk\n\ndo we actully need to copy the data twice or is the actul requrieemt just that the target lvol is create in the desintaion host?","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":287,"context_line":" \u003cvhost_base_path_from_conf\u003e/vhost.\u003cinstance_uuid\u003e_\u003cdisk_name\u003e"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":""},{"line_number":290,"context_line":"It is also necessary to perform SPDK snapshot of *lvol* on source compute and"},{"line_number":291,"context_line":"pre-fill *lvol* on destination compute with that data (by copying it over in a"},{"line_number":292,"context_line":"way similar to one used for cold-migrate) before initiating live migration in"},{"line_number":293,"context_line":"libvirt. Otherwise the instance fails to boot."},{"line_number":294,"context_line":""},{"line_number":295,"context_line":"Instead of using system_metadata to transfer necessary information between"},{"line_number":296,"context_line":"computes, LibvirtLiveMigrateData is expanded with new fields."}],"source_content_type":"text/x-rst","patch_set":2,"id":"0e819d75_96fb58e6","line":293,"range":{"start_line":290,"start_character":0,"end_line":293,"end_character":46},"in_reply_to":"272a4c05_e8dc8c7c","updated":"2026-06-12 10:41:41.000000000","message":"On our environments when I tried running live migration with just precreating the lvol and without copying over data first,\nit ended up with instance failing to boot. \nI assume you\u0027d prefer that instead of introducing the process mentioned in this spec, I looked into this further and potentially filed a bug \nwith QEMU if that is indeed a bug there?","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":293,"context_line":"libvirt. Otherwise the instance fails to boot."},{"line_number":294,"context_line":""},{"line_number":295,"context_line":"Instead of using system_metadata to transfer necessary information between"},{"line_number":296,"context_line":"computes, LibvirtLiveMigrateData is expanded with new fields."},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"Data import/export"},{"line_number":299,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fb74b0c2_30329b9a","line":296,"updated":"2026-06-04 13:37:55.000000000","message":"we do not sue teh system_metadata to transfer info for mighration that used to store info that does nto change over the lifetime of the guest outside of action liek rebuild or resize like image properties","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":293,"context_line":"libvirt. Otherwise the instance fails to boot."},{"line_number":294,"context_line":""},{"line_number":295,"context_line":"Instead of using system_metadata to transfer necessary information between"},{"line_number":296,"context_line":"computes, LibvirtLiveMigrateData is expanded with new fields."},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"Data import/export"},{"line_number":299,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"e60190ca_e75599f8","line":296,"in_reply_to":"fb74b0c2_30329b9a","updated":"2026-06-12 10:41:41.000000000","message":"I updated this paragraph to not mention instance.system_metadata.\n\nI initially wrote it like this to contrast this approach with how things are done for resize/cold-migration\nwhere instance.system_metadata is used for transferring information between computes.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":304,"context_line":"1. Standard approach is to download the image into host\u0027s file system from"},{"line_number":305,"context_line":"   Glance using HTTP, open up an *NBD* exposing empty *lvol* in host\u0027s"},{"line_number":306,"context_line":"   file system and copy over contents of downloaded image to the *NBD*."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"2. Another approach is to reuse (with some modifications) clone mechanism"},{"line_number":309,"context_line":"   used in RBD storage backend: collect image location on RBD cluster from"},{"line_number":310,"context_line":"   Glance, open up an *NBD* exposing empty *lvol* in host\u0027s file system and"},{"line_number":311,"context_line":"   perform an `rbd export` operation on image in RBD cluster with its output"},{"line_number":312,"context_line":"   piped to the *NBD*."},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"The second approach is much faster, does not require data to be transferred"},{"line_number":315,"context_line":"through the control node where Glance is running and does not require large"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bb23648a_d88fbb31","line":312,"range":{"start_line":307,"start_character":1,"end_line":312,"end_character":22},"updated":"2026-06-04 13:37:55.000000000","message":"we have a feature called direct download that allow use to pull disk directionl form RBD locally into the image cache, if we supprot approch 2 we whoudl reuse as much fo the rbd direct download path as possible.\n\nby the way i think we currently do that via qemu-image passign the rbd voluem as the souce and we shoudl be able ot use the nbd server as the dest.\n\ni do think we should also confimr if qemu-image has native supprot for spdk\nit can use the nbd protocl if im not mistake so if we can use the nbd server to export spdk lvols we shoudl eb able to use them directly in qemu-img as weell with the correct nbd protocol path.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":304,"context_line":"1. Standard approach is to download the image into host\u0027s file system from"},{"line_number":305,"context_line":"   Glance using HTTP, open up an *NBD* exposing empty *lvol* in host\u0027s"},{"line_number":306,"context_line":"   file system and copy over contents of downloaded image to the *NBD*."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"2. Another approach is to reuse (with some modifications) clone mechanism"},{"line_number":309,"context_line":"   used in RBD storage backend: collect image location on RBD cluster from"},{"line_number":310,"context_line":"   Glance, open up an *NBD* exposing empty *lvol* in host\u0027s file system and"},{"line_number":311,"context_line":"   perform an `rbd export` operation on image in RBD cluster with its output"},{"line_number":312,"context_line":"   piped to the *NBD*."},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"The second approach is much faster, does not require data to be transferred"},{"line_number":315,"context_line":"through the control node where Glance is running and does not require large"}],"source_content_type":"text/x-rst","patch_set":2,"id":"95e5a7eb_a5c0d0ff","line":312,"range":{"start_line":307,"start_character":1,"end_line":312,"end_character":22},"in_reply_to":"bb23648a_d88fbb31","updated":"2026-06-12 10:41:41.000000000","message":"Unfortunately I don\u0027t think we can reuse that path. It works by downloading the image from RBD into a specified path (creating a file there). The `qemu-img` conversion are done later by transfering file from this path to the final destination. This approach requires space on host\u0027s non-SPDK storage to keep a temporary copy\nof the image. This is the part we are trying to avoid. With approach specified in the spec we can pull the image directly from rbd into lvol on on SPDK without interacting with host\u0027s non-SPDK storage.\n\nI have updated the spec to account for image conversion. I have not considered it initially since on our environments we required raw images for SPDK-based instances.\n\nThere should be no issue with using `qemu-img` for this purpose. As you say we can pass rbd volume path directly to it as input. And throughout the entire spec\nwe only use NBD through kernel\u0027s nbd module - so block devices exposed under `/dev/nbd*`. We do not use NBD protocol directly. There should be no issue with \npassing `/dev/nbd*` as output device for `qemu-img convert` as long as we pass `-n` to make sure creation of `/dev/nbd*` is not attempted.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":337,"context_line":"Again the second approach is much faster and avoids transferring data through"},{"line_number":338,"context_line":"Glance, but has the same downsides as the faster approach for pulling."},{"line_number":339,"context_line":""},{"line_number":340,"context_line":"The flag that controls approach used for pulling will also control approach"},{"line_number":341,"context_line":"used for pushing images."},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"Alternatives"},{"line_number":344,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"41277b04_346090e8","line":341,"range":{"start_line":340,"start_character":0,"end_line":341,"end_character":24},"updated":"2026-06-04 13:37:55.000000000","message":"you are referint to `enable_spdk_rbd_direct_operations`","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":337,"context_line":"Again the second approach is much faster and avoids transferring data through"},{"line_number":338,"context_line":"Glance, but has the same downsides as the faster approach for pulling."},{"line_number":339,"context_line":""},{"line_number":340,"context_line":"The flag that controls approach used for pulling will also control approach"},{"line_number":341,"context_line":"used for pushing images."},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"Alternatives"},{"line_number":344,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"b73d7604_b3f94e3a","line":341,"range":{"start_line":340,"start_character":0,"end_line":341,"end_character":24},"in_reply_to":"41277b04_346090e8","updated":"2026-06-12 10:41:41.000000000","message":"Correct. I have updated the spec to mention this option by name here.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":345,"context_line":""},{"line_number":346,"context_line":"An alternative approach to transferring *lvol* contents between computes during"},{"line_number":347,"context_line":"a cold-migration is to use NVMe-oF capabilities of SPDK. They allow exposing"},{"line_number":348,"context_line":"SPDK *bdevs* over a network."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"At startup, nova would call SPDK to enable exposing *lvols* using NVMe-OF:"},{"line_number":351,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"f86a8708_07ce5bfb","line":348,"updated":"2026-06-04 13:37:55.000000000","message":"we coudl evovle to this\none fo the reasons we use ssh for this by default however is because fo the encyption it provides\n\nwe genreally try to ensure you can use tls for all data tansport so unless the nvme-of layer aslo suprpot that im not sure that woudl be ideal\n\nwith that said we can alwasy do this later so i agree with not doing this in v1","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":400,"context_line":"  every pair of SPDK host, this requires specifying on each host with SPDK"},{"line_number":401,"context_line":"  a list of every other host with SPDK by making a `nvmf_subsystem_add_host`"},{"line_number":402,"context_line":"  for each of them. If new hosts with SPDK were added to the environment,"},{"line_number":403,"context_line":"  additional calls would need to be made on existing hosts."},{"line_number":404,"context_line":""},{"line_number":405,"context_line":"Data model impact"},{"line_number":406,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"b75ab6e5_5c69c2cf","line":403,"updated":"2026-06-04 13:37:55.000000000","message":"yep this is extra complexity but we can alwasy condier this in the future if the performace justifies it.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":408,"context_line":"New fields will be added to `LibvirtLiveMigrateData`:"},{"line_number":409,"context_line":""},{"line_number":410,"context_line":"* `dest_ip` - a string field containing the IP address of destination node,"},{"line_number":411,"context_line":"  used when transferring *lvol* contents over ssh to destination host."},{"line_number":412,"context_line":""},{"line_number":413,"context_line":"* `dest_lvol_nbd_paths` - a *dict* of strings field, that contains *NBD* paths"},{"line_number":414,"context_line":"  opened up on destination host that are ready to accept *lvol* contents"}],"source_content_type":"text/x-rst","patch_set":2,"id":"a187549b_76842440","line":411,"updated":"2026-06-04 13:37:55.000000000","message":"this is not reuiqed we already know this and have a way to scp/ssh between the host.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":408,"context_line":"New fields will be added to `LibvirtLiveMigrateData`:"},{"line_number":409,"context_line":""},{"line_number":410,"context_line":"* `dest_ip` - a string field containing the IP address of destination node,"},{"line_number":411,"context_line":"  used when transferring *lvol* contents over ssh to destination host."},{"line_number":412,"context_line":""},{"line_number":413,"context_line":"* `dest_lvol_nbd_paths` - a *dict* of strings field, that contains *NBD* paths"},{"line_number":414,"context_line":"  opened up on destination host that are ready to accept *lvol* contents"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5aa995e2_cebf4312","line":411,"in_reply_to":"a187549b_76842440","updated":"2026-06-12 10:41:41.000000000","message":"I rechecked nova-objects and you are correct. Field migratation.dest_host of LiveMigrateData servers the same function.\nI removed dest_ip from the spec.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":414,"context_line":"  opened up on destination host that are ready to accept *lvol* contents"},{"line_number":415,"context_line":"  transferred in from source host. It will be keyed by *vhost* socket of given"},{"line_number":416,"context_line":"  disk. For non-SPDK disks there won\u0027t be a corresponding entry in this *dict*."},{"line_number":417,"context_line":""},{"line_number":418,"context_line":"This will necessitate a version bump of non-persisted `LibvirtLiveMigrateData`"},{"line_number":419,"context_line":"object."},{"line_number":420,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"d5ba5d8b_4b7cdd14","line":417,"updated":"2026-06-04 13:37:55.000000000","message":"yes this would be required\nbut we als need the vhost-user socket paths for the desitnation.\n\nso the corect th8ing to do is to intoduce a new class like the vif object and have a list of those not a dict of string\n\nso we can use the host vhost user path or the disk alias as the key with the dest rbd path and dest socket as values in the obejct","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":414,"context_line":"  opened up on destination host that are ready to accept *lvol* contents"},{"line_number":415,"context_line":"  transferred in from source host. It will be keyed by *vhost* socket of given"},{"line_number":416,"context_line":"  disk. For non-SPDK disks there won\u0027t be a corresponding entry in this *dict*."},{"line_number":417,"context_line":""},{"line_number":418,"context_line":"This will necessitate a version bump of non-persisted `LibvirtLiveMigrateData`"},{"line_number":419,"context_line":"object."},{"line_number":420,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"843fc5b5_f7cee5ab","line":417,"in_reply_to":"d5ba5d8b_4b7cdd14","updated":"2026-06-12 10:41:41.000000000","message":"I have updated the spec to turn this into appropriately named list of object fields.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":426,"context_line":"Security impact"},{"line_number":427,"context_line":"---------------"},{"line_number":428,"context_line":""},{"line_number":429,"context_line":"Direct approach to creating a snapshot of instance using SPDK disks needs to"},{"line_number":430,"context_line":"determine parent image pool of instance\u0027s base image. To do that, it needs to"},{"line_number":431,"context_line":"pull image metadata from glance. It needs to be able to do this even if the"},{"line_number":432,"context_line":"base image is no longer accessible to the user. For this reason the call to"},{"line_number":433,"context_line":"glance uses nova service token as user token. Image metadata pulled this way is"},{"line_number":434,"context_line":"never exposed to the user, it is only used internally in nova-compute to"},{"line_number":435,"context_line":"determine the image pool in RBD cluster of the image."},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"To perform transfers of *lvol* contents between computes it is necessary to"},{"line_number":438,"context_line":"mount host\u0027s `/dev` in *libvirt-ssh* container."}],"source_content_type":"text/x-rst","patch_set":2,"id":"4997a67d_5cfc9c82","line":435,"range":{"start_line":429,"start_character":0,"end_line":435,"end_character":53},"updated":"2026-06-04 13:37:55.000000000","message":"so none of this shoudl really change\n\nwhen we create a vm we snapshot the image propeies in the system_metadta\n\nfor the spdk backend you shoudl not need to talk to glance to look this up as you\ncan read it form the system_metadta when needed\n\nthe user toke will have acces to the image when creating the vm or that should alwasy be rejected so i dont think there is a security impact here as you shoudl not be changing any of that logic.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":426,"context_line":"Security impact"},{"line_number":427,"context_line":"---------------"},{"line_number":428,"context_line":""},{"line_number":429,"context_line":"Direct approach to creating a snapshot of instance using SPDK disks needs to"},{"line_number":430,"context_line":"determine parent image pool of instance\u0027s base image. To do that, it needs to"},{"line_number":431,"context_line":"pull image metadata from glance. It needs to be able to do this even if the"},{"line_number":432,"context_line":"base image is no longer accessible to the user. For this reason the call to"},{"line_number":433,"context_line":"glance uses nova service token as user token. Image metadata pulled this way is"},{"line_number":434,"context_line":"never exposed to the user, it is only used internally in nova-compute to"},{"line_number":435,"context_line":"determine the image pool in RBD cluster of the image."},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"To perform transfers of *lvol* contents between computes it is necessary to"},{"line_number":438,"context_line":"mount host\u0027s `/dev` in *libvirt-ssh* container."}],"source_content_type":"text/x-rst","patch_set":2,"id":"e9d244ca_1e5d1662","line":435,"range":{"start_line":429,"start_character":0,"end_line":435,"end_character":53},"in_reply_to":"4997a67d_5cfc9c82","updated":"2026-06-12 10:41:41.000000000","message":"Unfortunately image locations are not saved in system_metadata. So we have to fetch them from glance.\nNote that process described in this spec is mostly reusing existing RBD mechanism (Defined here https://github.com/openstack/nova/blob/d8c997c350d745a118b34dfce62701f0013d7a16/nova/virt/libvirt/imagebackend.py#L1191).\nOnly we cannot use the default approach for RBD (from try block in above link) since we don\u0027t have an RBD device at hand.\nWe always use \"the hard way\" which includes asking glance for details. I\u0027d assume that this \"hard way\" fallback mechanism also would not work for RBD\nin case parent image is not accessible at time of unshelve (without a fix mentioned here in Security impact.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":434,"context_line":"never exposed to the user, it is only used internally in nova-compute to"},{"line_number":435,"context_line":"determine the image pool in RBD cluster of the image."},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"To perform transfers of *lvol* contents between computes it is necessary to"},{"line_number":438,"context_line":"mount host\u0027s `/dev` in *libvirt-ssh* container."},{"line_number":439,"context_line":""},{"line_number":440,"context_line":"Notifications impact"},{"line_number":441,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"46cb2bce_2262f89c","line":438,"range":{"start_line":437,"start_character":0,"end_line":438,"end_character":47},"updated":"2026-06-04 13:37:55.000000000","message":"again contierisatrion is out of scope fo this spec. it might be relevent in a kolla-ansible one but not here.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":434,"context_line":"never exposed to the user, it is only used internally in nova-compute to"},{"line_number":435,"context_line":"determine the image pool in RBD cluster of the image."},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"To perform transfers of *lvol* contents between computes it is necessary to"},{"line_number":438,"context_line":"mount host\u0027s `/dev` in *libvirt-ssh* container."},{"line_number":439,"context_line":""},{"line_number":440,"context_line":"Notifications impact"},{"line_number":441,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"204c99e2_0ddc6081","line":438,"range":{"start_line":437,"start_character":0,"end_line":438,"end_character":47},"in_reply_to":"46cb2bce_2262f89c","updated":"2026-06-12 10:41:41.000000000","message":"I removed this line from the spec.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":451,"context_line":"------------------"},{"line_number":452,"context_line":""},{"line_number":453,"context_line":"If standard approaches to pulling and pushing images were used, this would"},{"line_number":454,"context_line":"have a significant impact on the performance of building instances."},{"line_number":455,"context_line":"Uploading images to glance can cause nova-compute service instability since"},{"line_number":456,"context_line":"it\u0027s done on main thread (See `Bug 1932127"},{"line_number":457,"context_line":"\u003chttps://bugs.launchpad.net/nova/+bug/1932127\u003e`_)."}],"source_content_type":"text/x-rst","patch_set":2,"id":"cd79a4f8_00e70cca","line":454,"updated":"2026-06-04 13:37:55.000000000","message":"it would be no differnt then the qcow/flat backend today so that is not really correct\n\nits less then if we do rbd_direct_download but i think the current presentaiton here is misleading","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":451,"context_line":"------------------"},{"line_number":452,"context_line":""},{"line_number":453,"context_line":"If standard approaches to pulling and pushing images were used, this would"},{"line_number":454,"context_line":"have a significant impact on the performance of building instances."},{"line_number":455,"context_line":"Uploading images to glance can cause nova-compute service instability since"},{"line_number":456,"context_line":"it\u0027s done on main thread (See `Bug 1932127"},{"line_number":457,"context_line":"\u003chttps://bugs.launchpad.net/nova/+bug/1932127\u003e`_)."}],"source_content_type":"text/x-rst","patch_set":2,"id":"0f98cd2e_301eceb8","line":454,"in_reply_to":"cd79a4f8_00e70cca","updated":"2026-06-12 10:41:41.000000000","message":"Understood. I have removed this paragraph from the spec.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":452,"context_line":""},{"line_number":453,"context_line":"If standard approaches to pulling and pushing images were used, this would"},{"line_number":454,"context_line":"have a significant impact on the performance of building instances."},{"line_number":455,"context_line":"Uploading images to glance can cause nova-compute service instability since"},{"line_number":456,"context_line":"it\u0027s done on main thread (See `Bug 1932127"},{"line_number":457,"context_line":"\u003chttps://bugs.launchpad.net/nova/+bug/1932127\u003e`_)."},{"line_number":458,"context_line":"Due to these reasons using clone/direct_snapshot approaches will be encouraged."},{"line_number":459,"context_line":""},{"line_number":460,"context_line":"RPC calls to SPDK will be performed synchronously, however almost all of them"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fbcdf6c8_411cae23","line":457,"range":{"start_line":455,"start_character":0,"end_line":457,"end_character":50},"updated":"2026-06-04 13:37:55.000000000","message":"that is a bug regardles of this feature so callign it out here is not helpful unless you plan to fix it by offloding it ot a native thread as we did for\n\nhttps://bugs.launchpad.net/nova/+bug/1932127\nhttps://review.opendev.org/c/openstack/nova/+/734776","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":459,"context_line":""},{"line_number":460,"context_line":"RPC calls to SPDK will be performed synchronously, however almost all of them"},{"line_number":461,"context_line":"finish near-instantly. For exceptions to this rule (like"},{"line_number":462,"context_line":"`bdev_lvol_decouple_parent`), an asynchronous approach might be considered."},{"line_number":463,"context_line":""},{"line_number":464,"context_line":"Other deployer impact"},{"line_number":465,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"d8ff3e9c_8cb06ca5","line":462,"updated":"2026-06-04 13:37:55.000000000","message":"if needed we woudl dispatch it to a backgroun thread pool executor and then wait on the freturned future but i think this si fine\n\nwe can decied that in the impleation review","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":475,"context_line":"with NVMe driver connected and *lvstores* set up."},{"line_number":476,"context_line":"In case of SPDK process restart appropriate measures must be undertaken by an"},{"line_number":477,"context_line":"operator to ensure that both those SPDK structures and *vhost* controller are"},{"line_number":478,"context_line":"recreated using SPDK RPC interface."},{"line_number":479,"context_line":"How each SPDK structure needed for use with nova can be persisted:"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"* *NVMe controllers* - configured at start-up through RPC calls from static"}],"source_content_type":"text/x-rst","patch_set":2,"id":"77e70af4_137a36c1","line":478,"updated":"2026-06-04 13:37:55.000000000","message":"so honetly that might be a blocker.\n\ni undertand htat it took year to ensure that ovs-dpdk could restart and auto reconnect, and that durign the restat there is a dataplane outage for the gues but it does not impact the runing gues bar that drop in network io\n\nfor the disks on the ohterhand restartign the spdk host applcaiton is likely to cause the gues to hard log or at least see a momentary disk io issue while the spdk stack restarts\n\nif it does not automaticly reconenct to the vhost-user sockets and recover however that might be a blocker as i dont think that is really something folks coudl rely on in production\n\nout of band RPCs to spdk for nova managed lvos is also not something we woudl want to allow/supprot on the nova side for a runnign vm\n\nso i think we shoudl expect a better oeprator storey here then interveitng ot recreate the spdk state manually.\n\n\nthis woudl at least need to be called out as a warning in the docs and maybe make the feature experimetnal.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":475,"context_line":"with NVMe driver connected and *lvstores* set up."},{"line_number":476,"context_line":"In case of SPDK process restart appropriate measures must be undertaken by an"},{"line_number":477,"context_line":"operator to ensure that both those SPDK structures and *vhost* controller are"},{"line_number":478,"context_line":"recreated using SPDK RPC interface."},{"line_number":479,"context_line":"How each SPDK structure needed for use with nova can be persisted:"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"* *NVMe controllers* - configured at start-up through RPC calls from static"}],"source_content_type":"text/x-rst","patch_set":2,"id":"0b175c47_70a54893","line":478,"in_reply_to":"77e70af4_137a36c1","updated":"2026-06-12 10:41:41.000000000","message":"To avoid out-of-band RPCs we could:\n- for static SPDK structures (NVMe controllers and lvstores) mention that a configuration file passed to SPDK process at startup could be used https://spdk.io/doc/app_overview.html#cmd_arg_config_file.\n- for vhost controllers we have all necessary information to recreate them in nova, so maybe a periodic task checking if recreation of any of them is neccessaryand running appropriate RPC commands could be used?\n\nNote that QEMU will automatically reconnect with the vhost sockets as long as they are created again under a specified path so we only need to ensure that by running `vhost_create_blk_controller` or (`vhost_create_scsi_controller`\n\u0026 `vhost_scsi_controller_add_target`).","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":477,"context_line":"operator to ensure that both those SPDK structures and *vhost* controller are"},{"line_number":478,"context_line":"recreated using SPDK RPC interface."},{"line_number":479,"context_line":"How each SPDK structure needed for use with nova can be persisted:"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"* *NVMe controllers* - configured at start-up through RPC calls from static"},{"line_number":482,"context_line":"  configuration"},{"line_number":483,"context_line":""},{"line_number":484,"context_line":"* *lvstores* - configured at start-up through RPC calls from static"},{"line_number":485,"context_line":"  configuration"},{"line_number":486,"context_line":""},{"line_number":487,"context_line":"* *lvols* - they are automatically persisted. When *lvstore* is recreated after"},{"line_number":488,"context_line":"  restart they will reappear in list of available *bdevs*."},{"line_number":489,"context_line":""},{"line_number":490,"context_line":"* *vhost* controllers - `save_subsystem_config` RPC call can be used to store"},{"line_number":491,"context_line":"  their configuration to a file at shutdown, then they can be loaded up from"},{"line_number":492,"context_line":"  a file at *start-up* with `load_subsystem_config`."},{"line_number":493,"context_line":""},{"line_number":494,"context_line":"Example configuration will be provided as part of documentation for this"},{"line_number":495,"context_line":"feature."}],"source_content_type":"text/x-rst","patch_set":2,"id":"e2262f17_b6a3fdbc","line":492,"range":{"start_line":480,"start_character":1,"end_line":492,"end_character":52},"updated":"2026-06-04 13:37:55.000000000","message":"this does not sound like the applciation is really ready for integraiton with nova\nthis shoudl realy be persited in a db like the ovs db or config files that the spdk application read for the nvme conotler o lvstores without any need for rpc\n\nthe stores shoudl be static an internally persited without operator intervention.\ni think you need to go back to the spdk comunity and impove the persitnce before we really go much fother in the nova integration.\n\nif spdk cant automaticlly confire itself on agent restgart and reconenct to the vhost user socket created by qemu im ot sure its mature enough for us to suport.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":477,"context_line":"operator to ensure that both those SPDK structures and *vhost* controller are"},{"line_number":478,"context_line":"recreated using SPDK RPC interface."},{"line_number":479,"context_line":"How each SPDK structure needed for use with nova can be persisted:"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"* *NVMe controllers* - configured at start-up through RPC calls from static"},{"line_number":482,"context_line":"  configuration"},{"line_number":483,"context_line":""},{"line_number":484,"context_line":"* *lvstores* - configured at start-up through RPC calls from static"},{"line_number":485,"context_line":"  configuration"},{"line_number":486,"context_line":""},{"line_number":487,"context_line":"* *lvols* - they are automatically persisted. When *lvstore* is recreated after"},{"line_number":488,"context_line":"  restart they will reappear in list of available *bdevs*."},{"line_number":489,"context_line":""},{"line_number":490,"context_line":"* *vhost* controllers - `save_subsystem_config` RPC call can be used to store"},{"line_number":491,"context_line":"  their configuration to a file at shutdown, then they can be loaded up from"},{"line_number":492,"context_line":"  a file at *start-up* with `load_subsystem_config`."},{"line_number":493,"context_line":""},{"line_number":494,"context_line":"Example configuration will be provided as part of documentation for this"},{"line_number":495,"context_line":"feature."}],"source_content_type":"text/x-rst","patch_set":2,"id":"603bcba2_bbfa07f3","line":492,"range":{"start_line":480,"start_character":1,"end_line":492,"end_character":52},"in_reply_to":"e2262f17_b6a3fdbc","updated":"2026-06-12 10:41:41.000000000","message":"As mentioned earlier startup configuration of static objects and vhost socket reconnection are not a issue. The only pain point is vhost controller recreation (so issuing appropriate RPC commands). If the aproach with nova recreating vhost controllers is not sufficient I will need some time to think of a better approach.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":493,"context_line":""},{"line_number":494,"context_line":"Example configuration will be provided as part of documentation for this"},{"line_number":495,"context_line":"feature."},{"line_number":496,"context_line":""},{"line_number":497,"context_line":"Nova does not currently support changing storage backend of compute while there"},{"line_number":498,"context_line":"are instances present on that compute. If operator wants to reconfigure"},{"line_number":499,"context_line":"a compute to use SPDK storage backend, that compute must be first emptied."},{"line_number":500,"context_line":""},{"line_number":501,"context_line":"Developer impact"},{"line_number":502,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"cf39ecf4_18239965","line":499,"range":{"start_line":496,"start_character":1,"end_line":499,"end_character":74},"updated":"2026-06-04 13:37:55.000000000","message":"we also do not supprot moving intnace betwen host with diffent sotage backend in general\n\nthere is one way to do this that works by acident which si shelve and unshelve but today if you are deploying mixed storage backend you are expected to partion the could useing host agrates or simialr to ensure that vms do not change storage backend on live or cold migration.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":565,"context_line":"Dependencies"},{"line_number":566,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":567,"context_line":""},{"line_number":568,"context_line":"* SPDK RPC client library will be introduced as new dependency to nova. Its"},{"line_number":569,"context_line":"  license (BSD 3-Clause) is compatible with Openstack dependency licensing"},{"line_number":570,"context_line":"  requirements. SPDK project does not specify any compatibility matrix between"},{"line_number":571,"context_line":"  server (*vhost_target*) and client (SPDK RPC client library) but one can"}],"source_content_type":"text/x-rst","patch_set":2,"id":"b6356fe6_0b0921c0","line":568,"updated":"2026-06-04 13:37:55.000000000","message":"this would need to be an optional dependencies and added to the requriemetn repo.\n\nalso what disto currently package supprot for SPDK and this libviary?\n\nis it aviabel in ubutnu or rhel\n\nwe did not allow supprot for ovs-dpdk to be merged until ovs-dpdk was packaged in ubuntu debian or centos so we woudl proably requrie the same for spdk\n\nagain if there is not wide spread adoption fo this isn the disto ecosystem its an indeicator that the technology may not be mature enough to integrate.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":565,"context_line":"Dependencies"},{"line_number":566,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":567,"context_line":""},{"line_number":568,"context_line":"* SPDK RPC client library will be introduced as new dependency to nova. Its"},{"line_number":569,"context_line":"  license (BSD 3-Clause) is compatible with Openstack dependency licensing"},{"line_number":570,"context_line":"  requirements. SPDK project does not specify any compatibility matrix between"},{"line_number":571,"context_line":"  server (*vhost_target*) and client (SPDK RPC client library) but one can"}],"source_content_type":"text/x-rst","patch_set":2,"id":"998971d6_637de1c5","line":568,"in_reply_to":"b6356fe6_0b0921c0","updated":"2026-06-12 10:41:41.000000000","message":"Can I assume that packaging of SPDK server and client library are separate issues?\nBecause SPDK server is not packaged anywhere. It is only buildable from source. (Mainly due to high configurability of the build process)\n\nRegarding the python library. From what I see it is not packaged for any distro. It is only available on PyPI.\nHowever after examining it one more time it seems that the RPC part of it moved away from providing a function for each SPDK RPC command and\nbecame more like a proxy to SPDK\u0027s RPC interface. This allows it to be used with any SPDK version. Moreover the part we actually need is contained\nalmost entirely within a single file (with a one line util outside): https://github.com/spdk/spdk/blob/master/python/spdk/rpc/client.py.\nPerhaps a better approach would be to include this class within nova (along with its license) and not use this library as a dependency at all.\n\nI have rewritten this section to describe this approach.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":569,"context_line":"  license (BSD 3-Clause) is compatible with Openstack dependency licensing"},{"line_number":570,"context_line":"  requirements. SPDK project does not specify any compatibility matrix between"},{"line_number":571,"context_line":"  server (*vhost_target*) and client (SPDK RPC client library) but one can"},{"line_number":572,"context_line":"  generally expect support with a difference of +-1 version. This means that if"},{"line_number":573,"context_line":"  we ever update one of these, the other has to be updated too. For this reason"},{"line_number":574,"context_line":"  we will start with the latest SPDK version (v26.01) as client version. We"},{"line_number":575,"context_line":"  will only update client version if future modifications to SPDK usage in nova"}],"source_content_type":"text/x-rst","patch_set":2,"id":"4982680b_4667cca7","line":572,"range":{"start_line":572,"start_character":33,"end_line":572,"end_character":60},"updated":"2026-06-04 13:37:55.000000000","message":"that may be unreasonable for use to require\nnova is expected to work with many distos\nso while it may be reasonable fo use to deocuemtn that distos shoudl package a compatibel spdk client lib and package we can only really express a lower bount for spdk in nova as the min version we would supprot and we woudl need to mange that over time vai the same requirement process as other libs.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":569,"context_line":"  license (BSD 3-Clause) is compatible with Openstack dependency licensing"},{"line_number":570,"context_line":"  requirements. SPDK project does not specify any compatibility matrix between"},{"line_number":571,"context_line":"  server (*vhost_target*) and client (SPDK RPC client library) but one can"},{"line_number":572,"context_line":"  generally expect support with a difference of +-1 version. This means that if"},{"line_number":573,"context_line":"  we ever update one of these, the other has to be updated too. For this reason"},{"line_number":574,"context_line":"  we will start with the latest SPDK version (v26.01) as client version. We"},{"line_number":575,"context_line":"  will only update client version if future modifications to SPDK usage in nova"}],"source_content_type":"text/x-rst","patch_set":2,"id":"4dcfbe71_1d520c29","line":572,"range":{"start_line":572,"start_character":33,"end_line":572,"end_character":60},"in_reply_to":"4982680b_4667cca7","updated":"2026-06-12 10:41:41.000000000","message":"I have rewritten this part to align with approach of copying over just JSONRPCClient.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":576,"context_line":"  require new SPDK features. We will encourage users to use the same server"},{"line_number":577,"context_line":"  version as the client version used in nova by verifying through RPC call if"},{"line_number":578,"context_line":"  client and server use the same version and failing critically if not"},{"line_number":579,"context_line":"  (controlled by configuration flag, enabled by default)."},{"line_number":580,"context_line":""},{"line_number":581,"context_line":""},{"line_number":582,"context_line":"Testing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"de638d59_4c4ef400","line":579,"updated":"2026-06-04 13:37:55.000000000","message":"ack i guess this coer what i said above","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"51e41b3b7f68428fad341c4b02d95f4732d84aa8","unresolved":true,"context_lines":[{"line_number":594,"context_line":"testing environment. It is possible to setup SPDK to use *malloc bdev* (in"},{"line_number":595,"context_line":"memory) or *linux AIO bdev* (standard linux block device or file) as its base"},{"line_number":596,"context_line":"storage. This degrades performance but does not change functionality in any"},{"line_number":597,"context_line":"way. Such setup could be used for testing on devstack."},{"line_number":598,"context_line":""},{"line_number":599,"context_line":"Documentation Impact"},{"line_number":600,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"08b07944_ecc7ea23","line":597,"updated":"2026-06-04 13:37:55.000000000","message":"yes i think we woudl rquire this before merging any code.\n\nthat could either live in its own devstack plugin or in the nova devstack plugin.","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"},{"author":{"_account_id":38512,"name":"Karol Klimaszewski","display_name":"kklimaszewski","email":"kklimaszewski@cloudferro.com","username":"kklimaszewski"},"change_message_id":"3e1da31caa9bde3f565295bbc0ba281fa3b69c21","unresolved":true,"context_lines":[{"line_number":594,"context_line":"testing environment. It is possible to setup SPDK to use *malloc bdev* (in"},{"line_number":595,"context_line":"memory) or *linux AIO bdev* (standard linux block device or file) as its base"},{"line_number":596,"context_line":"storage. This degrades performance but does not change functionality in any"},{"line_number":597,"context_line":"way. Such setup could be used for testing on devstack."},{"line_number":598,"context_line":""},{"line_number":599,"context_line":"Documentation Impact"},{"line_number":600,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"e0107683_eed542a4","line":597,"in_reply_to":"08b07944_ecc7ea23","updated":"2026-06-12 10:41:41.000000000","message":"Understood. So this would be a prerequisite for spec implementation, but not for approval?","commit_id":"41a731620daaeb571b2933bef7d023940aa16473"}]}