)]}'
{"specs/mitaka/approved/migrate-libvirt-volumes.rst":[{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"f42ac2239e006f725144b49954976877f23ed343","unresolved":false,"context_lines":[{"line_number":186,"context_line":"`Use libvirt storage pools`_"},{"line_number":187,"context_line":""},{"line_number":188,"context_line":".. _Use libvirt storage pools:"},{"line_number":189,"context_line":"   https://blueprints.launchpad.net/nova/+spec/use-libvirt-storage-pools"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"Testing"},{"line_number":192,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_04605f00","line":189,"range":{"start_line":189,"start_character":3,"end_line":189,"end_character":72},"updated":"2015-11-10 12:03:27.000000000","message":"This is not approved for mitaka, I think we need to depend on that being re-proposed for mitaka:\nhttp://specs.openstack.org/openstack/nova-specs/specs/kilo/approved/use-libvirt-storage-pools.html\n\nThe person who was previously working on that has now moved on from working on Nova, I believe.","commit_id":"ae2993fcfe0779cbcd89a5d3d259603d325f5a5e"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"77fef15e721cd7606709686e8b3db34a0c589d37","unresolved":false,"context_lines":[{"line_number":186,"context_line":"`Use libvirt storage pools`_"},{"line_number":187,"context_line":""},{"line_number":188,"context_line":".. _Use libvirt storage pools:"},{"line_number":189,"context_line":"   https://blueprints.launchpad.net/nova/+spec/use-libvirt-storage-pools"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"Testing"},{"line_number":192,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_5d7dda6c","line":189,"in_reply_to":"da85f559_04605f00","updated":"2015-11-10 12:48:13.000000000","message":"John, I\u0027m singing up for both this and the the use-libvirt-storage-pools spec, if you could change the assignee on the use-libvirt-storage-pools spec to paul-carlton2 that would be great, thanks.","commit_id":"ae2993fcfe0779cbcd89a5d3d259603d325f5a5e"},{"author":{"_account_id":7461,"name":"Paul Murray","email":"pmurray@hpe.com","username":"ptm"},"change_message_id":"0cddb7404e1b9c902d1a1e2dcb334a985280a828","unresolved":false,"context_lines":[{"line_number":186,"context_line":"`Use libvirt storage pools`_"},{"line_number":187,"context_line":""},{"line_number":188,"context_line":".. _Use libvirt storage pools:"},{"line_number":189,"context_line":"   https://blueprints.launchpad.net/nova/+spec/use-libvirt-storage-pools"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"Testing"},{"line_number":192,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_6d2ca302","line":189,"in_reply_to":"da85f559_5d7dda6c","updated":"2015-11-10 15:26:14.000000000","message":"This is the spec for the storage pools work - Paul intends to do both\n\nSee: https://review.openstack.org/#/c/232045/","commit_id":"ae2993fcfe0779cbcd89a5d3d259603d325f5a5e"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"a90629b10b5d7910a89b3f78295ac0b529b1f33e","unresolved":false,"context_lines":[{"line_number":33,"context_line":"* From a setup perspective, it adds several extra steps to a setup:"},{"line_number":34,"context_line":"  System administrators, or their setup tools, must generate a keypair"},{"line_number":35,"context_line":"  for each compute node, and upload the public key to each of the other"},{"line_number":36,"context_line":"  compute nodes."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Use Cases"},{"line_number":39,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_38e0b1cc","line":36,"updated":"2015-11-12 11:40:36.000000000","message":"Nit: Feels like we should mention this now means you need to configure SASL or TLS for the libvirt\u003c-\u003elibvirt comms","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"0adbbe3fa657880d11626bbb8f5aea4064f7cef4","unresolved":false,"context_lines":[{"line_number":33,"context_line":"* From a setup perspective, it adds several extra steps to a setup:"},{"line_number":34,"context_line":"  System administrators, or their setup tools, must generate a keypair"},{"line_number":35,"context_line":"  for each compute node, and upload the public key to each of the other"},{"line_number":36,"context_line":"  compute nodes."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Use Cases"},{"line_number":39,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_0956bd05","line":36,"in_reply_to":"da85f559_38e0b1cc","updated":"2015-11-12 12:24:16.000000000","message":"Done","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"a90629b10b5d7910a89b3f78295ac0b529b1f33e","unresolved":false,"context_lines":[{"line_number":43,"context_line":"migrations, as long as the deployer is using the new libvirt storage pools"},{"line_number":44,"context_line":"image backend."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"None"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Proposed change"},{"line_number":49,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_98a75da6","line":46,"updated":"2015-11-12 11:40:36.000000000","message":"Nit: I would remove this.","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"0adbbe3fa657880d11626bbb8f5aea4064f7cef4","unresolved":false,"context_lines":[{"line_number":43,"context_line":"migrations, as long as the deployer is using the new libvirt storage pools"},{"line_number":44,"context_line":"image backend."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"None"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Proposed change"},{"line_number":49,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_29594135","line":46,"in_reply_to":"da85f559_98a75da6","updated":"2015-11-12 12:24:16.000000000","message":"Done","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"a90629b10b5d7910a89b3f78295ac0b529b1f33e","unresolved":false,"context_lines":[{"line_number":136,"context_line":"  IO, detection of partial sends, etc).  While this would be less performant"},{"line_number":137,"context_line":"  than having C code which would do the transfer, I suspect there are ways in"},{"line_number":138,"context_line":"  which we could optimize the code, or other methods (such as utilizing the"},{"line_number":139,"context_line":"  libvirt command line client)."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Other deployer impact"},{"line_number":142,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_9848bdb4","line":139,"updated":"2015-11-12 11:40:36.000000000","message":"We could fixate on this bit, lets leave this as an implementation detail.\n\nIts very tempting to shell out to the CLI to deal with the transfer.","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"0adbbe3fa657880d11626bbb8f5aea4064f7cef4","unresolved":false,"context_lines":[{"line_number":136,"context_line":"  IO, detection of partial sends, etc).  While this would be less performant"},{"line_number":137,"context_line":"  than having C code which would do the transfer, I suspect there are ways in"},{"line_number":138,"context_line":"  which we could optimize the code, or other methods (such as utilizing the"},{"line_number":139,"context_line":"  libvirt command line client)."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Other deployer impact"},{"line_number":142,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_7bc39338","line":139,"in_reply_to":"da85f559_9848bdb4","updated":"2015-11-12 12:24:16.000000000","message":"Agreed, will find the right method during implementation and we can cover this issue when reviewing those changes","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"a90629b10b5d7910a89b3f78295ac0b529b1f33e","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":".. _Use libvirt storage pools:"},{"line_number":191,"context_line":"   https://blueprints.launchpad.net/nova/+spec/use-libvirt-storage-pools"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"Testing"},{"line_number":194,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_98c2fd51","line":192,"updated":"2015-11-12 11:40:36.000000000","message":"So storage pools requires a libvirt version newer than our minimum version. Where does that leave us with making this the default method?","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":1779,"name":"Daniel Berrange","email":"berrange@redhat.com","username":"berrange"},"change_message_id":"5814c11d5a49a0e231c0b96d51cbbfbdaed31054","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":".. _Use libvirt storage pools:"},{"line_number":191,"context_line":"   https://blueprints.launchpad.net/nova/+spec/use-libvirt-storage-pools"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"Testing"},{"line_number":194,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_09f23d06","line":192,"in_reply_to":"da85f559_98c2fd51","updated":"2015-11-12 12:14:41.000000000","message":"We should not require a newer libvirt version - the storage pools feature has existed in libvirt for a long time.  So if the code is written to require features from a new version, I would consider that a bug and -1 the patch.\n\nIt is my expectation that the impl of the storage pools feature would use storage pools as the default immediately - this was one of the reasons I -1 the original code patches, because they only used it optionally, leaving us with 2 codepaths to maintain","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"0adbbe3fa657880d11626bbb8f5aea4064f7cef4","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":".. _Use libvirt storage pools:"},{"line_number":191,"context_line":"   https://blueprints.launchpad.net/nova/+spec/use-libvirt-storage-pools"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"Testing"},{"line_number":194,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_a9be91ab","line":192,"in_reply_to":"da85f559_98c2fd51","updated":"2015-11-12 12:24:16.000000000","message":"understood, will implement storage pools in a way that does not require a bump in the  minimum version so that it can be the default immediately","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"a90629b10b5d7910a89b3f78295ac0b529b1f33e","unresolved":false,"context_lines":[{"line_number":199,"context_line":"backend is also in use."},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"One potential location to enable these code paths is the \"bleeding edge\""},{"line_number":202,"context_line":"libvirt tests which are being created."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_b8982179","line":202,"updated":"2015-11-12 11:40:36.000000000","message":"+1 for that, could we make this sound like more of a commitment?","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"0adbbe3fa657880d11626bbb8f5aea4064f7cef4","unresolved":false,"context_lines":[{"line_number":199,"context_line":"backend is also in use."},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"One potential location to enable these code paths is the \"bleeding edge\""},{"line_number":202,"context_line":"libvirt tests which are being created."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":8,"id":"da85f559_29438114","line":202,"in_reply_to":"da85f559_b8982179","updated":"2015-11-12 12:24:16.000000000","message":"Done","commit_id":"1a28f541f7e305dc64edf7a69ef426e9417ce037"},{"author":{"_account_id":7461,"name":"Paul Murray","email":"pmurray@hpe.com","username":"ptm"},"change_message_id":"a813474ad6ba40023463ed7f0d227242a823eed0","unresolved":false,"context_lines":[{"line_number":32,"context_line":""},{"line_number":33,"context_line":"* From a setup perspective, it adds several extra steps to a setup:"},{"line_number":34,"context_line":"  System administrators, or their setup tools, configure libvirt to use"},{"line_number":35,"context_line":"  secure a secure communication channel between source and target node."},{"line_number":36,"context_line":"  This could be the use of TLS for example.  The must also generate a"},{"line_number":37,"context_line":"  keypair for each compute node, and upload the public key to each of"},{"line_number":38,"context_line":"  the other compute nodes."}],"source_content_type":"text/x-rst","patch_set":9,"id":"da85f559_d43fe984","line":35,"updated":"2015-11-12 13:48:05.000000000","message":"nit: secure a secure","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":32,"context_line":""},{"line_number":33,"context_line":"* From a setup perspective, it adds several extra steps to a setup:"},{"line_number":34,"context_line":"  System administrators, or their setup tools, configure libvirt to use"},{"line_number":35,"context_line":"  secure a secure communication channel between source and target node."},{"line_number":36,"context_line":"  This could be the use of TLS for example.  The must also generate a"},{"line_number":37,"context_line":"  keypair for each compute node, and upload the public key to each of"},{"line_number":38,"context_line":"  the other compute nodes."}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_c0fbb145","line":35,"in_reply_to":"da85f559_d43fe984","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":7461,"name":"Paul Murray","email":"pmurray@hpe.com","username":"ptm"},"change_message_id":"a813474ad6ba40023463ed7f0d227242a823eed0","unresolved":false,"context_lines":[{"line_number":33,"context_line":"* From a setup perspective, it adds several extra steps to a setup:"},{"line_number":34,"context_line":"  System administrators, or their setup tools, configure libvirt to use"},{"line_number":35,"context_line":"  secure a secure communication channel between source and target node."},{"line_number":36,"context_line":"  This could be the use of TLS for example.  The must also generate a"},{"line_number":37,"context_line":"  keypair for each compute node, and upload the public key to each of"},{"line_number":38,"context_line":"  the other compute nodes."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"da85f559_f4ff8d63","line":36,"updated":"2015-11-12 13:48:05.000000000","message":"nit: s/The must/They must/","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":33,"context_line":"* From a setup perspective, it adds several extra steps to a setup:"},{"line_number":34,"context_line":"  System administrators, or their setup tools, configure libvirt to use"},{"line_number":35,"context_line":"  secure a secure communication channel between source and target node."},{"line_number":36,"context_line":"  This could be the use of TLS for example.  The must also generate a"},{"line_number":37,"context_line":"  keypair for each compute node, and upload the public key to each of"},{"line_number":38,"context_line":"  the other compute nodes."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_8098491d","line":36,"in_reply_to":"da85f559_f4ff8d63","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":9555,"name":"Matthew Booth","email":"mbooth@redhat.com","username":"MatthewBooth"},"change_message_id":"c0e530f009b33528cb5d36ab34ac32cc3a701bcb","unresolved":false,"context_lines":[{"line_number":53,"context_line":":code:`[libvirt]vol_upload_migration` option."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"At migration time, a new volume would be created in the destination node\u0027s"},{"line_number":56,"context_line":"storage pool, and the methods virStorageVolDowload and virStorageVolUpload"},{"line_number":57,"context_line":"would be used to stream the contents of the disk between compute nodes"},{"line_number":58,"context_line":"(http://libvirt.org/html/libvirt-libvirt-storage.html#virStorageVolUpload)."},{"line_number":59,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_bce3d104","line":56,"updated":"2015-11-17 12:06:29.000000000","message":"nit: virStorageVolDownload","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":53,"context_line":":code:`[libvirt]vol_upload_migration` option."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"At migration time, a new volume would be created in the destination node\u0027s"},{"line_number":56,"context_line":"storage pool, and the methods virStorageVolDowload and virStorageVolUpload"},{"line_number":57,"context_line":"would be used to stream the contents of the disk between compute nodes"},{"line_number":58,"context_line":"(http://libvirt.org/html/libvirt-libvirt-storage.html#virStorageVolUpload)."},{"line_number":59,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_a069cdfa","line":56,"in_reply_to":"ba8a016a_bce3d104","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":9555,"name":"Matthew Booth","email":"mbooth@redhat.com","username":"MatthewBooth"},"change_message_id":"c0e530f009b33528cb5d36ab34ac32cc3a701bcb","unresolved":false,"context_lines":[{"line_number":127,"context_line":"* Rsync compresses the contents to be transfered, but AFAIK libvirt"},{"line_number":128,"context_line":"  does not (although this is being worked on in conjunction with the libvirt"},{"line_number":129,"context_line":"  developers).  This could result in more data being transfered over the"},{"line_number":130,"context_line":"  network."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"* The actually streaming process requires some use of intermediary code to call"},{"line_number":133,"context_line":"  the write function on the contents of the read function.  One way would be"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_42167ea8","line":130,"updated":"2015-11-17 12:06:29.000000000","message":"As the former maintainer of a v2v tool I have some experience in this area. In my experience, compression degrades transfer performance on a fast network, whilst also having a severe cpu impact on the sender. However, losing efficient sparse handling is often a severe performance issue.\n\nRsync doesn\u0027t compress by default, and we aren\u0027t enabling it. However, it does do efficient sparse handling by default.","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":127,"context_line":"* Rsync compresses the contents to be transfered, but AFAIK libvirt"},{"line_number":128,"context_line":"  does not (although this is being worked on in conjunction with the libvirt"},{"line_number":129,"context_line":"  developers).  This could result in more data being transfered over the"},{"line_number":130,"context_line":"  network."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"* The actually streaming process requires some use of intermediary code to call"},{"line_number":133,"context_line":"  the write function on the contents of the read function.  One way would be"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_bbb81e2b","line":130,"in_reply_to":"ba8a016a_42167ea8","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":9555,"name":"Matthew Booth","email":"mbooth@redhat.com","username":"MatthewBooth"},"change_message_id":"4b889d7be0a8919a3f9b3eede4f74cdb842c1295","unresolved":false,"context_lines":[{"line_number":127,"context_line":"* Rsync compresses the contents to be transfered, but AFAIK libvirt"},{"line_number":128,"context_line":"  does not (although this is being worked on in conjunction with the libvirt"},{"line_number":129,"context_line":"  developers).  This could result in more data being transfered over the"},{"line_number":130,"context_line":"  network."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"* The actually streaming process requires some use of intermediary code to call"},{"line_number":133,"context_line":"  the write function on the contents of the read function.  One way would be"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_6535c9d8","line":130,"in_reply_to":"ba8a016a_42167ea8","updated":"2015-11-17 12:52:41.000000000","message":"I\u0027m going to ratchet this one up a notch: the libvirt driver does metadata preallocation of qcow2 images, which results in them being full sized sparse files. This means that the performance impact of not being able to do sparse detection is almost always going to be severe. With this in mind, I think we must leave rsync as a first-class option for the forseeable future.","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":127,"context_line":"* Rsync compresses the contents to be transfered, but AFAIK libvirt"},{"line_number":128,"context_line":"  does not (although this is being worked on in conjunction with the libvirt"},{"line_number":129,"context_line":"  developers).  This could result in more data being transfered over the"},{"line_number":130,"context_line":"  network."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"* The actually streaming process requires some use of intermediary code to call"},{"line_number":133,"context_line":"  the write function on the contents of the read function.  One way would be"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_5b7ed26a","line":130,"in_reply_to":"ba8a016a_6535c9d8","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":14131,"name":"shaohef","email":"shaohe.feng@intel.com","username":"shaohefeng"},"change_message_id":"b9bc88985375ba09ea49ceca0e403131422b5bbc","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  using python as an intermediary (e.g.  :code:`data \u003d stream1.recv(1024*64);"},{"line_number":135,"context_line":"  stream2.send(data)`, although the actual code would properly support async"},{"line_number":136,"context_line":"  IO, detection of partial sends, etc).  While this would be less performant"},{"line_number":137,"context_line":"  than having C code which would do the transfer, I suspect there are ways in"},{"line_number":138,"context_line":"  which we could optimize the code, or other methods (such as utilizing the"},{"line_number":139,"context_line":"  libvirt command line client)."},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_586f3192","line":137,"updated":"2015-11-17 20:23:29.000000000","message":"which good ways to reduce  performance degradation?","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  using python as an intermediary (e.g.  :code:`data \u003d stream1.recv(1024*64);"},{"line_number":135,"context_line":"  stream2.send(data)`, although the actual code would properly support async"},{"line_number":136,"context_line":"  IO, detection of partial sends, etc).  While this would be less performant"},{"line_number":137,"context_line":"  than having C code which would do the transfer, I suspect there are ways in"},{"line_number":138,"context_line":"  which we could optimize the code, or other methods (such as utilizing the"},{"line_number":139,"context_line":"  libvirt command line client)."},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_1b404a33","line":137,"in_reply_to":"ba8a016a_586f3192","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":9555,"name":"Matthew Booth","email":"mbooth@redhat.com","username":"MatthewBooth"},"change_message_id":"c0e530f009b33528cb5d36ab34ac32cc3a701bcb","unresolved":false,"context_lines":[{"line_number":136,"context_line":"  IO, detection of partial sends, etc).  While this would be less performant"},{"line_number":137,"context_line":"  than having C code which would do the transfer, I suspect there are ways in"},{"line_number":138,"context_line":"  which we could optimize the code, or other methods (such as utilizing the"},{"line_number":139,"context_line":"  libvirt command line client)."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Other deployer impact"},{"line_number":142,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_e2dd3243","line":139,"updated":"2015-11-17 12:06:29.000000000","message":"FYI, I have experience of doing this in Ruby, which has similar non-threading issues to Python. I was updating a data transfer which previously called out to native scp to use libssh2 instead, which involved the ruby intermediary you describe above. The performance impact was catastrophic: the new code performed at about 20% of the original. The issue was not underlying performance, it was scheduling latency. I eventually managed to restore original performance on the third design iteration by writing a custom ruby extension which did the copy in a native thread, in C. As that\u0027s not a viable option for us, I suspect we\u0027ll be shelling out to something.\n\nTo the best of my knowledge, our test infrastructure doesn\u0027t currently allow us to test for performance regressions. However, we should take care to design, run, and document a test for this. I would say that anything less than 80% of native performance would be unacceptable.","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":136,"context_line":"  IO, detection of partial sends, etc).  While this would be less performant"},{"line_number":137,"context_line":"  than having C code which would do the transfer, I suspect there are ways in"},{"line_number":138,"context_line":"  which we could optimize the code, or other methods (such as utilizing the"},{"line_number":139,"context_line":"  libvirt command line client)."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Other deployer impact"},{"line_number":142,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_5b7012a4","line":139,"in_reply_to":"ba8a016a_e2dd3243","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":14131,"name":"shaohef","email":"shaohe.feng@intel.com","username":"shaohefeng"},"change_message_id":"b9bc88985375ba09ea49ceca0e403131422b5bbc","unresolved":false,"context_lines":[{"line_number":225,"context_line":""},{"line_number":226,"context_line":"* http://libvirt.org/html/libvirt-libvirt.html#virStorageVolUpload"},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"* http://libvirt.org/auth.html)"},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"* http://libvirt.org/remote.html#Remote_libvirtd_configuration"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_9047b71c","line":228,"updated":"2015-11-17 20:23:29.000000000","message":"nit: redundant \")\"","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":6608,"name":"Paul Carlton","email":"paul.carlton2@hpe.com","username":"paul-carlton2"},"change_message_id":"2238960facece6572224c357ca9d3848defe76de","unresolved":false,"context_lines":[{"line_number":225,"context_line":""},{"line_number":226,"context_line":"* http://libvirt.org/html/libvirt-libvirt.html#virStorageVolUpload"},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"* http://libvirt.org/auth.html)"},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"* http://libvirt.org/remote.html#Remote_libvirtd_configuration"}],"source_content_type":"text/x-rst","patch_set":9,"id":"ba8a016a_9ba19afa","line":228,"in_reply_to":"ba8a016a_9047b71c","updated":"2015-11-19 18:36:52.000000000","message":"Done","commit_id":"f14f90ebec600dc69fdeb945cf8fdf58a04a7955"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"6a4e4d9988fb77075ee6660468f56eeb96c4889f","unresolved":false,"context_lines":[{"line_number":199,"context_line":"backend is also in use."},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"Enable these code paths and work with the \"bleeding edge\" libvirt tests"},{"line_number":202,"context_line":"which are being created to test."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":10,"id":"ba8a016a_794ed0c2","line":202,"updated":"2015-11-23 12:16:15.000000000","message":"Looking forward to seeing the functional tests passing before we merge this.","commit_id":"57c4859cd1571027454175a494d57a7186e67d71"}]}