)]}'
{"specs/mitaka/scalable-backup-service.rst":[{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cd45a1577a146c096ba5b0734f3d19b2d563dc2c","unresolved":false,"context_lines":[{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Note that the need for concurrent backup and restore operations,"},{"line_number":102,"context_line":"especially to the same backend, may be artificially suppressed today"},{"line_number":103,"context_line":"because of the lack of support for *live* backups.  We anticipate that"},{"line_number":104,"context_line":"when backups of *in-use* volumes are supported [3], this need will"},{"line_number":105,"context_line":"significantly increase."},{"line_number":106,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_55251ec0","line":103,"updated":"2015-11-03 13:44:56.000000000","message":"I think that they are supported now, as BP is in \"implemented\" state.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"49e255edf2f224005a3f9ba2a3b9ef2616c8b4f6","unresolved":false,"context_lines":[{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Note that the need for concurrent backup and restore operations,"},{"line_number":102,"context_line":"especially to the same backend, may be artificially suppressed today"},{"line_number":103,"context_line":"because of the lack of support for *live* backups.  We anticipate that"},{"line_number":104,"context_line":"when backups of *in-use* volumes are supported [3], this need will"},{"line_number":105,"context_line":"significantly increase."},{"line_number":106,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_23a0ee42","line":103,"in_reply_to":"fa80f949_55251ec0","updated":"2015-11-05 17:40:46.000000000","message":"Good point, will update.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cd45a1577a146c096ba5b0734f3d19b2d563dc2c","unresolved":false,"context_lines":[{"line_number":117,"context_line":"builds an appropriate ``connector`` for the attachment given"},{"line_number":118,"context_line":"information about the backend (iSCSI, FibreChannel, NFS, whatever)"},{"line_number":119,"context_line":"obtained by running a volume driver ``initialize_connection`` method"},{"line_number":120,"context_line":"-- either directly or via an rpc to the appropriate volume service."},{"line_number":121,"context_line":"In the case of a *remote* attachment, ``initialize_connection`` is"},{"line_number":122,"context_line":"invoked by rpc.  This is the only part of the workflow that actually"},{"line_number":123,"context_line":"requires the volume service, and since it can be an rpc invocation,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_39adf088","line":120,"updated":"2015-11-03 13:44:56.000000000","message":"We have non-disruptive backups implemented now. This solution will work when backup will be created from temporary volume. What if it is created from temporary snapshot like in LVM situation? IMO review [1] made the backup service actually more coupled with c-vol for LVM driver. How are we going to solve that?\n\n[1] https://review.openstack.org/#/c/193937/","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"2ff2b3c426ae2b3a2bf60131ba8f6fda2ec53edb","unresolved":false,"context_lines":[{"line_number":117,"context_line":"builds an appropriate ``connector`` for the attachment given"},{"line_number":118,"context_line":"information about the backend (iSCSI, FibreChannel, NFS, whatever)"},{"line_number":119,"context_line":"obtained by running a volume driver ``initialize_connection`` method"},{"line_number":120,"context_line":"-- either directly or via an rpc to the appropriate volume service."},{"line_number":121,"context_line":"In the case of a *remote* attachment, ``initialize_connection`` is"},{"line_number":122,"context_line":"invoked by rpc.  This is the only part of the workflow that actually"},{"line_number":123,"context_line":"requires the volume service, and since it can be an rpc invocation,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_1bf432f0","line":120,"in_reply_to":"fa80f949_29d20fd3","updated":"2015-11-05 18:24:25.000000000","message":"OK, cool.  Thanks!","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"a62145137436b595910faf91eca5b41a34009be5","unresolved":false,"context_lines":[{"line_number":117,"context_line":"builds an appropriate ``connector`` for the attachment given"},{"line_number":118,"context_line":"information about the backend (iSCSI, FibreChannel, NFS, whatever)"},{"line_number":119,"context_line":"obtained by running a volume driver ``initialize_connection`` method"},{"line_number":120,"context_line":"-- either directly or via an rpc to the appropriate volume service."},{"line_number":121,"context_line":"In the case of a *remote* attachment, ``initialize_connection`` is"},{"line_number":122,"context_line":"invoked by rpc.  This is the only part of the workflow that actually"},{"line_number":123,"context_line":"requires the volume service, and since it can be an rpc invocation,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_29d20fd3","line":120,"in_reply_to":"fa80f949_39adf088","updated":"2015-11-05 18:21:54.000000000","message":"Using a temp snapshot will work on a remote node if the driver has implemented functions to allow snapshot to be attached to the host.  In the case of LVM, there were concerns that it will break LVM if we allow snapshot to be attached as LVM code is already flaky.  I have implemented it but can\u0027t provide that solution because of the concerns raised during the review.  So we need to use temp volume for LVM if we decouple backup node from volume node.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cd45a1577a146c096ba5b0734f3d19b2d563dc2c","unresolved":false,"context_lines":[{"line_number":189,"context_line":"needs to distinguish betwen in-flight operations that are owned by"},{"line_number":190,"context_line":"another backup-service instance and orphaned operations."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"When a backup service starts up it can query the database for all"},{"line_number":193,"context_line":"active backup services and select all whose ``host`` field is other"},{"line_number":194,"context_line":"than itself.  (If in the future multiple services are running on a"},{"line_number":195,"context_line":"single node, we can enrich the ``host`` field to distinguish among"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_1c0a5ad3","line":192,"updated":"2015-11-03 13:44:56.000000000","message":"Why don\u0027t we leave things as they are now and let c-bak clean up only it\u0027s own backups as first step? We\u0027re going the same way with c-vol A/A initially. Cleanup of other\u0027s nodes work would require c-bak to autofence itself once its connection to the DB is lost (the reasons are similar to these in case of c-vol A/A [1]).\n\n[1] https://review.openstack.org/#/c/237076/","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"49e255edf2f224005a3f9ba2a3b9ef2616c8b4f6","unresolved":false,"context_lines":[{"line_number":189,"context_line":"needs to distinguish betwen in-flight operations that are owned by"},{"line_number":190,"context_line":"another backup-service instance and orphaned operations."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"When a backup service starts up it can query the database for all"},{"line_number":193,"context_line":"active backup services and select all whose ``host`` field is other"},{"line_number":194,"context_line":"than itself.  (If in the future multiple services are running on a"},{"line_number":195,"context_line":"single node, we can enrich the ``host`` field to distinguish among"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_c38fa2ca","line":192,"in_reply_to":"fa80f949_1c0a5ad3","updated":"2015-11-05 17:40:46.000000000","message":"Yes, the issues are similar and we should likely restrict the cleanup at startup for now as is beiing done for c-vol A/A.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"b6d7e2a7c84f85aac9bd7e2d9ae8a56fa6419167","unresolved":false,"context_lines":[{"line_number":189,"context_line":"needs to distinguish betwen in-flight operations that are owned by"},{"line_number":190,"context_line":"another backup-service instance and orphaned operations."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"When a backup service starts up it can query the database for all"},{"line_number":193,"context_line":"active backup services and select all whose ``host`` field is other"},{"line_number":194,"context_line":"than itself.  (If in the future multiple services are running on a"},{"line_number":195,"context_line":"single node, we can enrich the ``host`` field to distinguish among"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da85f559_58d4deea","line":192,"in_reply_to":"fa80f949_c38fa2ca","updated":"2015-11-11 07:21:32.000000000","message":"I agree to keep things as they are now.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"b6d7e2a7c84f85aac9bd7e2d9ae8a56fa6419167","unresolved":false,"context_lines":[{"line_number":214,"context_line":"(or, down the road, a scheduler) when the backup or restore rpc is"},{"line_number":215,"context_line":"cast to a backup host and cleared by the backup service itself at"},{"line_number":216,"context_line":"completion of the operation.  When doing ``init_host`` cleanup, the"},{"line_number":217,"context_line":"backup service will *claim* the in-flight operation by putting its own"},{"line_number":218,"context_line":"``host`` value in the ``backup host`` field, and then clearing that"},{"line_number":219,"context_line":"field at the completion of the cleanup operation for that orphaned"},{"line_number":220,"context_line":"backup or restore."}],"source_content_type":"text/x-rst","patch_set":2,"id":"da85f559_d82d2eff","line":217,"updated":"2015-11-11 07:21:32.000000000","message":"If we all agree comments in line 192, this part needs to update.\n\nAnd host needs to change in following cases: \n\nEnv: volume is in node A, backup service 1 is in node B, and backup service 2 is in node C.\n\nOperations:\n\n1. When creating a backup service from above volume, the task is scheduled to node B, so the host of the backup service is set to B. \n\n2. If restoring another volume from above backup, the task is scheduled to node C, the host of the backup service needs to set to C. \n\nThat is where the backup service is operating, its host needs to update.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cd45a1577a146c096ba5b0734f3d19b2d563dc2c","unresolved":false,"context_lines":[{"line_number":248,"context_line":"-----------------"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"None."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"Cross-project impact"},{"line_number":253,"context_line":"--------------------"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_19302c56","line":251,"updated":"2015-11-03 13:44:56.000000000","message":"We don\u0027t have that section in the template, I think it should be removed from the spec.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"49e255edf2f224005a3f9ba2a3b9ef2616c8b4f6","unresolved":false,"context_lines":[{"line_number":248,"context_line":"-----------------"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"None."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"Cross-project impact"},{"line_number":253,"context_line":"--------------------"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_83c0bae1","line":251,"in_reply_to":"fa80f949_19302c56","updated":"2015-11-05 17:40:46.000000000","message":"OK.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cd45a1577a146c096ba5b0734f3d19b2d563dc2c","unresolved":false,"context_lines":[{"line_number":292,"context_line":""},{"line_number":293,"context_line":"Other deployer impact"},{"line_number":294,"context_line":"---------------------"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"None"},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"Developer impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_b98240c7","line":295,"updated":"2015-11-03 13:44:56.000000000","message":"Deployment workflow is certainly affected by the fact that c-bak doesn\u0027t need to work together with c-vol.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"49e255edf2f224005a3f9ba2a3b9ef2616c8b4f6","unresolved":false,"context_lines":[{"line_number":292,"context_line":""},{"line_number":293,"context_line":"Other deployer impact"},{"line_number":294,"context_line":"---------------------"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"None"},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"Developer impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_a3ab7e1c","line":295,"in_reply_to":"fa80f949_b98240c7","updated":"2015-11-05 17:40:46.000000000","message":"Good point :-)","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"a62145137436b595910faf91eca5b41a34009be5","unresolved":false,"context_lines":[{"line_number":312,"context_line":"* Tom Barron (tbarron, tpb@dyncloud.net)"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"Other contributors:"},{"line_number":315,"context_line":"xs"},{"line_number":316,"context_line":"* LisaLi (lixiaoy11, xiaoyan.li@intel.com)"},{"line_number":317,"context_line":"* Huang Zhiteng (winston-d, winston.d@gmail.com)"},{"line_number":318,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_7b51be0d","line":315,"updated":"2015-11-05 18:21:54.000000000","message":"What is \"xs\"?","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"2ff2b3c426ae2b3a2bf60131ba8f6fda2ec53edb","unresolved":false,"context_lines":[{"line_number":312,"context_line":"* Tom Barron (tbarron, tpb@dyncloud.net)"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"Other contributors:"},{"line_number":315,"context_line":"xs"},{"line_number":316,"context_line":"* LisaLi (lixiaoy11, xiaoyan.li@intel.com)"},{"line_number":317,"context_line":"* Huang Zhiteng (winston-d, winston.d@gmail.com)"},{"line_number":318,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_bb8c0680","line":315,"in_reply_to":"fa80f949_7b51be0d","updated":"2015-11-05 18:24:25.000000000","message":"Woops!  Thanks.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cd45a1577a146c096ba5b0734f3d19b2d563dc2c","unresolved":false,"context_lines":[{"line_number":343,"context_line":"  reflect code removed from the backup service to load volume manager,"},{"line_number":344,"context_line":"  run volume drivers, etc. and from the volume driver to run backup"},{"line_number":345,"context_line":"  and restore operations."},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"* Existing tempest tests should provide sufficient coverage to ensure"},{"line_number":348,"context_line":"  that current functionality does not regress.  Potentially new"},{"line_number":349,"context_line":"  multi-node tempest tests could be added to verify distributed"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_199a6c15","line":346,"updated":"2015-11-03 13:44:56.000000000","message":"We actually have very small coverage of backup functionalities in Tempest. Anyway that\u0027s not the focus of this spec.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"49e255edf2f224005a3f9ba2a3b9ef2616c8b4f6","unresolved":false,"context_lines":[{"line_number":343,"context_line":"  reflect code removed from the backup service to load volume manager,"},{"line_number":344,"context_line":"  run volume drivers, etc. and from the volume driver to run backup"},{"line_number":345,"context_line":"  and restore operations."},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"* Existing tempest tests should provide sufficient coverage to ensure"},{"line_number":348,"context_line":"  that current functionality does not regress.  Potentially new"},{"line_number":349,"context_line":"  multi-node tempest tests could be added to verify distributed"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_83675a24","line":346,"in_reply_to":"fa80f949_199a6c15","updated":"2015-11-05 17:40:46.000000000","message":"True.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cd45a1577a146c096ba5b0734f3d19b2d563dc2c","unresolved":false,"context_lines":[{"line_number":351,"context_line":""},{"line_number":352,"context_line":"Documentation Impact"},{"line_number":353,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":354,"context_line":""},{"line_number":355,"context_line":"None"},{"line_number":356,"context_line":""},{"line_number":357,"context_line":"References"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_b96c8009","line":354,"updated":"2015-11-03 13:44:56.000000000","message":"I do believe that we should update deployment documentation to reflect new deployment possibilities.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"49e255edf2f224005a3f9ba2a3b9ef2616c8b4f6","unresolved":false,"context_lines":[{"line_number":351,"context_line":""},{"line_number":352,"context_line":"Documentation Impact"},{"line_number":353,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":354,"context_line":""},{"line_number":355,"context_line":"None"},{"line_number":356,"context_line":""},{"line_number":357,"context_line":"References"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fa80f949_834e7a94","line":354,"in_reply_to":"fa80f949_b96c8009","updated":"2015-11-05 17:40:46.000000000","message":"Makes sense, I\u0027ll add to that effect.","commit_id":"3cc5380f4ff1d76734f4cfc0f891ec247f2cfbd0"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0ec5c0a5367dc01417dcf58a30ab37da1f731bf9","unresolved":false,"context_lines":[{"line_number":133,"context_line":"method was added that only allows for local attaches and the reference"},{"line_number":134,"context_line":"``lvm driver`` explicitly uses ``local_path`` when getting volumes"},{"line_number":135,"context_line":"for backup operations [5].  As part of the work implementing this"},{"line_number":136,"context_line":"blueprint spec, we will need to revisit snapshot attachment to allow"},{"line_number":137,"context_line":"for remote attaches.  Drivers that cannot support remote attach"},{"line_number":138,"context_line":"for snapshots will need to fall back to using temporary volumes"},{"line_number":139,"context_line":"instead."}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_724a9b57","line":136,"updated":"2015-11-13 11:04:12.000000000","message":"I would prefer exact solution for LVM described here, but \"we\u0027ll try and if it won\u0027t be possible we\u0027ll use temp volumes\" seems okay.","commit_id":"4075d0c3aeb217b8692cdd5e08569dc675928d61"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"a9a58db3353b78183d8a574a3f2f0d653077dde3","unresolved":false,"context_lines":[{"line_number":133,"context_line":"method was added that only allows for local attaches and the reference"},{"line_number":134,"context_line":"``lvm driver`` explicitly uses ``local_path`` when getting volumes"},{"line_number":135,"context_line":"for backup operations [5].  As part of the work implementing this"},{"line_number":136,"context_line":"blueprint spec, we will need to revisit snapshot attachment to allow"},{"line_number":137,"context_line":"for remote attaches.  Drivers that cannot support remote attach"},{"line_number":138,"context_line":"for snapshots will need to fall back to using temporary volumes"},{"line_number":139,"context_line":"instead."}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_68d107cd","line":136,"in_reply_to":"da85f559_724a9b57","updated":"2015-11-16 06:57:50.000000000","message":"Based on Yang Xing\u0027s comments in patch 2, we need to use temporary volume solution. \nThat is: \n* Create a temporary volume from the original volume\n* Backup the temporary volume\n* Clean up the temporary volume\n\nAnd note here that once lvm.attach_snapshot works, we can use temporary snapshot solution.","commit_id":"4075d0c3aeb217b8692cdd5e08569dc675928d61"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0ec5c0a5367dc01417dcf58a30ab37da1f731bf9","unresolved":false,"context_lines":[{"line_number":222,"context_line":"service process that handles the operation can clear that field at the"},{"line_number":223,"context_line":"end of the operation.  Cleanup at initialization can then be"},{"line_number":224,"context_line":"restricted to leftover objects that chain through their corresponding"},{"line_number":225,"context_line":"backup object to a ``host`` field matching oneself."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"For an example of how the ``host`` field will be set, consider"},{"line_number":228,"context_line":"a volume with a backend handled by volume service on node A where"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_cd3de6bc","line":225,"updated":"2015-11-13 11:04:12.000000000","message":"This seems safe to me - we\u0027ll be cleaning up only after our own failures.\n\nBut… How do we upgrade from previous scheme? Preferably in a live manner? We have major logic change here (if host is not NULL, don\u0027t touch a backup). Please evaluate if solution proposed below is sane:\n\nOn c-bak start if a backup is in ACTIVE state and I\u0027m an owner (my hostname matches backup\u0027s host column), then clear backup\u0027s host column.\n\nThis will require deployer to upgrade all his existing c-bak services, but once he\u0027ll complete that he should be able to scale his c-bak plane in any way he likes.","commit_id":"4075d0c3aeb217b8692cdd5e08569dc675928d61"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"6db97f456bc65a950eb6656b90743366d3554030","unresolved":false,"context_lines":[{"line_number":222,"context_line":"service process that handles the operation can clear that field at the"},{"line_number":223,"context_line":"end of the operation.  Cleanup at initialization can then be"},{"line_number":224,"context_line":"restricted to leftover objects that chain through their corresponding"},{"line_number":225,"context_line":"backup object to a ``host`` field matching oneself."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"For an example of how the ``host`` field will be set, consider"},{"line_number":228,"context_line":"a volume with a backend handled by volume service on node A where"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_e8dbe219","line":225,"in_reply_to":"ba8a016a_b671f0e2","updated":"2015-11-17 06:56:12.000000000","message":"Yes.\n\nThe problem is when to do the cleanup if we want to do it automatically.\n\nSo I suggest to ask the users run commands to do it if there are ongoing backups during decoupling backup service.\nCurrently changing state of backup only changes the state, we can add a parameter like \"--clean_volume_status\" to clean the corresponding volume\u0027s status.","commit_id":"4075d0c3aeb217b8692cdd5e08569dc675928d61"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"563d78f283137c65ba2736ac2dc0f3d52061037c","unresolved":false,"context_lines":[{"line_number":222,"context_line":"service process that handles the operation can clear that field at the"},{"line_number":223,"context_line":"end of the operation.  Cleanup at initialization can then be"},{"line_number":224,"context_line":"restricted to leftover objects that chain through their corresponding"},{"line_number":225,"context_line":"backup object to a ``host`` field matching oneself."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"For an example of how the ``host`` field will be set, consider"},{"line_number":228,"context_line":"a volume with a backend handled by volume service on node A where"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_b671f0e2","line":225,"in_reply_to":"da85f559_6b7ca172","updated":"2015-11-16 09:59:33.000000000","message":"Ah, I\u0027ve only considered working on status of backups and not volumes. Let\u0027s see… \n\nThis doesn\u0027t seem a problem to clean up volumes in backing-up/restoring states. We can simply do an RPC call to unattach it and switch it\u0027s state.\n\nIs this what you meant?","commit_id":"4075d0c3aeb217b8692cdd5e08569dc675928d61"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"a9a58db3353b78183d8a574a3f2f0d653077dde3","unresolved":false,"context_lines":[{"line_number":222,"context_line":"service process that handles the operation can clear that field at the"},{"line_number":223,"context_line":"end of the operation.  Cleanup at initialization can then be"},{"line_number":224,"context_line":"restricted to leftover objects that chain through their corresponding"},{"line_number":225,"context_line":"backup object to a ``host`` field matching oneself."},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"For an example of how the ``host`` field will be set, consider"},{"line_number":228,"context_line":"a volume with a backend handled by volume service on node A where"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_6b7ca172","line":225,"in_reply_to":"da85f559_cd3de6bc","updated":"2015-11-16 06:57:50.000000000","message":"Hi Michal/Tom, \n\nI think the point is how to clean the restoring/backing-up backups during the upgrade. \n\nIn Michal\u0027s method, after upgrading the existing c-bak services, if restoring/backing-up relaunch(some backups are doing) in this node, we only can move the back service to other nodes after backups complete.\n\nI think another way is to ask the users update the status of backups by themselves, and meanwhile, the status of related volumes need update.","commit_id":"4075d0c3aeb217b8692cdd5e08569dc675928d61"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0ec5c0a5367dc01417dcf58a30ab37da1f731bf9","unresolved":false,"context_lines":[{"line_number":232,"context_line":"backup using the backup service on node C, the ``host`` field for"},{"line_number":233,"context_line":"the backup object will be set to C."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Cleanup of associated volumes, temporary volumes, and temporary"},{"line_number":236,"context_line":"snapshots will be done via rpc to the appropriate volume service host."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Note that the backup object contains a ``volume_id`` field for the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da85f559_cd6e4696","line":235,"updated":"2015-11-13 11:04:12.000000000","message":"This should also be fine.","commit_id":"4075d0c3aeb217b8692cdd5e08569dc675928d61"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"5cd857076793d49c6ea67b7e6b1b65e5e72b32af","unresolved":false,"context_lines":[{"line_number":138,"context_line":"remote attach for snapshots will need to fall back to using temporary"},{"line_number":139,"context_line":"volumes instead [5]."},{"line_number":140,"context_line":"* Create a temporary volume from the original volume."},{"line_number":141,"context_line":"* Bacup the temporary volume."},{"line_number":142,"context_line":"* Clean up the temporary volume."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_eaad3832","line":141,"updated":"2015-11-24 17:28:38.000000000","message":"s/Bacup/Backup","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"c3793b65eb0dce4ab0a212ab34e7894205ba8ab5","unresolved":false,"context_lines":[{"line_number":138,"context_line":"remote attach for snapshots will need to fall back to using temporary"},{"line_number":139,"context_line":"volumes instead [5]."},{"line_number":140,"context_line":"* Create a temporary volume from the original volume."},{"line_number":141,"context_line":"* Bacup the temporary volume."},{"line_number":142,"context_line":"* Clean up the temporary volume."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_7d0c9994","line":141,"in_reply_to":"9a8ffd7b_eaad3832","updated":"2015-11-25 03:02:54.000000000","message":"Done","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"b413da5692418816ddf5f51579f3086ba598a03a","unresolved":false,"context_lines":[{"line_number":222,"context_line":""},{"line_number":223,"context_line":"The ``host`` field for a backup object will be set to the host for the"},{"line_number":224,"context_line":"backup service to which the backup operation is cast, and the backup"},{"line_number":225,"context_line":"service process that handles the operation can clear that field at the"},{"line_number":226,"context_line":"end of the operation.  Cleanup at initialization can then be"},{"line_number":227,"context_line":"restricted to leftover objects that chain through their corresponding"},{"line_number":228,"context_line":"backup object to a ``host`` field matching oneself."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_13943ae0","line":225,"updated":"2015-11-25 15:58:15.000000000","message":"Based on the discussion on IRC - we don\u0027t need to clear host field after we\u0027ll finish. We just need to acquire control over a backup using compare-and-swap DB operation (comparing host and status).","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"90bce1981ad9042eb2fc4f9c1918be37423faeec","unresolved":false,"context_lines":[{"line_number":222,"context_line":""},{"line_number":223,"context_line":"The ``host`` field for a backup object will be set to the host for the"},{"line_number":224,"context_line":"backup service to which the backup operation is cast, and the backup"},{"line_number":225,"context_line":"service process that handles the operation can clear that field at the"},{"line_number":226,"context_line":"end of the operation.  Cleanup at initialization can then be"},{"line_number":227,"context_line":"restricted to leftover objects that chain through their corresponding"},{"line_number":228,"context_line":"backup object to a ``host`` field matching oneself."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_fecd9a5d","line":225,"in_reply_to":"9a8ffd7b_13943ae0","updated":"2015-11-26 07:19:25.000000000","message":"Done","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"5cd857076793d49c6ea67b7e6b1b65e5e72b32af","unresolved":false,"context_lines":[{"line_number":235,"context_line":"backup using the backup service on node C, the ``host`` field for"},{"line_number":236,"context_line":"the backup object will be set to C."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Cleanup of associated volumes, temporary volumes, and temporary"},{"line_number":239,"context_line":"snapshots will be done via rpc to the appropriate volume service host."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"Note that the backup object contains a ``volume_id`` field for the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_00209d9a","line":238,"updated":"2015-11-24 17:28:38.000000000","message":"How do we know which c-bak was working on a volume once we decouple c-bak and c-vol? Right now we\u0027re cleaning up only local volumes, but that would not work with solution described.\n\nI would be fine if we\u0027ll skip automatic cleanup of stalled volumes on c-bak startup, but this would need to be mentioned in the spec I think.","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"c3793b65eb0dce4ab0a212ab34e7894205ba8ab5","unresolved":false,"context_lines":[{"line_number":235,"context_line":"backup using the backup service on node C, the ``host`` field for"},{"line_number":236,"context_line":"the backup object will be set to C."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Cleanup of associated volumes, temporary volumes, and temporary"},{"line_number":239,"context_line":"snapshots will be done via rpc to the appropriate volume service host."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"Note that the backup object contains a ``volume_id`` field for the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_bd1201b3","line":238,"in_reply_to":"9a8ffd7b_00209d9a","updated":"2015-11-25 03:02:54.000000000","message":"We needs to cleanup volumes related to restoring/backing backups. \n\nWhen backup is in backing status, we can find the corresponding volumes by volume_id, temp_volume_id, and temp_snapshot_id in the db table backups. Please see L241-244.\n\nWhen backup is in restoring status, please see L: we need to add a field to record the volume. Please see L214-215.","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"fc7d71cf7fc802a5c991da8a91e19ba8e8136232","unresolved":false,"context_lines":[{"line_number":235,"context_line":"backup using the backup service on node C, the ``host`` field for"},{"line_number":236,"context_line":"the backup object will be set to C."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Cleanup of associated volumes, temporary volumes, and temporary"},{"line_number":239,"context_line":"snapshots will be done via rpc to the appropriate volume service host."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"Note that the backup object contains a ``volume_id`` field for the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_a25abe81","line":238,"in_reply_to":"9a8ffd7b_bd1201b3","updated":"2015-11-25 15:22:07.000000000","message":"Oh, sorry, you\u0027re right. Please disregard my comment.","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"5cd857076793d49c6ea67b7e6b1b65e5e72b32af","unresolved":false,"context_lines":[{"line_number":245,"context_line":"will need to add this field in order to determine orphaned"},{"line_number":246,"context_line":"restore-operation volumes."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"During backups creating or restoring progress, if backup service"},{"line_number":249,"context_line":"is scaling to other host and when the backup service starts, the backups"},{"line_number":250,"context_line":"can not be cleaned. For such cases, it needs manual recovery. Backups"},{"line_number":251,"context_line":"and volumes can be recovered by resetting status."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_ad73ca4c","line":248,"updated":"2015-11-24 17:28:38.000000000","message":"I don\u0027t quite understand that. paragraph. We\u0027re setting backup\u0027s host column only if we\u0027re working on a backup. Change (atomic) of it\u0027s state will prevent other actions from being accepted for that backup. At the c-bak start if host column of a backup is set to our hostname, then we can be sure that noone else is working on it and we can safely clean up the status. Am I missing something here?\n\nAs I think we can automatically clean up backups on startup it\u0027s harder with volumes. Once c-bak is decoupled from c-vol we don\u0027t know who is working on backing up or restoring a volume…","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"c3793b65eb0dce4ab0a212ab34e7894205ba8ab5","unresolved":false,"context_lines":[{"line_number":245,"context_line":"will need to add this field in order to determine orphaned"},{"line_number":246,"context_line":"restore-operation volumes."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"During backups creating or restoring progress, if backup service"},{"line_number":249,"context_line":"is scaling to other host and when the backup service starts, the backups"},{"line_number":250,"context_line":"can not be cleaned. For such cases, it needs manual recovery. Backups"},{"line_number":251,"context_line":"and volumes can be recovered by resetting status."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_fd772900","line":248,"in_reply_to":"9a8ffd7b_ad73ca4c","updated":"2015-11-25 03:02:54.000000000","message":"Here I mean c-bak service is moved to other host. So when it starts up, the c-bak service host will be different. As a result, the restoring/backing backups which are stalled as service down in original host can\u0027t be  cleaned.","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"90bce1981ad9042eb2fc4f9c1918be37423faeec","unresolved":false,"context_lines":[{"line_number":245,"context_line":"will need to add this field in order to determine orphaned"},{"line_number":246,"context_line":"restore-operation volumes."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"During backups creating or restoring progress, if backup service"},{"line_number":249,"context_line":"is scaling to other host and when the backup service starts, the backups"},{"line_number":250,"context_line":"can not be cleaned. For such cases, it needs manual recovery. Backups"},{"line_number":251,"context_line":"and volumes can be recovered by resetting status."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_bea9527d","line":248,"in_reply_to":"9a8ffd7b_e24a0683","updated":"2015-11-26 07:19:25.000000000","message":"Yeah, this only occurs when the services went down by error.","commit_id":"22287efd6909b1830b158e1c0884020699731df4"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"fc7d71cf7fc802a5c991da8a91e19ba8e8136232","unresolved":false,"context_lines":[{"line_number":245,"context_line":"will need to add this field in order to determine orphaned"},{"line_number":246,"context_line":"restore-operation volumes."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"During backups creating or restoring progress, if backup service"},{"line_number":249,"context_line":"is scaling to other host and when the backup service starts, the backups"},{"line_number":250,"context_line":"can not be cleaned. For such cases, it needs manual recovery. Backups"},{"line_number":251,"context_line":"and volumes can be recovered by resetting status."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a8ffd7b_e24a0683","line":248,"in_reply_to":"9a8ffd7b_fd772900","updated":"2015-11-25 15:22:07.000000000","message":"This shouldn\u0027t be needed if we\u0027re moving the c-bak service in a right way. Basically we should stop it in a graceful manner. I believe the services are waiting for their greenthreads to end, so we should be able to reliably close the service without interrupting any operation.","commit_id":"22287efd6909b1830b158e1c0884020699731df4"}]}
