)]}'
{"specs/liberty/cinder-volume-active-active-ha.rst":[{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"758b4203a69536461209b5da1c3e6a7614cf70f0","unresolved":false,"context_lines":[{"line_number":14,"context_line":"This is due to several reasons:"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"* Local file locks in manager that protect resources used for example for"},{"line_number":17,"context_line":"  creation of volumes from deleting."},{"line_number":18,"context_line":"* Local file locks in RemoteFS (and dependent) drivers. These locks are used"},{"line_number":19,"context_line":"  to block operations that cannot run concurrently (like taking a snapshot)."},{"line_number":20,"context_line":"* Non-atomic state transitions in c-api that may cause race conditions."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_ebe6afaf","line":17,"updated":"2015-07-16 21:59:45.000000000","message":"I get that file locks protect resources, but your example is not clear.","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"566e008a0f4772a7a8b600a5100d65ea7db2ba50","unresolved":false,"context_lines":[{"line_number":14,"context_line":"This is due to several reasons:"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"* Local file locks in manager that protect resources used for example for"},{"line_number":17,"context_line":"  creation of volumes from deleting."},{"line_number":18,"context_line":"* Local file locks in RemoteFS (and dependent) drivers. These locks are used"},{"line_number":19,"context_line":"  to block operations that cannot run concurrently (like taking a snapshot)."},{"line_number":20,"context_line":"* Non-atomic state transitions in c-api that may cause race conditions."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_0c1748ee","line":17,"in_reply_to":"9a41bdd9_ebe6afaf","updated":"2015-07-17 13:42:04.000000000","message":"Done","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"758b4203a69536461209b5da1c3e6a7614cf70f0","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Problem description"},{"line_number":25,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"Currently there can be only one active cinder-volume service in the whole"},{"line_number":28,"context_line":"stack. This means that to achieve high availability of it we need external"},{"line_number":29,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":30,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_0ba3d350","line":27,"updated":"2015-07-16 21:59:45.000000000","message":"\"in the whole stack\"... do you mean per-backend?","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"566e008a0f4772a7a8b600a5100d65ea7db2ba50","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Problem description"},{"line_number":25,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"Currently there can be only one active cinder-volume service in the whole"},{"line_number":28,"context_line":"stack. This means that to achieve high availability of it we need external"},{"line_number":29,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":30,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_cc3080a1","line":27,"in_reply_to":"9a41bdd9_0ba3d350","updated":"2015-07-17 13:42:04.000000000","message":"Good catch, actually you cannot run multiple c-vol\u0027s per backend.","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"758b4203a69536461209b5da1c3e6a7614cf70f0","unresolved":false,"context_lines":[{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"},{"line_number":35,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"Operator would want to deploy multiple cinder-volume services to increase"},{"line_number":38,"context_line":"reliablity, HA and scalability of his cloud. Redundant cinder-volume can"},{"line_number":39,"context_line":"address these needs. Cloud user will get a better cloud."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_cb088b30","line":36,"updated":"2015-07-16 21:59:45.000000000","message":"I think you could add some specifics here, i.e.\nMultiple Active-active c-vol attached to the same storage for performance (higher throughput of requests)\nMultiple A-A c-vol for resiliency to Hardware failure.","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"566e008a0f4772a7a8b600a5100d65ea7db2ba50","unresolved":false,"context_lines":[{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Use Cases"},{"line_number":35,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"Operator would want to deploy multiple cinder-volume services to increase"},{"line_number":38,"context_line":"reliablity, HA and scalability of his cloud. Redundant cinder-volume can"},{"line_number":39,"context_line":"address these needs. Cloud user will get a better cloud."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_8c2a782b","line":36,"in_reply_to":"9a41bdd9_cb088b30","updated":"2015-07-17 13:42:04.000000000","message":"Done","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"a4770617cccf32d518a5afc3ff8fd45907bd03e5","unresolved":false,"context_lines":[{"line_number":63,"context_line":"Cinder API would response with VolumeIsBusy exception instead of queuing"},{"line_number":64,"context_line":"the request, so this is big change in API behavior and would probably requre a"},{"line_number":65,"context_line":"V3 API version to prevent from breaking scripts and clients that users and"},{"line_number":66,"context_line":"operators are using."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"To solve the problem with locks in RemoteFS-based volume drivers we may simply"},{"line_number":69,"context_line":"deprecate them and make operators switch to use lock-free drivers. This may be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_fa78b62d","line":66,"updated":"2015-07-16 22:33:07.000000000","message":"I still think this is the right approach to take long term.   by delaying/queuing the command into Cinder via tooz/locks for a volume that IS busy, seems like the wrong approach.","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"566e008a0f4772a7a8b600a5100d65ea7db2ba50","unresolved":false,"context_lines":[{"line_number":63,"context_line":"Cinder API would response with VolumeIsBusy exception instead of queuing"},{"line_number":64,"context_line":"the request, so this is big change in API behavior and would probably requre a"},{"line_number":65,"context_line":"V3 API version to prevent from breaking scripts and clients that users and"},{"line_number":66,"context_line":"operators are using."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"To solve the problem with locks in RemoteFS-based volume drivers we may simply"},{"line_number":69,"context_line":"deprecate them and make operators switch to use lock-free drivers. This may be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_2ca32ca0","line":66,"in_reply_to":"9a41bdd9_fa78b62d","updated":"2015-07-17 13:42:04.000000000","message":"I\u0027ve added a comment for that in the next patchset. Generally I agree, but solution proposed by this blueprint have chances to get into Liberty, and we don\u0027t have any estimate when we\u0027ll be able to develop VolumeIsBusy-based one.","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"758b4203a69536461209b5da1c3e6a7614cf70f0","unresolved":false,"context_lines":[{"line_number":98,"context_line":"Performance Impact"},{"line_number":99,"context_line":"------------------"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Using an external service for locking will affect the performance. Also having"},{"line_number":102,"context_line":"a heartbeat thread may add a tiny performance overhead."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_e6c826ab","line":101,"updated":"2015-07-16 21:59:45.000000000","message":"Is there a POC of this solution and some performance data? It seems that someone had posted perf info...","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"566e008a0f4772a7a8b600a5100d65ea7db2ba50","unresolved":false,"context_lines":[{"line_number":98,"context_line":"Performance Impact"},{"line_number":99,"context_line":"------------------"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Using an external service for locking will affect the performance. Also having"},{"line_number":102,"context_line":"a heartbeat thread may add a tiny performance overhead."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_2cd1ecfb","line":101,"in_reply_to":"9a41bdd9_e6c826ab","updated":"2015-07-17 13:42:04.000000000","message":"I\u0027ve added link to Gorka\u0027s tests. These weren\u0027t focused on performance, but have some numbers on that.","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"758b4203a69536461209b5da1c3e6a7614cf70f0","unresolved":false,"context_lines":[{"line_number":109,"context_line":"* backend_url\u003dfile://$state_path - Tooz backend connection string"},{"line_number":110,"context_line":"* heartbeat\u003d1.0 - number of seconds between heartbeats for distributed"},{"line_number":111,"context_line":"  coordination"},{"line_number":112,"context_line":"* inital_reconnect_backoff\u003d0.1 - number of seconds to wait after failed"},{"line_number":113,"context_line":"  reconnection"},{"line_number":114,"context_line":"* max_reconnect_backoff\u003d60.0 - Maximum number of seconds between sequential"},{"line_number":115,"context_line":"  reconnection retries"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_c66e4296","line":112,"updated":"2015-07-16 21:59:45.000000000","message":"what reconnect is this referring to? I\u0027m assuming is has something to do with tooz?","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"566e008a0f4772a7a8b600a5100d65ea7db2ba50","unresolved":false,"context_lines":[{"line_number":109,"context_line":"* backend_url\u003dfile://$state_path - Tooz backend connection string"},{"line_number":110,"context_line":"* heartbeat\u003d1.0 - number of seconds between heartbeats for distributed"},{"line_number":111,"context_line":"  coordination"},{"line_number":112,"context_line":"* inital_reconnect_backoff\u003d0.1 - number of seconds to wait after failed"},{"line_number":113,"context_line":"  reconnection"},{"line_number":114,"context_line":"* max_reconnect_backoff\u003d60.0 - Maximum number of seconds between sequential"},{"line_number":115,"context_line":"  reconnection retries"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_8c1038aa","line":112,"in_reply_to":"9a41bdd9_c66e4296","updated":"2015-07-17 13:42:04.000000000","message":"You\u0027re right, it\u0027s for reconnecting to Tooz backend in case of connection failure.","commit_id":"d84c264a8faec760c5a32990638d59e271100ae3"},{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"d03cb77f342241ee526a1db53631ec9d3a2f8083","unresolved":false,"context_lines":[{"line_number":68,"context_line":"Cinder API would response with VolumeIsBusy exception instead of queuing"},{"line_number":69,"context_line":"the request, so this is big change in API behavior and would probably require a"},{"line_number":70,"context_line":"V3 API version to prevent from breaking scripts and clients that users and"},{"line_number":71,"context_line":"operators are using. This is probably a desired long-term solution."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"To solve the problem with locks in RemoteFS-based volume drivers we may simply"},{"line_number":74,"context_line":"deprecate them and make operators switch to use lock-free drivers. This may be"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a50d1a3_b1e6bb42","line":71,"updated":"2015-07-24 15:33:40.000000000","message":"So is the replacement of locks and return of VolumeIsBusy a long-term solution that provides the same functionality as the use of tooz as a DLM? In other words, once we have VolumeIsBusy returned, Nova using \u0027try:\u0027 blocks, and other scripts aware of the possible exception and handling them, will tooz/DLM go away?\nI\u0027m sceptical of a solution as weighty as DLM that will ultimately be replaced by something else.","commit_id":"a0d7e4c9e069031364f022df2212e40cc674db09"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"fcd498caaaac2e53252dde384fb06286a0d4d9fa","unresolved":false,"context_lines":[{"line_number":68,"context_line":"Cinder API would response with VolumeIsBusy exception instead of queuing"},{"line_number":69,"context_line":"the request, so this is big change in API behavior and would probably require a"},{"line_number":70,"context_line":"V3 API version to prevent from breaking scripts and clients that users and"},{"line_number":71,"context_line":"operators are using. This is probably a desired long-term solution."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"To solve the problem with locks in RemoteFS-based volume drivers we may simply"},{"line_number":74,"context_line":"deprecate them and make operators switch to use lock-free drivers. This may be"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a50d1a3_2cd014c9","line":71,"in_reply_to":"3a50d1a3_b1e6bb42","updated":"2015-07-24 15:59:35.000000000","message":"No. If we want to keep c-vol A/A HA abilities after implementing VolumeIsBusy, we would still need some DLM for locks inside some drivers. For example in all drivers inheriting from RemoteFSSnapDriver we have a lot of operations locked on a volume_id [1]. These cannot go in parallel. I was able to confirm that making snapshots in parallel in GlusterFS drivers ends up with 50% of them actually non-existing.\n\nAccording to this year\u0027s OpenStack Users Survey GlusterFS drivers is running in 9% of OpenStack deployments. Along goes NFS with 5% share - it has the same issues [2].\n\nThis means that DLM will be eventually needed to achieve c-vol A/A HA abilities.\n\n[1] https://github.com/openstack/cinder/blob/master/cinder/volume/drivers/remotefs.py#L1420-L1448\n[2] http://a4.res.cloudinary.com/hqq9ey1mh/image/upload/c_limit,w_793/v1431734712/gbb9s6wmsqdfcsoy9m7q.jpg","commit_id":"a0d7e4c9e069031364f022df2212e40cc674db09"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"75beff827f129913ce413b21a6a417d7153a3a7e","unresolved":false,"context_lines":[{"line_number":17,"context_line":"  example when creating a volume A from volume B a lock on B is created to"},{"line_number":18,"context_line":"  protect B from getting deleted during creation of A. They are not shared"},{"line_number":19,"context_line":"  between different nodes of a cluster."},{"line_number":20,"context_line":"* Local file locks in drivers, like RemoteFS (and dependent) drivers. These"},{"line_number":21,"context_line":"  locks are used to block operations that cannot run concurrently (like taking"},{"line_number":22,"context_line":"  a snapshot)."},{"line_number":23,"context_line":"* Volume nodes consider that they are the only ones accessing configured"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a50d1a3_b6492362","line":20,"updated":"2015-07-29 10:36:03.000000000","message":"Just FYI, some of the RemoteFS dependent drivers override the methods with the lock decorators with undecorated methods.  I suspect this doesn\u0027t affect your claim about the proportion of our drivers that do this kind of locking since the one instance of this I know (NetApp) does get broken out separately from NFS in the study you cite.","commit_id":"34a3d17caf950302f1600cfeac9d1ef3cd49eca9"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"75beff827f129913ce413b21a6a417d7153a3a7e","unresolved":false,"context_lines":[{"line_number":75,"context_line":"everything will work as today."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"For Active-Active configuration where operator would want to run multiple"},{"line_number":78,"context_line":"cinder-volume services, he would need to deploy a DLM and configure Tooz_"},{"line_number":79,"context_line":"backend service to use that DLM by setting the appropriate configuration"},{"line_number":80,"context_line":"parameters in cinder.conf."},{"line_number":81,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a50d1a3_536eb100","line":78,"updated":"2015-07-29 10:36:03.000000000","message":"he or she","commit_id":"34a3d17caf950302f1600cfeac9d1ef3cd49eca9"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"75beff827f129913ce413b21a6a417d7153a3a7e","unresolved":false,"context_lines":[{"line_number":79,"context_line":"backend service to use that DLM by setting the appropriate configuration"},{"line_number":80,"context_line":"parameters in cinder.conf."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Currently most reliable Tooz_ DLM backends are ZooKeeper and Redis, and both"},{"line_number":83,"context_line":"provide a mechanisms that allow locks from failed nodes to be dropped"},{"line_number":84,"context_line":"automatically. For this, Redis requires sending periodical heartbeats, that"},{"line_number":85,"context_line":"will prevent locks from expiring, so cinder-volume manager will start a new"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a50d1a3_33420569","line":82,"updated":"2015-07-29 10:36:03.000000000","message":"Suggest: \u0027Currently the most reliable ...\u0027","commit_id":"34a3d17caf950302f1600cfeac9d1ef3cd49eca9"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"75beff827f129913ce413b21a6a417d7153a3a7e","unresolved":false,"context_lines":[{"line_number":80,"context_line":"parameters in cinder.conf."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Currently most reliable Tooz_ DLM backends are ZooKeeper and Redis, and both"},{"line_number":83,"context_line":"provide a mechanisms that allow locks from failed nodes to be dropped"},{"line_number":84,"context_line":"automatically. For this, Redis requires sending periodical heartbeats, that"},{"line_number":85,"context_line":"will prevent locks from expiring, so cinder-volume manager will start a new"},{"line_number":86,"context_line":"thread that will send them. This will be a regular thread (non-eventlet) to be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a50d1a3_d3516131","line":83,"updated":"2015-07-29 10:36:03.000000000","message":"s/allow/allows","commit_id":"34a3d17caf950302f1600cfeac9d1ef3cd49eca9"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ba6dd8f5f789dddafbf7ecc3aa1e20a0bf3de7d8","unresolved":false,"context_lines":[{"line_number":80,"context_line":"parameters in cinder.conf."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Currently most reliable Tooz_ DLM backends are ZooKeeper and Redis, and both"},{"line_number":83,"context_line":"provide a mechanisms that allow locks from failed nodes to be dropped"},{"line_number":84,"context_line":"automatically. For this, Redis requires sending periodical heartbeats, that"},{"line_number":85,"context_line":"will prevent locks from expiring, so cinder-volume manager will start a new"},{"line_number":86,"context_line":"thread that will send them. This will be a regular thread (non-eventlet) to be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a50d1a3_a360f501","line":83,"in_reply_to":"3a50d1a3_d3516131","updated":"2015-07-29 15:59:59.000000000","message":"Actually allow was ok, it was \"a\" that needed to be removed.  Because they have different mechanisms (plural).","commit_id":"34a3d17caf950302f1600cfeac9d1ef3cd49eca9"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"2ab5afe5293a801b21ef4bd3d842ae32789320b9","unresolved":false,"context_lines":[{"line_number":89,"context_line":"Locks will be changed in Volume Manager as well as in drivers, and current"},{"line_number":90,"context_line":"operations will be reviewed to add missing resource locks."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"Alternatives"},{"line_number":93,"context_line":"------------"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"To achieve A/A HA we can replace locks in cinder-volume manager by doing atomic"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1a4dcd0f_a7953889","line":92,"updated":"2015-08-08 10:37:13.000000000","message":"I\u0027ve read the \"more complex way\" http://gorka.eguileor.com/a-cinder-road-to-activeactive-ha/ and I believe the alternative - citing - \" Even if we decided to go with a DB solution we would be basically using a DLM; atomic changes to the DB are locks on a row at the node level and at the cluster level requires consensus for the update.\" should be given more attention in this spec.\n\nSo, is really an alternative to rely on atomic changes to the DB and delegate the distributed consensus to its (DB) built-in capabilities?","commit_id":"3650e49d19d971650cca25a661e5f84ad0550b78"}],"specs/mitaka/ha-aa-tooz-locks.rst":[{"author":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"change_message_id":"59ce9ed8ea4ad6d57a7355a6371b978e4f5c9b65","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"},{"line_number":36,"context_line":"preventing that, because the lock won\u0027t be shared between different c-vol"},{"line_number":37,"context_line":"services running on different hosts. This may casue multiple issues, resulting"},{"line_number":38,"context_line":"even in data loss."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":10,"id":"da85f559_d6a68852","line":37,"updated":"2015-11-10 14:15:25.000000000","message":"s/casue/cause/","commit_id":"50a3145ba4715fd1a3bc2ac242630472411d014a"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"716eb40f382cae64e5b3a7a9f8ebdfdd8659511d","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"},{"line_number":36,"context_line":"preventing that, because the lock won\u0027t be shared between different c-vol"},{"line_number":37,"context_line":"services running on different hosts. This may casue multiple issues, resulting"},{"line_number":38,"context_line":"even in data loss."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":10,"id":"da85f559_9c09ffdf","line":37,"in_reply_to":"da85f559_d6a68852","updated":"2015-11-10 14:33:53.000000000","message":"Done","commit_id":"50a3145ba4715fd1a3bc2ac242630472411d014a"},{"author":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"change_message_id":"59ce9ed8ea4ad6d57a7355a6371b978e4f5c9b65","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Operator would want to deploy multiple cinder-volume services. The motivation"},{"line_number":44,"context_line":"may be to run multiple cinder-volume attached to a single storage backend to"},{"line_number":45,"context_line":"increase throuput of request and therefore perfomance. Another advantage is"},{"line_number":46,"context_line":"increased resiliency to hardware failure when running multiple instances of the"},{"line_number":47,"context_line":"service."},{"line_number":48,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"da85f559_76323c12","line":45,"updated":"2015-11-10 14:15:25.000000000","message":"s/throuput/throughput/\ns/perfomance/performance/","commit_id":"50a3145ba4715fd1a3bc2ac242630472411d014a"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"716eb40f382cae64e5b3a7a9f8ebdfdd8659511d","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Operator would want to deploy multiple cinder-volume services. The motivation"},{"line_number":44,"context_line":"may be to run multiple cinder-volume attached to a single storage backend to"},{"line_number":45,"context_line":"increase throuput of request and therefore perfomance. Another advantage is"},{"line_number":46,"context_line":"increased resiliency to hardware failure when running multiple instances of the"},{"line_number":47,"context_line":"service."},{"line_number":48,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"da85f559_3c101301","line":45,"in_reply_to":"da85f559_76323c12","updated":"2015-11-10 14:33:53.000000000","message":"Done","commit_id":"50a3145ba4715fd1a3bc2ac242630472411d014a"},{"author":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"change_message_id":"59ce9ed8ea4ad6d57a7355a6371b978e4f5c9b65","unresolved":false,"context_lines":[{"line_number":69,"context_line":"and Redis."},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Redis requires sending periodical heartbeats, so cinder-volume manager will"},{"line_number":72,"context_line":"start a new thread that will send them. This will be a regular thread"},{"line_number":73,"context_line":"(non-eventlet) to be safe from a situations when eventlet greenthreads are"},{"line_number":74,"context_line":"blocked."},{"line_number":75,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"da85f559_79754d96","line":72,"updated":"2015-11-10 14:15:25.000000000","message":"So just to be clear, even when tooz will be used as just an abstraction for local locks, we will still spin up a new dedicated thread for heartbeats?","commit_id":"50a3145ba4715fd1a3bc2ac242630472411d014a"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"716eb40f382cae64e5b3a7a9f8ebdfdd8659511d","unresolved":false,"context_lines":[{"line_number":69,"context_line":"and Redis."},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Redis requires sending periodical heartbeats, so cinder-volume manager will"},{"line_number":72,"context_line":"start a new thread that will send them. This will be a regular thread"},{"line_number":73,"context_line":"(non-eventlet) to be safe from a situations when eventlet greenthreads are"},{"line_number":74,"context_line":"blocked."},{"line_number":75,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"da85f559_1c1d0f0f","line":72,"in_reply_to":"da85f559_79754d96","updated":"2015-11-10 14:33:53.000000000","message":"Ah, good catch. We should spin it only when it\u0027s needed.","commit_id":"50a3145ba4715fd1a3bc2ac242630472411d014a"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"78647dc47851d37978d0ceb42e0b829ecb253c44","unresolved":false,"context_lines":[{"line_number":135,"context_line":"* backend_url\u003dfile://$state_path - Tooz backend connection string."},{"line_number":136,"context_line":"* heartbeat\u003d1.0 - number of seconds between heartbeats for distributed"},{"line_number":137,"context_line":"  coordination."},{"line_number":138,"context_line":"* inital_reconnect_backoff\u003d0.1 - number of seconds to wait after failed"},{"line_number":139,"context_line":"  reconnection to Tooz backend."},{"line_number":140,"context_line":"* max_reconnect_backoff\u003d60.0 - Maximum number of seconds between sequential"},{"line_number":141,"context_line":"  reconnection retries to Tooz backend."}],"source_content_type":"text/x-rst","patch_set":11,"id":"da85f559_e5e8c318","line":138,"updated":"2015-11-16 04:09:21.000000000","message":"s/inital/initial","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"584d1fadab8fb8c32be9453f9817ee5f1acdb2c3","unresolved":false,"context_lines":[{"line_number":135,"context_line":"* backend_url\u003dfile://$state_path - Tooz backend connection string."},{"line_number":136,"context_line":"* heartbeat\u003d1.0 - number of seconds between heartbeats for distributed"},{"line_number":137,"context_line":"  coordination."},{"line_number":138,"context_line":"* inital_reconnect_backoff\u003d0.1 - number of seconds to wait after failed"},{"line_number":139,"context_line":"  reconnection to Tooz backend."},{"line_number":140,"context_line":"* max_reconnect_backoff\u003d60.0 - Maximum number of seconds between sequential"},{"line_number":141,"context_line":"  reconnection retries to Tooz backend."}],"source_content_type":"text/x-rst","patch_set":11,"id":"ba8a016a_8a63261a","line":138,"in_reply_to":"da85f559_e5e8c318","updated":"2015-11-17 10:13:44.000000000","message":"Done","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"78647dc47851d37978d0ceb42e0b829ecb253c44","unresolved":false,"context_lines":[{"line_number":167,"context_line":"----------"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Tooz locking implementation in Cinder (done)."},{"line_number":170,"context_line":"* Switch current locks to use Tooz implementation."},{"line_number":171,"context_line":"* Add DevStack patches to set up a CI testing Cinder with Redis as Tooz"},{"line_number":172,"context_line":"  backend."},{"line_number":173,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"da85f559_25becbff","line":170,"updated":"2015-11-16 04:09:21.000000000","message":"Are you going to switch locks in drivers too or will that be driver maintainer\u0027s job to make the switch?","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"584d1fadab8fb8c32be9453f9817ee5f1acdb2c3","unresolved":false,"context_lines":[{"line_number":167,"context_line":"----------"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"* Tooz locking implementation in Cinder (done)."},{"line_number":170,"context_line":"* Switch current locks to use Tooz implementation."},{"line_number":171,"context_line":"* Add DevStack patches to set up a CI testing Cinder with Redis as Tooz"},{"line_number":172,"context_line":"  backend."},{"line_number":173,"context_line":""}],"source_content_type":"text/x-rst","patch_set":11,"id":"ba8a016a_0ab1768f","line":170,"in_reply_to":"da85f559_25becbff","updated":"2015-11-17 10:13:44.000000000","message":"We had some base work done [1]. We can probably start with splitting and rebasing it and asking maintainers for approvals.\n\n[1] https://review.openstack.org/#/c/185646/","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"78647dc47851d37978d0ceb42e0b829ecb253c44","unresolved":false,"context_lines":[{"line_number":196,"context_line":".. _Openstack Users Survey: http://www.openstack.org/assets/survey/Public-User-Survey-Report.pdf"},{"line_number":197,"context_line":".. _performance tests done by geguileo: https://github.com/Akrog/test-cinder-atomic-states"},{"line_number":198,"context_line":".. _session about allowing projects to have a hard requirement on DLM software: https://etherpad.openstack.org/p/mitaka-cross-project-dlm"},{"line_number":199,"context_line":".. _his: https://review.openstack.org/#/c/237602/"},{"line_number":200,"context_line":".. _specs: https://review.openstack.org/#/c/237604/"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"* https://etherpad.openstack.org/p/cinder-active-active-vol-service-issues"}],"source_content_type":"text/x-rst","patch_set":11,"id":"da85f559_8546971f","line":199,"updated":"2015-11-16 04:09:21.000000000","message":"\"his\"?","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"584d1fadab8fb8c32be9453f9817ee5f1acdb2c3","unresolved":false,"context_lines":[{"line_number":196,"context_line":".. _Openstack Users Survey: http://www.openstack.org/assets/survey/Public-User-Survey-Report.pdf"},{"line_number":197,"context_line":".. _performance tests done by geguileo: https://github.com/Akrog/test-cinder-atomic-states"},{"line_number":198,"context_line":".. _session about allowing projects to have a hard requirement on DLM software: https://etherpad.openstack.org/p/mitaka-cross-project-dlm"},{"line_number":199,"context_line":".. _his: https://review.openstack.org/#/c/237602/"},{"line_number":200,"context_line":".. _specs: https://review.openstack.org/#/c/237604/"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"* https://etherpad.openstack.org/p/cinder-active-active-vol-service-issues"}],"source_content_type":"text/x-rst","patch_set":11,"id":"ba8a016a_ca4e0ec1","line":199,"in_reply_to":"da85f559_8546971f","updated":"2015-11-17 10:13:44.000000000","message":"That\u0027s rst syntax for links [1].\n\n[1] http://docutils.sourceforge.net/docs/user/rst/quickref.html#external-hyperlink-targets","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"ca9324104c6fa0ce64ba86855abe32826334dbdf","unresolved":false,"context_lines":[{"line_number":201,"context_line":""},{"line_number":202,"context_line":"* https://etherpad.openstack.org/p/cinder-active-active-vol-service-issues"},{"line_number":203,"context_line":"* http://lists.openstack.org/pipermail/openstack-dev/2015-June/068151.html"},{"line_number":204,"context_line":"* https://review.openstack.org/#/c/183537/"}],"source_content_type":"text/x-rst","patch_set":11,"id":"da85f559_aaf57e57","line":204,"updated":"2015-11-10 21:09:56.000000000","message":"Might be nice to have a reference to the recorded discussion.\n\nhttps://www.youtube.com/watch?v\u003dFs9LC_sjnRM","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"584d1fadab8fb8c32be9453f9817ee5f1acdb2c3","unresolved":false,"context_lines":[{"line_number":201,"context_line":""},{"line_number":202,"context_line":"* https://etherpad.openstack.org/p/cinder-active-active-vol-service-issues"},{"line_number":203,"context_line":"* http://lists.openstack.org/pipermail/openstack-dev/2015-June/068151.html"},{"line_number":204,"context_line":"* https://review.openstack.org/#/c/183537/"}],"source_content_type":"text/x-rst","patch_set":11,"id":"ba8a016a_caaa8e19","line":204,"in_reply_to":"da85f559_aaf57e57","updated":"2015-11-17 10:13:44.000000000","message":"Done","commit_id":"4ce372797279e185672d736dad355d6197787ef7"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":26,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Currently there can be only one active cinder-volume service per volume"},{"line_number":29,"context_line":"backend. This means that to achieve high availability of it we need external"},{"line_number":30,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":31,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"},{"line_number":32,"context_line":"completely unscalable, because operator cannot just spin up additional service"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_b7caf59f","line":29,"updated":"2015-11-17 17:49:08.000000000","message":"nit: s/need external/need an external","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":26,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Currently there can be only one active cinder-volume service per volume"},{"line_number":29,"context_line":"backend. This means that to achieve high availability of it we need external"},{"line_number":30,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":31,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"},{"line_number":32,"context_line":"completely unscalable, because operator cannot just spin up additional service"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_8a143ca0","line":29,"in_reply_to":"ba8a016a_b7caf59f","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":29,"context_line":"backend. This means that to achieve high availability of it we need external"},{"line_number":30,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":31,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"},{"line_number":32,"context_line":"completely unscalable, because operator cannot just spin up additional service"},{"line_number":33,"context_line":"if load is high and need to rely on having just one instance."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_b7b435f2","line":32,"updated":"2015-11-17 17:49:08.000000000","message":"nit: s/up/up an\n\nor\n\ns/service/services","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":29,"context_line":"backend. This means that to achieve high availability of it we need external"},{"line_number":30,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":31,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"},{"line_number":32,"context_line":"completely unscalable, because operator cannot just spin up additional service"},{"line_number":33,"context_line":"if load is high and need to rely on having just one instance."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_aa1900d9","line":32,"in_reply_to":"ba8a016a_b7b435f2","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":30,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":31,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"},{"line_number":32,"context_line":"completely unscalable, because operator cannot just spin up additional service"},{"line_number":33,"context_line":"if load is high and need to rely on having just one instance."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"},{"line_number":36,"context_line":"preventing that, because the lock won\u0027t be shared between different c-vol"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_d7b779f3","line":33,"updated":"2015-11-17 17:49:08.000000000","message":"s/need/needs","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":30,"context_line":"service (like Pacemaker) that\u0027s monitoring state of the c-vol and makes a new"},{"line_number":31,"context_line":"instance active in case of a failure of a previous one. Also this makes c-vol"},{"line_number":32,"context_line":"completely unscalable, because operator cannot just spin up additional service"},{"line_number":33,"context_line":"if load is high and need to rely on having just one instance."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"},{"line_number":36,"context_line":"preventing that, because the lock won\u0027t be shared between different c-vol"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_4a0ad47a","line":33,"in_reply_to":"ba8a016a_d7b779f3","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"},{"line_number":36,"context_line":"preventing that, because the lock won\u0027t be shared between different c-vol"},{"line_number":37,"context_line":"services running on different hosts. This may cause multiple issues, resulting"},{"line_number":38,"context_line":"even in data loss."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_57a9497a","line":37,"updated":"2015-11-17 17:49:08.000000000","message":"nit: I think you should clarify what you are referring here with \"This\" as it could be clearer mentioning you are referring to trying to do A-A as it is.","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Local file locks that are used in c-vol\u0027s manager and some drivers are"},{"line_number":36,"context_line":"preventing that, because the lock won\u0027t be shared between different c-vol"},{"line_number":37,"context_line":"services running on different hosts. This may cause multiple issues, resulting"},{"line_number":38,"context_line":"even in data loss."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_6a0f9888","line":37,"in_reply_to":"ba8a016a_57a9497a","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":55,"context_line":"hard requirement on DLM software`_. Main conclusions from the session are:"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Projects can hard-depend on having a DLM."},{"line_number":58,"context_line":"* Tooz will be an abstraction layer."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"That\u0027s why proposed solution is to convert current locks that are local to use"},{"line_number":61,"context_line":"Tooz_ abstraction layer."}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_77d24d02","line":58,"updated":"2015-11-17 17:49:08.000000000","message":"nit: s/an/the\n\nYou can also make Tooz a like here  ;-)","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":55,"context_line":"hard requirement on DLM software`_. Main conclusions from the session are:"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Projects can hard-depend on having a DLM."},{"line_number":58,"context_line":"* Tooz will be an abstraction layer."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"That\u0027s why proposed solution is to convert current locks that are local to use"},{"line_number":61,"context_line":"Tooz_ abstraction layer."}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_0a292ce6","line":58,"in_reply_to":"ba8a016a_77d24d02","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"889ed4c1bf09dbdcb8ed2d61ed91ce3912837997","unresolved":false,"context_lines":[{"line_number":84,"context_line":"using local locks too."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"We could also replace current locks with some DB-based locking. This was"},{"line_number":87,"context_line":"proposed by gegulieo in his_ specs_, but increased the complexity of the"},{"line_number":88,"context_line":"solution greatly."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"In the future we will probably want to try to remove locks from volume manager"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_b1f45a87","line":87,"updated":"2015-11-17 11:00:28.000000000","message":"I think \u0027his_ specs_\u0027 is too fancy: the reader is unlikely to know that clicking on \u0027his\u0027 will get one spec and clicking on \u0027specs\u0027 will get another.\n\nHow about: This was proposed by geguileo in specs to remove local locks from the manager_ and from drivers_, but increased ...","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":84,"context_line":"using local locks too."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"We could also replace current locks with some DB-based locking. This was"},{"line_number":87,"context_line":"proposed by gegulieo in his_ specs_, but increased the complexity of the"},{"line_number":88,"context_line":"solution greatly."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"In the future we will probably want to try to remove locks from volume manager"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_0a57ec58","line":87,"in_reply_to":"ba8a016a_37828595","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":84,"context_line":"using local locks too."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"We could also replace current locks with some DB-based locking. This was"},{"line_number":87,"context_line":"proposed by gegulieo in his_ specs_, but increased the complexity of the"},{"line_number":88,"context_line":"solution greatly."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"In the future we will probably want to try to remove locks from volume manager"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_37828595","line":87,"in_reply_to":"ba8a016a_b1f45a87","updated":"2015-11-17 17:49:08.000000000","message":"I don\u0027t think it increases it greatly  :-P\n\nBut I do think that it would require a lot more testing than relying on a broadly used DLM.  ;-)","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":84,"context_line":"using local locks too."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"We could also replace current locks with some DB-based locking. This was"},{"line_number":87,"context_line":"proposed by gegulieo in his_ specs_, but increased the complexity of the"},{"line_number":88,"context_line":"solution greatly."},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"In the future we will probably want to try to remove locks from volume manager"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_aa67c04b","line":87,"in_reply_to":"ba8a016a_b1f45a87","updated":"2015-11-18 13:15:36.000000000","message":"I was a little careless on that, your proposal is much clearer.","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":172,"context_line":"  * We have some `work already done`_. We should split it for each driver and"},{"line_number":173,"context_line":"    work with driver maintainers to get patches merged."},{"line_number":174,"context_line":""},{"line_number":175,"context_line":"* Add DevStack patches to set up a CI testing Cinder with Redis as Tooz"},{"line_number":176,"context_line":"  backend."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_3a1b4c1d","line":175,"updated":"2015-11-17 17:49:08.000000000","message":"?: There\u0027s already a devstack patch to add Zookeeper, do you intend to add Redis as well?","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":172,"context_line":"  * We have some `work already done`_. We should split it for each driver and"},{"line_number":173,"context_line":"    work with driver maintainers to get patches merged."},{"line_number":174,"context_line":""},{"line_number":175,"context_line":"* Add DevStack patches to set up a CI testing Cinder with Redis as Tooz"},{"line_number":176,"context_line":"  backend."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_ca048419","line":175,"in_reply_to":"ba8a016a_3a1b4c1d","updated":"2015-11-18 13:15:36.000000000","message":"There\u0027s Redis plugin for DevStack and I think Szymon had some old code that was proving that we can setup Tooz with it.\n\nAnyway to be safe for unforeseen complications I\u0027ve changed it to Redis or ZooKeeper.\n\nBTW: Redis is mentioned here as I recall Red Hat had some issues with packaging ZooKeeper (or Java), so we needed an alternative to be sure that introduction of Tooz isn\u0027t a problem for RH OSP and anyone running RH/CentOS.","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ce6b580546c6079c496dff764b8ca8514d4b42d6","unresolved":false,"context_lines":[{"line_number":184,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"Unit tests for Tooz code will be added and a CI configured to test Cinder"},{"line_number":187,"context_line":"with Redis as lock backend will be set up. Possibly we can do that with"},{"line_number":188,"context_line":"multinode Tempest."},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_5a9bb072","line":187,"updated":"2015-11-17 17:49:08.000000000","message":"?: Same question as above, since there\u0027s a devstack patch for Zookeeper, will you still create the CI with Redis?","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":184,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"Unit tests for Tooz code will be added and a CI configured to test Cinder"},{"line_number":187,"context_line":"with Redis as lock backend will be set up. Possibly we can do that with"},{"line_number":188,"context_line":"multinode Tempest."},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_7d01a004","line":187,"in_reply_to":"ba8a016a_5a9bb072","updated":"2015-11-18 13:15:36.000000000","message":"ditto","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"889ed4c1bf09dbdcb8ed2d61ed91ce3912837997","unresolved":false,"context_lines":[{"line_number":200,"context_line":".. _Openstack Users Survey: http://www.openstack.org/assets/survey/Public-User-Survey-Report.pdf"},{"line_number":201,"context_line":".. _performance tests done by geguileo: https://github.com/Akrog/test-cinder-atomic-states"},{"line_number":202,"context_line":".. _session about allowing projects to have a hard requirement on DLM software: https://etherpad.openstack.org/p/mitaka-cross-project-dlm"},{"line_number":203,"context_line":".. _his: https://review.openstack.org/#/c/237602/"},{"line_number":204,"context_line":".. _specs: https://review.openstack.org/#/c/237604/"},{"line_number":205,"context_line":".. _work already done: https://review.openstack.org/#/c/185646/"},{"line_number":206,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_51216efa","line":203,"updated":"2015-11-17 11:00:28.000000000","message":"suggest s/_his:/_manager:","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":200,"context_line":".. _Openstack Users Survey: http://www.openstack.org/assets/survey/Public-User-Survey-Report.pdf"},{"line_number":201,"context_line":".. _performance tests done by geguileo: https://github.com/Akrog/test-cinder-atomic-states"},{"line_number":202,"context_line":".. _session about allowing projects to have a hard requirement on DLM software: https://etherpad.openstack.org/p/mitaka-cross-project-dlm"},{"line_number":203,"context_line":".. _his: https://review.openstack.org/#/c/237602/"},{"line_number":204,"context_line":".. _specs: https://review.openstack.org/#/c/237604/"},{"line_number":205,"context_line":".. _work already done: https://review.openstack.org/#/c/185646/"},{"line_number":206,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_2a3f5025","line":203,"in_reply_to":"ba8a016a_51216efa","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":9003,"name":"Tom Barron","email":"tpb@dyncloud.net","username":"tbarron"},"change_message_id":"889ed4c1bf09dbdcb8ed2d61ed91ce3912837997","unresolved":false,"context_lines":[{"line_number":201,"context_line":".. _performance tests done by geguileo: https://github.com/Akrog/test-cinder-atomic-states"},{"line_number":202,"context_line":".. _session about allowing projects to have a hard requirement on DLM software: https://etherpad.openstack.org/p/mitaka-cross-project-dlm"},{"line_number":203,"context_line":".. _his: https://review.openstack.org/#/c/237602/"},{"line_number":204,"context_line":".. _specs: https://review.openstack.org/#/c/237604/"},{"line_number":205,"context_line":".. _work already done: https://review.openstack.org/#/c/185646/"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"* https://etherpad.openstack.org/p/cinder-active-active-vol-service-issues"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_317dcae7","line":204,"updated":"2015-11-17 11:00:28.000000000","message":"suggest s/_specs:/_drivers:","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"120937b212a3dc2a4d51e07022334dc4305a2caf","unresolved":false,"context_lines":[{"line_number":201,"context_line":".. _performance tests done by geguileo: https://github.com/Akrog/test-cinder-atomic-states"},{"line_number":202,"context_line":".. _session about allowing projects to have a hard requirement on DLM software: https://etherpad.openstack.org/p/mitaka-cross-project-dlm"},{"line_number":203,"context_line":".. _his: https://review.openstack.org/#/c/237602/"},{"line_number":204,"context_line":".. _specs: https://review.openstack.org/#/c/237604/"},{"line_number":205,"context_line":".. _work already done: https://review.openstack.org/#/c/185646/"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"* https://etherpad.openstack.org/p/cinder-active-active-vol-service-issues"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ba8a016a_ca47a4ba","line":204,"in_reply_to":"ba8a016a_317dcae7","updated":"2015-11-18 13:15:36.000000000","message":"Done","commit_id":"06f10dc870600dfbf4d21e00936b6c1f0b9f2dae"},{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"2790f6fa548aae3bab4f681d750423294381d59c","unresolved":false,"context_lines":[{"line_number":142,"context_line":"  reconnection retries to Tooz backend."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Deployment tools maintainers will need to decide if they want to use new"},{"line_number":145,"context_line":"possibility."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Developer impact"},{"line_number":148,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":13,"id":"ba8a016a_e2184fd5","line":145,"updated":"2015-11-19 13:32:37.000000000","message":"Please, add note that deployers should setup Tooz backend if needed","commit_id":"e25a1ca02717dfcc8cf882705a157ce51ea8d07a"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cbf354608380bd83b766552ef58a50b2e35267d9","unresolved":false,"context_lines":[{"line_number":142,"context_line":"  reconnection retries to Tooz backend."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Deployment tools maintainers will need to decide if they want to use new"},{"line_number":145,"context_line":"possibility."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Developer impact"},{"line_number":148,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":13,"id":"ba8a016a_c885e5ad","line":145,"in_reply_to":"ba8a016a_e2184fd5","updated":"2015-11-20 15:48:36.000000000","message":"Not a bad idea - this isn\u0027t obvious for someone not involved in this stuff.","commit_id":"e25a1ca02717dfcc8cf882705a157ce51ea8d07a"}]}