)]}'
{"specs/kilo/create-volume-persistence-taskflow-support.rst":[{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"c69dfd71cb74fd130507314518aa016bb89fd620","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Proposed change"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_471258b2","line":49,"updated":"2015-02-02 15:12:45.000000000","message":"SQLite is a bad choice on production env. We need to specify database connection string in a config.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"c98bed51fdfa0f285492a3f07049d7c91ac4bf0b","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Proposed change"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_77c0e0e0","line":49,"in_reply_to":"da86d52c_471258b2","updated":"2015-02-02 18:25:19.000000000","message":"I\u0027m not sure if I would say its a bad choice for this; IMHO sqllite is designed for this kind of application-local data where a full database doesn\u0027t make sense (in fact firefox + others uses it just like this https://www.sqlite.org/famous.html). So since this information is cinder application-local data I would think sqlite is fine; and using a remote-db will just create a ton of traffic for data that is applicaiton-local; which seems bad.\n\nAnyways it should be a database connection string, but I don\u0027t think the default should be a remote mysql (or other) database; because of the type of data this is...","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"bd5fc56cf994319f3a8134e27b149258e524834f","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Proposed change"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_cab99116","line":49,"in_reply_to":"da86d52c_471258b2","updated":"2015-02-04 09:42:14.000000000","message":"There is database connection string in config for workflows, so you can use your remote DB if you need to. As I want to use one logbook per host it should work the same way as local sqlite DB.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"7198179af971f63f88e93b9dfce9be34c5ae5ee8","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"},{"line_number":53,"context_line":"limitation is assumed to narrow change scope. In the future it should be"},{"line_number":54,"context_line":"considered to implement support for TaskFlow\u0027s jobboards to allow workflows"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_2bbb99c5","line":51,"updated":"2015-01-20 20:04:57.000000000","message":"Are you planning to save the workflow state ? If yes then why can\u0027t be it stored to any backend supported by taskflow http://docs.openstack.org/developer/taskflow/persistence.html ?","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"4c44a8dcf06c4c5102c0231e52dee3c7162c6aee","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"},{"line_number":53,"context_line":"limitation is assumed to narrow change scope. In the future it should be"},{"line_number":54,"context_line":"considered to implement support for TaskFlow\u0027s jobboards to allow workflows"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_c5783e2e","line":51,"in_reply_to":"1a930d6b_2bbb99c5","updated":"2015-01-20 23:42:47.000000000","message":"I think that\u0027s the plan (but correct me if I am wrong); to use/extend/build-on the taskflow functionality where possible.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d28cb0c39d8774f0faa0e85d82c7d52119ca4954","unresolved":false,"context_lines":[{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"},{"line_number":53,"context_line":"limitation is assumed to narrow change scope. In the future it should be"},{"line_number":54,"context_line":"considered to implement support for TaskFlow\u0027s jobboards to allow workflows"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_91b4aaf3","line":51,"in_reply_to":"1a930d6b_c5783e2e","updated":"2015-01-21 11:00:23.000000000","message":"That\u0027s true, this will use TaskFlow\u0027s persistence - as stated in the top of the spec.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"c69dfd71cb74fd130507314518aa016bb89fd620","unresolved":false,"context_lines":[{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"},{"line_number":53,"context_line":"limitation is assumed to narrow change scope. In the future it should be"},{"line_number":54,"context_line":"considered to implement support for TaskFlow\u0027s jobboards to allow workflows"},{"line_number":55,"context_line":"resumptions also by another instance of the service. This however will require"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_877450d0","line":52,"updated":"2015-02-02 15:12:45.000000000","message":"Will this limitation be actual if we\u0027ll use remote DB?","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"c98bed51fdfa0f285492a3f07049d7c91ac4bf0b","unresolved":false,"context_lines":[{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"},{"line_number":53,"context_line":"limitation is assumed to narrow change scope. In the future it should be"},{"line_number":54,"context_line":"considered to implement support for TaskFlow\u0027s jobboards to allow workflows"},{"line_number":55,"context_line":"resumptions also by another instance of the service. This however will require"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_574bdc86","line":52,"in_reply_to":"da86d52c_877450d0","updated":"2015-02-02 18:25:19.000000000","message":"No, but then u are writing a lot of data to a remote DB for a scenario that should hopefully not be common (the need to restart a workflow elsewhere); once you start to do this you start to get into the http://docs.openstack.org/developer/taskflow/jobs.html territory and I think that\u0027s somewhat far off for this (start small....)","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"bd5fc56cf994319f3a8134e27b149258e524834f","unresolved":false,"context_lines":[{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"},{"line_number":53,"context_line":"limitation is assumed to narrow change scope. In the future it should be"},{"line_number":54,"context_line":"considered to implement support for TaskFlow\u0027s jobboards to allow workflows"},{"line_number":55,"context_line":"resumptions also by another instance of the service. This however will require"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_ca9a716d","line":52,"in_reply_to":"da86d52c_877450d0","updated":"2015-02-04 09:42:14.000000000","message":"The problem is - when should a service instance decide that workflow is abandoned by another service instance? What if there were network partition? We can end up with two volumes scheduled and created because one service decided that a workflow is abandoned and continued it while another instance was also doing that. As a workaround to this problem I\u0027m assuming that workflow can be continued only by same service instance.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"4c44a8dcf06c4c5102c0231e52dee3c7162c6aee","unresolved":false,"context_lines":[{"line_number":57,"context_line":""},{"line_number":58,"context_line":"The decision of using local SQLite DB is also to prevent TaskFlow from limiting"},{"line_number":59,"context_line":"Cinder\u0027s DB options. There will be a DB per node with a logbook per service"},{"line_number":60,"context_line":"instance. DBs will be stored in /var/lib/cinder directories. The SQL connection"},{"line_number":61,"context_line":"string will be configurable so user can use MySQL or PostgreSQL backend also."},{"line_number":62,"context_line":"Possible backends are listed in `TaskFlow\u0027s backend connection docs`_."},{"line_number":63,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_859e3655","line":60,"updated":"2015-01-20 23:42:47.000000000","message":"This will be an interesting addition. I don\u0027t expect size to be an issue but it will be the first time that a project has a \u0027temporary\u0027 (or somewhat temporary/transient) sqlite database and a more \u0027reliable\u0027 one used for non-transient/workflow information... Let\u0027s see what the cinder folks think. I understand the desire to do this; I just haven\u0027t seen this done in openstack *to my knowledge*.\n\nWhat vilobh is working on @ https://review.openstack.org/#/c/124205/ will be an interesting connection into this. Seems like we should figure out how they overlap and where.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d28cb0c39d8774f0faa0e85d82c7d52119ca4954","unresolved":false,"context_lines":[{"line_number":57,"context_line":""},{"line_number":58,"context_line":"The decision of using local SQLite DB is also to prevent TaskFlow from limiting"},{"line_number":59,"context_line":"Cinder\u0027s DB options. There will be a DB per node with a logbook per service"},{"line_number":60,"context_line":"instance. DBs will be stored in /var/lib/cinder directories. The SQL connection"},{"line_number":61,"context_line":"string will be configurable so user can use MySQL or PostgreSQL backend also."},{"line_number":62,"context_line":"Possible backends are listed in `TaskFlow\u0027s backend connection docs`_."},{"line_number":63,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_717e5ee6","line":60,"in_reply_to":"1a930d6b_859e3655","updated":"2015-01-21 11:00:23.000000000","message":"If operator want the workflows to be saved in normal OpenStack DB - fine, it can be configured that way and should work normally, as there\u0027s logbook per service instance. The problem is this would require a DB TaskFlow will support (majority of deployments have that but certainly not all) and I don\u0027t want to have limit there.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"c69dfd71cb74fd130507314518aa016bb89fd620","unresolved":false,"context_lines":[{"line_number":66,"context_line":"already resumed in task details."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"At service startup and as a periodic task with configurable period the database"},{"line_number":69,"context_line":"will be also cleaned up of flows that are completed or were tried to be"},{"line_number":70,"context_line":"resumed. This will ensure that the DB will not grow beyond expectations."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"There are also some problems we need to cover before the implementation can"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_42d08650","line":69,"updated":"2015-02-02 15:12:45.000000000","message":"Do we really need this periodic?","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"c98bed51fdfa0f285492a3f07049d7c91ac4bf0b","unresolved":false,"context_lines":[{"line_number":66,"context_line":"already resumed in task details."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"At service startup and as a periodic task with configurable period the database"},{"line_number":69,"context_line":"will be also cleaned up of flows that are completed or were tried to be"},{"line_number":70,"context_line":"resumed. This will ensure that the DB will not grow beyond expectations."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"There are also some problems we need to cover before the implementation can"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_771c6078","line":69,"in_reply_to":"da86d52c_42d08650","updated":"2015-02-02 18:25:19.000000000","message":"So other option I guess is to have the code that runs the workflow have a final \u0027action\u0027 that deletes it from the database after it has completed... Then you wouldn\u0027t need a periodic job to cleanup things (although you would need some periodic job to resume things that never finished).","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"bd5fc56cf994319f3a8134e27b149258e524834f","unresolved":false,"context_lines":[{"line_number":66,"context_line":"already resumed in task details."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"At service startup and as a periodic task with configurable period the database"},{"line_number":69,"context_line":"will be also cleaned up of flows that are completed or were tried to be"},{"line_number":70,"context_line":"resumed. This will ensure that the DB will not grow beyond expectations."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"There are also some problems we need to cover before the implementation can"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_6add05e0","line":69,"in_reply_to":"da86d52c_771c6078","updated":"2015-02-04 09:42:14.000000000","message":"We can probably just clean DB at service restart, it should be enough.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"7198179af971f63f88e93b9dfce9be34c5ae5ee8","unresolved":false,"context_lines":[{"line_number":67,"context_line":""},{"line_number":68,"context_line":"At service startup and as a periodic task with configurable period the database"},{"line_number":69,"context_line":"will be also cleaned up of flows that are completed or were tried to be"},{"line_number":70,"context_line":"resumed. This will ensure that the DB will not grow beyond expectations."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"There are also some problems we need to cover before the implementation can"},{"line_number":73,"context_line":"progress. One of them is objects that are going through task details (e.g."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_2b2b9983","line":70,"updated":"2015-01-20 20:04:57.000000000","message":"Will this be a separate job ? If yes, how frequently will the reaper job run ?","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d28cb0c39d8774f0faa0e85d82c7d52119ca4954","unresolved":false,"context_lines":[{"line_number":67,"context_line":""},{"line_number":68,"context_line":"At service startup and as a periodic task with configurable period the database"},{"line_number":69,"context_line":"will be also cleaned up of flows that are completed or were tried to be"},{"line_number":70,"context_line":"resumed. This will ensure that the DB will not grow beyond expectations."},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"There are also some problems we need to cover before the implementation can"},{"line_number":73,"context_line":"progress. One of them is objects that are going through task details (e.g."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_314bf6b1","line":70,"in_reply_to":"1a930d6b_2b2b9983","updated":"2015-01-21 11:00:23.000000000","message":"This is explained in config section. I\u0027m thinking of starting it every hour by default.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"7198179af971f63f88e93b9dfce9be34c5ae5ee8","unresolved":false,"context_lines":[{"line_number":96,"context_line":"-----------------"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":99,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"REST API impact"},{"line_number":102,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_2b655945","line":99,"updated":"2015-01-20 20:04:57.000000000","message":"Why we need a different DB why can\u0027t we store these details the same way we store the input/output for a task in the workflow ?","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d28cb0c39d8774f0faa0e85d82c7d52119ca4954","unresolved":false,"context_lines":[{"line_number":96,"context_line":"-----------------"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":99,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"REST API impact"},{"line_number":102,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_b17286e3","line":99,"in_reply_to":"1a930d6b_2b655945","updated":"2015-01-21 11:00:23.000000000","message":"This is to prevent from Cinder\u0027s DB capabilities being limited by TaskFlow.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"c69dfd71cb74fd130507314518aa016bb89fd620","unresolved":false,"context_lines":[{"line_number":96,"context_line":"-----------------"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":99,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"REST API impact"},{"line_number":102,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_02320ea7","line":99,"in_reply_to":"1a930d6b_b17286e3","updated":"2015-02-02 15:12:45.000000000","message":"The same concern as above: could we use non-local db?","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"bd5fc56cf994319f3a8134e27b149258e524834f","unresolved":false,"context_lines":[{"line_number":96,"context_line":"-----------------"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":99,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"REST API impact"},{"line_number":102,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_aa91bdf2","line":99,"in_reply_to":"da86d52c_02320ea7","updated":"2015-02-04 09:42:14.000000000","message":"As I explained above - you can, just specify it in config. Everything should still work fine.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"7198179af971f63f88e93b9dfce9be34c5ae5ee8","unresolved":false,"context_lines":[{"line_number":121,"context_line":"Performance Impact"},{"line_number":122,"context_line":"------------------"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"During workflow execution task states and results must be saved in a database."},{"line_number":125,"context_line":"This should have slight impact on performance. Moreover a periodic task for"},{"line_number":126,"context_line":"workflow cleanup will be added."},{"line_number":127,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_ebe1919b","line":124,"updated":"2015-01-20 20:04:57.000000000","message":"Isn\u0027t this already there ? http://docs.openstack.org/developer/taskflow/persistence.html especially saving the task results that must be saved because thats the way the next executing task is going to fetch the input.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"9e14149e3acc0b9e4fdf0d6cf545fa4722c9efe4","unresolved":false,"context_lines":[{"line_number":121,"context_line":"Performance Impact"},{"line_number":122,"context_line":"------------------"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"During workflow execution task states and results must be saved in a database."},{"line_number":125,"context_line":"This should have slight impact on performance. Moreover a periodic task for"},{"line_number":126,"context_line":"workflow cleanup will be added."},{"line_number":127,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_045b0888","line":124,"in_reply_to":"1a930d6b_31a77651","updated":"2015-01-21 18:32:43.000000000","message":"That\u0027s correct, it\u0027s just all saved in memory right now (for better or worse, but u have to start somewhere...)","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d28cb0c39d8774f0faa0e85d82c7d52119ca4954","unresolved":false,"context_lines":[{"line_number":121,"context_line":"Performance Impact"},{"line_number":122,"context_line":"------------------"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"During workflow execution task states and results must be saved in a database."},{"line_number":125,"context_line":"This should have slight impact on performance. Moreover a periodic task for"},{"line_number":126,"context_line":"workflow cleanup will be added."},{"line_number":127,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_31a77651","line":124,"in_reply_to":"1a930d6b_ebe1919b","updated":"2015-01-21 11:00:23.000000000","message":"I think that in the current Cinder\u0027s implementation workflow states are saved in memory and next task are given just references to the data. Correct me if I\u0027m wrong.\n\nSo the point of this blueprint is to add TaskFlow\u0027s persistence into Cinder.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"c69dfd71cb74fd130507314518aa016bb89fd620","unresolved":false,"context_lines":[{"line_number":126,"context_line":"workflow cleanup will be added."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Other deployer impact"},{"line_number":129,"context_line":"---------------------"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Three new configuration options will be added:"},{"line_number":132,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_22b552ee","line":129,"updated":"2015-02-02 15:12:45.000000000","message":"Need to add details about new DB for taskflow","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"c98bed51fdfa0f285492a3f07049d7c91ac4bf0b","unresolved":false,"context_lines":[{"line_number":126,"context_line":"workflow cleanup will be added."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Other deployer impact"},{"line_number":129,"context_line":"---------------------"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Three new configuration options will be added:"},{"line_number":132,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_ba0ddd39","line":129,"in_reply_to":"da86d52c_22b552ee","updated":"2015-02-02 18:25:19.000000000","message":"http://docs.openstack.org/developer/taskflow/persistence.html#sqlalchemy","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"82bdfc161acfdfc0c5748aaa8fd4f67efd1c82e5","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* Possibility of turning workflow resumption off (resume_workflows\u003dTrue flag)."},{"line_number":134,"context_line":"* Configurable time to cleanup the workflows database"},{"line_number":135,"context_line":"  (workflows_cleanup_interval\u003d60)."},{"line_number":136,"context_line":"* SQL connection string to the DB with workflow statuses"},{"line_number":137,"context_line":"  (workflows_sql_connection\u003dsqlite://///var/lib/cinder/workflows.sqlite)"},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_4509ae94","line":135,"updated":"2015-01-20 23:44:56.000000000","message":"Ya, this is an interesting one, probably we need to have some knowledge/shared info that prevents this cleanup job from destroying workflows that are still being ran (but may say have elapsd 61 seconds of work...)","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d28cb0c39d8774f0faa0e85d82c7d52119ca4954","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* Possibility of turning workflow resumption off (resume_workflows\u003dTrue flag)."},{"line_number":134,"context_line":"* Configurable time to cleanup the workflows database"},{"line_number":135,"context_line":"  (workflows_cleanup_interval\u003d60)."},{"line_number":136,"context_line":"* SQL connection string to the DB with workflow statuses"},{"line_number":137,"context_line":"  (workflows_sql_connection\u003dsqlite://///var/lib/cinder/workflows.sqlite)"},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_6c792593","line":135,"in_reply_to":"1a930d6b_4509ae94","updated":"2015-01-21 11:00:23.000000000","message":"We know which workflows are in SUCESS state - we can remove only them. Anyway there\u0027s my mistake, I was thinking of running them every hour, so s/60/3600. I\u0027ll fix this in the next patchset.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"c69dfd71cb74fd130507314518aa016bb89fd620","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* Possibility of turning workflow resumption off (resume_workflows\u003dTrue flag)."},{"line_number":134,"context_line":"* Configurable time to cleanup the workflows database"},{"line_number":135,"context_line":"  (workflows_cleanup_interval\u003d60)."},{"line_number":136,"context_line":"* SQL connection string to the DB with workflow statuses"},{"line_number":137,"context_line":"  (workflows_sql_connection\u003dsqlite://///var/lib/cinder/workflows.sqlite)"},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_020a0e8a","line":135,"in_reply_to":"1a930d6b_64512c62","updated":"2015-02-02 15:12:45.000000000","message":"Suck bulk operations could raise performance impact in a high-load production environments. We should be able to turn-off this periodic tack and do it manually via cinder-manage tool","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"9e14149e3acc0b9e4fdf0d6cf545fa4722c9efe4","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* Possibility of turning workflow resumption off (resume_workflows\u003dTrue flag)."},{"line_number":134,"context_line":"* Configurable time to cleanup the workflows database"},{"line_number":135,"context_line":"  (workflows_cleanup_interval\u003d60)."},{"line_number":136,"context_line":"* SQL connection string to the DB with workflow statuses"},{"line_number":137,"context_line":"  (workflows_sql_connection\u003dsqlite://///var/lib/cinder/workflows.sqlite)"},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_64512c62","line":135,"in_reply_to":"1a930d6b_6c792593","updated":"2015-01-21 18:32:43.000000000","message":"Ah, good point, that should be fine.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"bd5fc56cf994319f3a8134e27b149258e524834f","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* Possibility of turning workflow resumption off (resume_workflows\u003dTrue flag)."},{"line_number":134,"context_line":"* Configurable time to cleanup the workflows database"},{"line_number":135,"context_line":"  (workflows_cleanup_interval\u003d60)."},{"line_number":136,"context_line":"* SQL connection string to the DB with workflow statuses"},{"line_number":137,"context_line":"  (workflows_sql_connection\u003dsqlite://///var/lib/cinder/workflows.sqlite)"},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da86d52c_4aa3e14e","line":135,"in_reply_to":"da86d52c_020a0e8a","updated":"2015-02-04 09:42:14.000000000","message":"Agreed.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"7198179af971f63f88e93b9dfce9be34c5ae5ee8","unresolved":false,"context_lines":[{"line_number":165,"context_line":""},{"line_number":166,"context_line":"- Resolve problems with non JSON serializable objects passed between tasks in"},{"line_number":167,"context_line":"  workflows."},{"line_number":168,"context_line":"- Resolve problems with non-idempotent tasks in workflows."},{"line_number":169,"context_line":"- Implement saving of workflows to local DB."},{"line_number":170,"context_line":"- Implement resumptions of workflows."},{"line_number":171,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_cb577525","line":168,"updated":"2015-01-20 20:04:57.000000000","message":"In case of non-idempotent tasks do you plan to save a micro-state which can help to keep a track of the point till which a task is idempotent so that we minimize the re-work that needs to be done once the task is restarted.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d28cb0c39d8774f0faa0e85d82c7d52119ca4954","unresolved":false,"context_lines":[{"line_number":165,"context_line":""},{"line_number":166,"context_line":"- Resolve problems with non JSON serializable objects passed between tasks in"},{"line_number":167,"context_line":"  workflows."},{"line_number":168,"context_line":"- Resolve problems with non-idempotent tasks in workflows."},{"line_number":169,"context_line":"- Implement saving of workflows to local DB."},{"line_number":170,"context_line":"- Implement resumptions of workflows."},{"line_number":171,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a930d6b_ccdcd977","line":168,"in_reply_to":"1a930d6b_cb577525","updated":"2015-01-21 11:00:23.000000000","message":"This would certainly help. We need to discuss how to integrate this with your work on micro-states.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"03a92685eb9d77ee4a0761e17039ef8a01bf8d11","unresolved":false,"context_lines":[{"line_number":165,"context_line":""},{"line_number":166,"context_line":"- Resolve problems with non JSON serializable objects passed between tasks in"},{"line_number":167,"context_line":"  workflows."},{"line_number":168,"context_line":"- Resolve problems with non-idempotent tasks in workflows."},{"line_number":169,"context_line":"- Implement saving of workflows to local DB."},{"line_number":170,"context_line":"- Implement resumptions of workflows."},{"line_number":171,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"fa81d914_08977891","line":168,"in_reply_to":"1a930d6b_ccdcd977","updated":"2015-01-29 02:02:11.000000000","message":"Sure.","commit_id":"4bea1692dad977c10c0ea24fc9fd22eaeafcfef8"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"033377d5c9fe2ccf4bbb363bb85598627ad78a51","unresolved":false,"context_lines":[{"line_number":36,"context_line":""},{"line_number":37,"context_line":"Very similar problems can be observed in scheduler and manager workflows."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This blueprint addresses also problems with clean shutdowns required to upgrade"},{"line_number":40,"context_line":"services. As workflows are resumed after service start, operator can safely"},{"line_number":41,"context_line":"shutdown the service at any time. TaskFlow supports persistent workflows"},{"line_number":42,"context_line":"upgrades so it can be used for that purpose and won\u0027t affect zero-downtime"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da86d52c_1261ce04","line":39,"updated":"2015-02-06 03:00:38.000000000","message":"also addresses problems with....","commit_id":"8c07948786c1557c091ec626db4165e1dd1f4aa8"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"033377d5c9fe2ccf4bbb363bb85598627ad78a51","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Proposed change"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da86d52c_9234defc","line":49,"updated":"2015-02-06 03:00:38.000000000","message":"why to a SQLite db?   This means that if the cinder node goes down and needs to start back up on another machine, we are still hosed.","commit_id":"8c07948786c1557c091ec626db4165e1dd1f4aa8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"57babd03bc12a91b87abe0a66a4398d42c0fd483","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Proposed change"},{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the local SQLite"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup (or"},{"line_number":51,"context_line":"another appropriate moment). The workflow can be resumed only by the same"},{"line_number":52,"context_line":"instance of a service (this will be assured by using local DB). This"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da86d52c_2e4821b2","line":49,"in_reply_to":"da86d52c_9234defc","updated":"2015-02-06 10:15:38.000000000","message":"I\u0027ve explained this in comment to line #102.","commit_id":"8c07948786c1557c091ec626db4165e1dd1f4aa8"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"b5609bd921e6c4acd88d972e5af14ffce044ca0f","unresolved":false,"context_lines":[{"line_number":99,"context_line":"-----------------"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":102,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"REST API impact"},{"line_number":105,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da86d52c_465c6719","line":102,"updated":"2015-02-06 03:06:26.000000000","message":"I don\u0027t see the value in this.  why not just create taskflow_* tables in the cinder DB, and centralize the persistence so that if the cinder controller pukes, the flow can be picked up by another cinder controller.   \n\nI don\u0027t like the idea of limiting us to running a single cinder controller.   We need to be thinking a bit bigger here and allow cinder to run on multiple hosts at the same time.","commit_id":"8c07948786c1557c091ec626db4165e1dd1f4aa8"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"3cc87ea9f23986a4ac447291e38bfb5b5226becc","unresolved":false,"context_lines":[{"line_number":99,"context_line":"-----------------"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":102,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"REST API impact"},{"line_number":105,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da86d52c_8ce651e7","line":102,"in_reply_to":"da86d52c_1249b28e","updated":"2015-02-07 19:34:23.000000000","message":"Thanks for the input guys.   I appreciate the work you guys are doing here for the Cinder project.  I really do.   I think we need to think about this a bit more before we go ahead, unless the PTL thinks this is the right way to go.   \n\nI personally don\u0027t think persisting to a local file DB is the direction we should be going.   We are actively working on eliminating local file locks in Cinder for the same reasons.  We want to have more than a single active Cinder controller that can service cinder requests.   If we intentionally bake in more of this, then we are further making it more and more difficult to have an active/active Cinder setup for larger deployments.   \n\nI don\u0027t have a simple solution to your issues you raised above, but I still don\u0027t think we should be putting in single controller limitations into Cinder intentionally any more.   We need to do better.","commit_id":"8c07948786c1557c091ec626db4165e1dd1f4aa8"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"57babd03bc12a91b87abe0a66a4398d42c0fd483","unresolved":false,"context_lines":[{"line_number":99,"context_line":"-----------------"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":102,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"REST API impact"},{"line_number":105,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da86d52c_4e7c75ec","line":102,"in_reply_to":"da86d52c_465c6719","updated":"2015-02-06 10:15:38.000000000","message":"TaskFlow supports just MySQL, SQLite and PostgreSQL DBs. Creating the tables in cinder DB adds a requirement that every new DB backend in Cinder would need to be supported by TaskFlow and this is what I wanted to avoid as I\u0027m aware of for example this bp: http://specs.openstack.org/openstack/cinder-specs/specs/kilo/db2-database.html However it\u0027s still possible to add this as an option if you\u0027re using standard DBs.\n\nThe problem with resuming workflows on another controllers is that you don\u0027t know when the flow is orphaned. It would require timeout + hearbeat mechanism to protect from creating duplicated resources. This is close to the idea of jobboards in TaskFlow (http://docs.openstack.org/developer/taskflow/jobs.html) where zookeeper is taking care of that.\n\nYou may seem that it\u0027s enough to check if service executing the workflow is down and if so - resume in-progress workflow. Unfortunately when you take network partitions into account I\u0027m worried that cinder can still end up with duplicated resources (i.e. partitioned cinder-scheduler instances talking to partitioned DB are resuming same workflow). I\u0027m I right here or will distributed DB mechanisms take care of these type of cases?","commit_id":"8c07948786c1557c091ec626db4165e1dd1f4aa8"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"81242e3c5259ff53331115cf212ec7b6491b9ab3","unresolved":false,"context_lines":[{"line_number":99,"context_line":"-----------------"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"We plan to create local DBs to avoid influencing Cinder\u0027s schema by a"},{"line_number":102,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"REST API impact"},{"line_number":105,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da86d52c_1249b28e","line":102,"in_reply_to":"da86d52c_4e7c75ec","updated":"2015-02-06 18:31:54.000000000","message":"So michal is right; taskflow doesn\u0027t use \u0027oslo.db\u0027 (although it may someday, depending on the outcome of https://review.openstack.org/#/c/149836/ which really seems like it doesn\u0027t recommend using \u0027oslo.*\u0027 libraries outside openstack); since taskflow isn\u0027t just used in openstack (and it doesn\u0027t use \u0027oslo.cfg\u0027 all over the place, since it\u0027s a library and can\u0027t force that configuration model on people...) taskflow just uses the sqlalchemy library directly (as would be expected of most libraries that can be used/exist outside openstack...)\n\nSo knowing that, it could still be fine to try this using mysql, postgres, sqlite (or other sqlalchemy directly supported dbs; or get the db2/ibm/? people to contribute whatever they did to sqlalchemy codebase itself...); but the situation around ownership that michal mentions is still going to be the case. Now sure; we can start thinking about using jobboards, handling workflow failover to other cinder nodes automatically, but that seems like a larger change and discussion, this spec would be more incremental and could be part of the 10 step program (maybe less than 10 steps, maybe more) that could eventually end up at the desired state...\n\nNow I\u0027m not an expert on what the 10 step program will be but gotta start somewhere IMHO (no matter what it\u0027s going to be a long journey...).","commit_id":"8c07948786c1557c091ec626db4165e1dd1f4aa8"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Very similar problems can be observed in scheduler and manager workflows."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This blueprint addresses also problems with clean shutdowns required to upgrade"},{"line_number":40,"context_line":"services. As workflows are resumed after service start, operator can safely"},{"line_number":41,"context_line":"shutdown the service at any time. TaskFlow supports persistent workflows"},{"line_number":42,"context_line":"upgrades so it can be used for that purpose and won\u0027t affect zero-downtime"},{"line_number":43,"context_line":"upgrades effort."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_c500ad03","line":40,"updated":"2015-02-11 01:00:43.000000000","message":"Can it handle scenarios for upgrade across different versions ? If yes how do you proposed to do that ? If no then lets say in a mixed environment when the customer is running version v1 and v2 how do you justify upgrade ?\n\nAlso what is the granularity from where the workflows can be resumed ? Do we always want to start from the first task in the workflow ? If yes won\u0027t it redo a lot amount of work if the workflow got interrupted in the last step ? If no then do we have a way to do a point in time restart ?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c661e8656fdacd0bfd6de6b93feca01a32f2c27d","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Very similar problems can be observed in scheduler and manager workflows."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This blueprint addresses also problems with clean shutdowns required to upgrade"},{"line_number":40,"context_line":"services. As workflows are resumed after service start, operator can safely"},{"line_number":41,"context_line":"shutdown the service at any time. TaskFlow supports persistent workflows"},{"line_number":42,"context_line":"upgrades so it can be used for that purpose and won\u0027t affect zero-downtime"},{"line_number":43,"context_line":"upgrades effort."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_3b9c415d","line":40,"in_reply_to":"da86d52c_c500ad03","updated":"2015-02-11 09:44:57.000000000","message":"Joshua answered you about upgrades. About the granularity - TaskFlow is saving state at completion of each tasks - so workflow can be resumed in the middle.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"f3bee712c7f92470482a5fabebef8e8525c62ef7","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Very similar problems can be observed in scheduler and manager workflows."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This blueprint addresses also problems with clean shutdowns required to upgrade"},{"line_number":40,"context_line":"services. As workflows are resumed after service start, operator can safely"},{"line_number":41,"context_line":"shutdown the service at any time. TaskFlow supports persistent workflows"},{"line_number":42,"context_line":"upgrades so it can be used for that purpose and won\u0027t affect zero-downtime"},{"line_number":43,"context_line":"upgrades effort."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_07a3a500","line":40,"in_reply_to":"da86d52c_c500ad03","updated":"2015-02-11 05:27:02.000000000","message":"So I\u0027ll address the granularity at which it can be resumed; since this is a taskflow question, the upgrade one is really dependent on what cinder wants to do in that situation...\n\nThe taskflow granularity for resumption is done by reading over http://docs.openstack.org/developer/taskflow/states.html and http://docs.openstack.org/developer/taskflow/engines.html#resumption and http://docs.openstack.org/developer/taskflow/persistence.html#how-it-is-used and http://docs.openstack.org/developer/taskflow/resumption.html \n\nThose links hopefully make this clear, that no its its not \u0027start from the first task in the workflow\u0027, each state in that states.html is like a checkpoint that gets persisted; and the taskflow engine can re-associate (by name) tasks prior checkpoints on resumption...\n\nFor the upgrade scenario the following is probably useful to look over:\n\nhttp://docs.openstack.org/developer/taskflow/resumption.html#scenarios","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the centralized"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup. Apart"},{"line_number":51,"context_line":"from that ownership of workflow can be transfered to another service instance"},{"line_number":52,"context_line":"after certain timeout (set in configuration) if current workflow owner is"},{"line_number":53,"context_line":"considered down by service groupi API."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_c5750d99","line":50,"updated":"2015-02-11 01:00:43.000000000","message":"Centralized database ? Mysql cinder db ? or any other persistent storage ?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c661e8656fdacd0bfd6de6b93feca01a32f2c27d","unresolved":false,"context_lines":[{"line_number":47,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"We propose to save currently executed workflows states to the centralized"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup. Apart"},{"line_number":51,"context_line":"from that ownership of workflow can be transfered to another service instance"},{"line_number":52,"context_line":"after certain timeout (set in configuration) if current workflow owner is"},{"line_number":53,"context_line":"considered down by service groupi API."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_5bd2d500","line":50,"in_reply_to":"da86d52c_c5750d99","updated":"2015-02-11 09:44:57.000000000","message":"This is answered later in the spec.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":49,"context_line":"We propose to save currently executed workflows states to the centralized"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup. Apart"},{"line_number":51,"context_line":"from that ownership of workflow can be transfered to another service instance"},{"line_number":52,"context_line":"after certain timeout (set in configuration) if current workflow owner is"},{"line_number":53,"context_line":"considered down by service groupi API."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"To prevent TaskFlow from limiting Cinder\u0027s DB options we assume that workflows"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_a55c991a","line":52,"updated":"2015-02-11 01:00:43.000000000","message":"How do you know current workflow owner is down ?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c661e8656fdacd0bfd6de6b93feca01a32f2c27d","unresolved":false,"context_lines":[{"line_number":49,"context_line":"We propose to save currently executed workflows states to the centralized"},{"line_number":50,"context_line":"database and resume workflows that weren\u0027t completed at service startup. Apart"},{"line_number":51,"context_line":"from that ownership of workflow can be transfered to another service instance"},{"line_number":52,"context_line":"after certain timeout (set in configuration) if current workflow owner is"},{"line_number":53,"context_line":"considered down by service groupi API."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"To prevent TaskFlow from limiting Cinder\u0027s DB options we assume that workflows"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_fb31c95a","line":52,"in_reply_to":"da86d52c_a55c991a","updated":"2015-02-11 09:44:57.000000000","message":"As explained in line below - I plan to use cinder service group API.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":53,"context_line":"considered down by service groupi API."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"To prevent TaskFlow from limiting Cinder\u0027s DB options we assume that workflows"},{"line_number":56,"context_line":"states are persisted in different DB than main Cinder DB. The SQL connection"},{"line_number":57,"context_line":"string will be configurable to allow users to configure different backend as"},{"line_number":58,"context_line":"main Cinder\u0027s DB and TaskFlow compatible one for workflow states (possible"},{"line_number":59,"context_line":"backends are listed in `TaskFlow\u0027s backend connection docs`_). Leaving empty"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_858dd57e","line":56,"updated":"2015-02-11 01:00:43.000000000","message":"Why can\u0027t it be a separate table in cinder database why a different db is needed ? Is the amount of data that needs to be saved that huge ?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c661e8656fdacd0bfd6de6b93feca01a32f2c27d","unresolved":false,"context_lines":[{"line_number":53,"context_line":"considered down by service groupi API."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"To prevent TaskFlow from limiting Cinder\u0027s DB options we assume that workflows"},{"line_number":56,"context_line":"states are persisted in different DB than main Cinder DB. The SQL connection"},{"line_number":57,"context_line":"string will be configurable to allow users to configure different backend as"},{"line_number":58,"context_line":"main Cinder\u0027s DB and TaskFlow compatible one for workflow states (possible"},{"line_number":59,"context_line":"backends are listed in `TaskFlow\u0027s backend connection docs`_). Leaving empty"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_9b1d2dc1","line":56,"in_reply_to":"da86d52c_858dd57e","updated":"2015-02-11 09:44:57.000000000","message":"I\u0027ve explained that - Cinder is working on support for DB2 and TaskFlow doesn\u0027t support it as backend. If we would use main cinder DB - we would limit Cinder\u0027s DB capabilities. Anyway deployer can configure it to use the same MySQL instance.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":61,"context_line":"to be resumed can be distinguished using workflow name saved in the DB."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"Particular task will be restarted just once to prevent problems with a task"},{"line_number":64,"context_line":"failing constantly. This will be ensured by storing information if task was"},{"line_number":65,"context_line":"already resumed in task details."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"At service startup the database will be also cleaned up of flows that are"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_a5b81912","line":64,"updated":"2015-02-11 01:00:43.000000000","message":"How do you justify transient vs fatal failures ? transient are the ones which can be recovered after few retries vs fatal ones are the ones which can\u0027t be recovered from..Shouldn\u0027t your retries be based of whether its a transient or a fatal failure ?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c661e8656fdacd0bfd6de6b93feca01a32f2c27d","unresolved":false,"context_lines":[{"line_number":61,"context_line":"to be resumed can be distinguished using workflow name saved in the DB."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"Particular task will be restarted just once to prevent problems with a task"},{"line_number":64,"context_line":"failing constantly. This will be ensured by storing information if task was"},{"line_number":65,"context_line":"already resumed in task details."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"At service startup the database will be also cleaned up of flows that are"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_db640524","line":64,"in_reply_to":"da86d52c_a5b81912","updated":"2015-02-11 09:44:57.000000000","message":"This may be interesting - can you explain how to recognize if failure is transient?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Another problem is tasks idempotency. We need to ensure that every task in the"},{"line_number":78,"context_line":"workflow can be safely started again and won\u0027t produce orphaned resource or"},{"line_number":79,"context_line":"return different value. For example self.driver_api.schedule_create_volume in"},{"line_number":80,"context_line":"ScheduleCreateVolumeTask fails if started again with same parameters. We need"},{"line_number":81,"context_line":"to check if volume exists before calling it."},{"line_number":82,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_e53211a6","line":79,"updated":"2015-02-11 01:00:43.000000000","message":"Why not store information for a workflow per resource for example resource being a volume, snapshot, image etc to make sure we don\u0027t have orphaned resources ?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c661e8656fdacd0bfd6de6b93feca01a32f2c27d","unresolved":false,"context_lines":[{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Another problem is tasks idempotency. We need to ensure that every task in the"},{"line_number":78,"context_line":"workflow can be safely started again and won\u0027t produce orphaned resource or"},{"line_number":79,"context_line":"return different value. For example self.driver_api.schedule_create_volume in"},{"line_number":80,"context_line":"ScheduleCreateVolumeTask fails if started again with same parameters. We need"},{"line_number":81,"context_line":"to check if volume exists before calling it."},{"line_number":82,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_3b70c1db","line":79,"in_reply_to":"da86d52c_e53211a6","updated":"2015-02-11 09:44:57.000000000","message":"I don\u0027t see how this would help. Can you explain more?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"798fec5521e6ac17a909495507e0151b489c596e","unresolved":false,"context_lines":[{"line_number":103,"context_line":""},{"line_number":104,"context_line":"We plan to add a new DB to avoid influencing Cinder\u0027s own schema by a"},{"line_number":105,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"REST API impact"},{"line_number":108,"context_line":"---------------"},{"line_number":109,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_b3f33ae6","line":106,"updated":"2015-02-10 01:24:28.000000000","message":"Another option is to provide your own backend impl:\n\nhttp://docs.openstack.org/developer/taskflow/persistence.html#module-taskflow.persistence.backends\n\nAs long as that API is maintained then you can write the data wherever and however u want... This could be better (or at least another option) that ties the projects together via that simplisitic object API instead of via a DB+schema...","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":103,"context_line":""},{"line_number":104,"context_line":"We plan to add a new DB to avoid influencing Cinder\u0027s own schema by a"},{"line_number":105,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"REST API impact"},{"line_number":108,"context_line":"---------------"},{"line_number":109,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_25aee998","line":106,"in_reply_to":"da86d52c_02eadc0e","updated":"2015-02-11 01:00:43.000000000","message":"Since now you are moving from your original idea of seperate SQlite db per node to a centralized db why not think of storing the data in other backend or in mysql itself ?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"f3bee712c7f92470482a5fabebef8e8525c62ef7","unresolved":false,"context_lines":[{"line_number":103,"context_line":""},{"line_number":104,"context_line":"We plan to add a new DB to avoid influencing Cinder\u0027s own schema by a"},{"line_number":105,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"REST API impact"},{"line_number":108,"context_line":"---------------"},{"line_number":109,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_a7b3b94b","line":106,"in_reply_to":"da86d52c_25aee998","updated":"2015-02-11 05:27:02.000000000","message":"I\u0027m confused by that vilobh; you mentioned \u0027to a  centralized db\u0027 but then u said mysql itself (isn\u0027t this such a  centralized db?)","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"4f913478c39cc67583497810469c47e51b734a4e","unresolved":false,"context_lines":[{"line_number":103,"context_line":""},{"line_number":104,"context_line":"We plan to add a new DB to avoid influencing Cinder\u0027s own schema by a"},{"line_number":105,"context_line":"`TaskFlow\u0027s persistence DB schema`_ hardcoded into the library."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"REST API impact"},{"line_number":108,"context_line":"---------------"},{"line_number":109,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_02eadc0e","line":106,"in_reply_to":"da86d52c_b3f33ae6","updated":"2015-02-10 10:45:22.000000000","message":"I\u0027m not sure if Cinder folks want to have a new thing to maintain. Lets see what they tell.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":128,"context_line":"------------------"},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"During workflow execution task states and results must be saved in a database."},{"line_number":131,"context_line":"This should have slight impact on performance. Moreover a periodic task"},{"line_number":132,"context_line":"iterating over running workflows will be added."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_8573154d","line":131,"updated":"2015-02-11 01:00:43.000000000","message":"Once the prototype is ready a statistics on this will be helpful for the project.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":129,"context_line":""},{"line_number":130,"context_line":"During workflow execution task states and results must be saved in a database."},{"line_number":131,"context_line":"This should have slight impact on performance. Moreover a periodic task"},{"line_number":132,"context_line":"iterating over running workflows will be added."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"Other deployer impact"},{"line_number":135,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_c590ad45","line":132,"updated":"2015-02-11 01:00:43.000000000","message":"Also a overhead added by this background task.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"e9c8bb932ecc9a67dd3968d08e975a9849c53d1f","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_62fe4e6d","line":147,"updated":"2015-02-09 23:54:39.000000000","message":"Intersting....","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"ad765b0d4b54e4a08c7791e9de66de5eb6809d0d","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_d077a09d","line":147,"in_reply_to":"da86d52c_01d2d0d1","updated":"2015-02-10 16:52:52.000000000","message":"This seems like a reasonable approach.   One caveat might be for long running tasks though, such as copy image to volume or volume to image operations that might take along time to complete.   Copying a 2TB volume to image might take longer than 5 minutes to complete.   How do we account for this?","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"951735bf90a8fd0f53ba9b2061e829c4a112cfd8","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_e50a1189","line":147,"in_reply_to":"da86d52c_0e2590c1","updated":"2015-02-11 01:00:43.000000000","message":"Agree to josh with this approach we will need additional dependency and moving to job boards stuff to implement this functionality as we will need a check for keep-alive or ping kind of mechanism will be nice.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c661e8656fdacd0bfd6de6b93feca01a32f2c27d","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_368618dc","line":147,"in_reply_to":"da86d52c_0e2590c1","updated":"2015-02-11 09:44:57.000000000","message":"There is servicegroup API in Cinder reporting the state for the service, so it can probably be used.\n\nAbout net-splits - I think that a distributed DB should take care of it. When Galera realizes that it lost the quorum - it makes DB read-only. As we\u0027re unable to write to the DB we can\u0027t move in the workflow - so it will fail. Only workflow running in a service connected to Galera instance *with quorum* can proceed. This means that only one workflow can run at the same time. Correct me if this is wrong assumption.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"281bb7b442cee503ba6909ca5899a7b0cc0bc674","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_64d9a856","line":147,"in_reply_to":"da86d52c_368618dc","updated":"2015-02-11 15:50:29.000000000","message":"Interesting, did not know that about galera.\n\nSince not everyone is running with galera (it\u0027s not a requirement to use openstack afaik) it is probably just worth mentioning this case.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"4f913478c39cc67583497810469c47e51b734a4e","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_c2b9a42f","line":147,"in_reply_to":"da86d52c_62fe4e6d","updated":"2015-02-10 10:45:22.000000000","message":"Can you elaborate why this is interesting? :)","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"06647262e1035f72c0c88d2cc6fb3c5f2b0b17a3","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_0e2590c1","line":147,"in_reply_to":"da86d52c_704994cf","updated":"2015-02-10 19:46:17.000000000","message":"So my natural inclination is that tooz (http://docs.openstack.org/developer/tooz/tutorial/group_membership.html) would fit here; does cinder have a service group api (it does appear a table exists in the db, not sure of what exposed api there is)?\n\nIf cinder already has some information it can likely just use it (note that I am hopefully going to get https://review.openstack.org/#/c/138607/ to happen someday, which is replacing the nova service group with tooz; this same information appears cloned in cinder due to historical reasons). Do note that if the workflows aren\u0027t idempotent and a workflow is being dual-ran by 2 services (ie a net-split occurs, 1st service is still running but hasn\u0027t reported in for a period, second service claims that other services work, both are running the same work now...) then it will have problems (and likely not things that u want to happen, ha).\n\nIMHO; to me though this is getting real close to the jobboard stuff in terms of functionality (but I understand cinder likely can\u0027t use it since jobboards and such are a bigger shift in mind-set/code that might not work out). The same issue mentioned above exists there noted @ http://docs.openstack.org/developer/taskflow/jobs.html#considerations (its made \u0027better\u0027 due to the granularity of taskflow tasks + saving, aka it can/should be noticed pretty quickly; when a state change is attempted that shouldn\u0027t be possible...)","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"0fdde83e45eee51bdacf571d3d8cdb2a1c456c13","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_01d2d0d1","line":147,"in_reply_to":"da86d52c_c2b9a42f","updated":"2015-02-10 15:20:40.000000000","message":"Transfer of workflows just seems cool :-P\n\nIt will be neat to see it work in cinder.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"cdce88a0468034ab09b5973bbd3c310d52ccc27d","unresolved":false,"context_lines":[{"line_number":144,"context_line":"* Interval of periodic task resuming the workflows"},{"line_number":145,"context_line":"  (workflows_resume_interval\u003d150)"},{"line_number":146,"context_line":"* Time after workflows is considered orphaned and can be transferred to another"},{"line_number":147,"context_line":"  instance of a service (workflows_timeout\u003d300)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Developer impact"},{"line_number":150,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_704994cf","line":147,"in_reply_to":"da86d52c_d077a09d","updated":"2015-02-10 17:03:44.000000000","message":"This would require some kind of heartbeating mechanism - so we want to check if service is running according to service group API (line #52).","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"e9c8bb932ecc9a67dd3968d08e975a9849c53d1f","unresolved":false,"context_lines":[{"line_number":198,"context_line":""},{"line_number":199,"context_line":"Unit testing is sufficient as there are no API changes to test them using"},{"line_number":200,"context_line":"Tempest. Also main problem is solved in TaskFlow, so it already has tests"},{"line_number":201,"context_line":"coverage."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_02ed92b7","line":201,"updated":"2015-02-09 23:54:39.000000000","message":"Some kind of crash testing would be neat (but is probably out of scope).","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"4f913478c39cc67583497810469c47e51b734a4e","unresolved":false,"context_lines":[{"line_number":198,"context_line":""},{"line_number":199,"context_line":"Unit testing is sufficient as there are no API changes to test them using"},{"line_number":200,"context_line":"Tempest. Also main problem is solved in TaskFlow, so it already has tests"},{"line_number":201,"context_line":"coverage."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_82688cbb","line":201,"in_reply_to":"da86d52c_02ed92b7","updated":"2015-02-10 10:45:22.000000000","message":"AFAIK crash testing is not possible in current infra.","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"0fdde83e45eee51bdacf571d3d8cdb2a1c456c13","unresolved":false,"context_lines":[{"line_number":198,"context_line":""},{"line_number":199,"context_line":"Unit testing is sufficient as there are no API changes to test them using"},{"line_number":200,"context_line":"Tempest. Also main problem is solved in TaskFlow, so it already has tests"},{"line_number":201,"context_line":"coverage."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da86d52c_61846cb1","line":201,"in_reply_to":"da86d52c_82688cbb","updated":"2015-02-10 15:20:40.000000000","message":"Ya, might be useful to comment on https://review.openstack.org/#/c/150653/ about this (thats trying to create some sort of testing guidelines).","commit_id":"f17c04b421360624a6b77aa1bb2d051d2e528e64"}]}
