)]}'
{"specs/juno/online-schema-changes.rst":[{"author":{"_account_id":9060,"name":"Mike Dorman","email":"dorm@dorm.org","username":"gd-mdorman"},"change_message_id":"a6e40093cfd6d1efdd19db1274f382d7da7dc277","unresolved":false,"context_lines":[{"line_number":20,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"* Database migrations are not required to be compatible with running code"},{"line_number":23,"context_line":"  which forces them to be run when offline."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Database migrations have historically been a source of lengthy downtime"},{"line_number":26,"context_line":"  during deployments."}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_92a8dc21","line":23,"updated":"2014-06-25 16:43:36.000000000","message":"Please reword to make this more clear.  I find that sentence very confusing.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"e3d0d0c754fc85b53d662fbca0589a8c0eb7eb8f","unresolved":false,"context_lines":[{"line_number":20,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"* Database migrations are not required to be compatible with running code"},{"line_number":23,"context_line":"  which forces them to be run when offline."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Database migrations have historically been a source of lengthy downtime"},{"line_number":26,"context_line":"  during deployments."}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_8c3285f0","line":23,"updated":"2014-06-25 15:37:52.000000000","message":"just checking, you mean after this change, migration will no longer be required to be compatible with running code?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":20,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"* Database migrations are not required to be compatible with running code"},{"line_number":23,"context_line":"  which forces them to be run when offline."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"* Database migrations have historically been a source of lengthy downtime"},{"line_number":26,"context_line":"  during deployments."}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_ba2f5df5","line":23,"in_reply_to":"1ae5cdf2_8c3285f0","updated":"2014-06-26 01:43:24.000000000","message":"This is just a problem description, it doesn\u0027t suggest any changes.\n\nBasically I\u0027m just stating that our database migrations make no guarantees about compatibility with code running (ie online). This then forces the operator to run them all with no code running (ie offline).\n\nI\u0027m certainly open to changing the wording since people have been confused by it, but I don\u0027t know what to change it to since I don\u0027t understand the confusion.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":9060,"name":"Mike Dorman","email":"dorm@dorm.org","username":"gd-mdorman"},"change_message_id":"a6e40093cfd6d1efdd19db1274f382d7da7dc277","unresolved":false,"context_lines":[{"line_number":34,"context_line":"#. Expand. This would apply changes that are backward compatible with"},{"line_number":35,"context_line":"   old running code."},{"line_number":36,"context_line":"#. Migrate. This would apply changes that are necessary to be run offline."},{"line_number":37,"context_line":"#. Contract. This would apply changes that are backward compatible with"},{"line_number":38,"context_line":"   new running code."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"A subset of schema changes will have DDL automatically generated by"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_f24be083","line":37,"updated":"2014-06-25 16:43:36.000000000","message":"Would it make more sense to say \"forward compatible\" here?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":34,"context_line":"#. Expand. This would apply changes that are backward compatible with"},{"line_number":35,"context_line":"   old running code."},{"line_number":36,"context_line":"#. Migrate. This would apply changes that are necessary to be run offline."},{"line_number":37,"context_line":"#. Contract. This would apply changes that are backward compatible with"},{"line_number":38,"context_line":"   new running code."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"A subset of schema changes will have DDL automatically generated by"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_1a3e9129","line":37,"in_reply_to":"1ae5cdf2_f24be083","updated":"2014-06-26 01:43:24.000000000","message":"Not really. Forward compatibility is for code that hasn\u0027t been written yet. This is for compatibility with code that is running while the schema changes happen.\n\nMaybe I should drop the \u0027backward\u0027 part.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7d99d5eb73ecc3cefc0bec115c20daca898346f2","unresolved":false,"context_lines":[{"line_number":35,"context_line":"   old running code."},{"line_number":36,"context_line":"#. Migrate. This would apply changes that are necessary to be run offline."},{"line_number":37,"context_line":"#. Contract. This would apply changes that are backward compatible with"},{"line_number":38,"context_line":"   new running code."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"A subset of schema changes will have DDL automatically generated by"},{"line_number":41,"context_line":"comparing the running database schema against the model defined in"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_89a8b721","line":38,"updated":"2014-06-25 15:28:20.000000000","message":"These are phases of conventional migrations, but I think listing only these three covers up the important bit of how the *data* gets migrated. For example, we discussed doing this in the object during a save() operation to write things into a differently-formatted new column or something.\n\nDoes the contract phase include data migrations that could happen during the live migration in the objects for data that has yet to be migrated?\n\nI assume you\u0027re suggesting that we can run a bunch of expands for version 7, 8, 9 and then later run the migrate for all of those, and then contract, right? Meaning, we don\u0027t have to expand/migrate/contract 7 and then expand/migrate/contract 8?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":35,"context_line":"   old running code."},{"line_number":36,"context_line":"#. Migrate. This would apply changes that are necessary to be run offline."},{"line_number":37,"context_line":"#. Contract. This would apply changes that are backward compatible with"},{"line_number":38,"context_line":"   new running code."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"A subset of schema changes will have DDL automatically generated by"},{"line_number":41,"context_line":"comparing the running database schema against the model defined in"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_9a78e1f9","line":38,"in_reply_to":"1ae5cdf2_89a8b721","updated":"2014-06-26 01:43:24.000000000","message":"This blueprint isn\u0027t about data migrations, only schema migrations.\n\nThis is complicated by the fact that we lump both into \"database migrations\".\n\nThere will still be \"database migrations\" that will be managed by sqlalchemy-migrate. Many of the existing migrations can be dropped after this blueprint is implemented because it will automate much of the schema changes. Only data migrations and those schema migrations that are hard to implement (like column type changes) will still be run by sqlalchemy-migrate.\n\nI\u0027m certainly not against changing the way data migrations are done, but it\u0027s orthogonal to this blueprint.\n\nAlso, expands don\u0027t have any versions. There is just the model. The expand (mostly) makes the running schema look like the model no matter where it starts from.\n\nIn fact, my development branch doesn\u0027t have a 216_havana.py migration anymore. It\u0027s all done automatically by the expand phase (with a tiny bit in the migrate phase).\n\nThat reminds me, I should add something about removing (most) schema changes from existing migrations.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":4573,"name":"yunhong-jiang","email":"yunhong.jiang@intel.com","username":"yunhong-jiang"},"change_message_id":"632b1393d830a3ac379e620fb805fdbcf38abae7","unresolved":false,"context_lines":[{"line_number":35,"context_line":"   old running code."},{"line_number":36,"context_line":"#. Migrate. This would apply changes that are necessary to be run offline."},{"line_number":37,"context_line":"#. Contract. This would apply changes that are backward compatible with"},{"line_number":38,"context_line":"   new running code."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"A subset of schema changes will have DDL automatically generated by"},{"line_number":41,"context_line":"comparing the running database schema against the model defined in"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_3a4ab9c5","line":38,"in_reply_to":"1ae5cdf2_9a78e1f9","updated":"2014-07-02 04:37:44.000000000","message":"Erdfelt, can you add the discussion of data migration and schema migration to the spec to make it clear?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"cec6af1e9246b167542dcbb4a87e8cc46fbeb262","unresolved":false,"context_lines":[{"line_number":35,"context_line":"   old running code."},{"line_number":36,"context_line":"#. Migrate. This would apply changes that are necessary to be run offline."},{"line_number":37,"context_line":"#. Contract. This would apply changes that are backward compatible with"},{"line_number":38,"context_line":"   new running code."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"A subset of schema changes will have DDL automatically generated by"},{"line_number":41,"context_line":"comparing the running database schema against the model defined in"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_9eb9b217","line":38,"in_reply_to":"baada198_3a4ab9c5","updated":"2014-07-08 14:22:17.000000000","message":"What isn\u0027t clear about it? I\u0027m not sure what I need to do to make it clearer.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":4573,"name":"yunhong-jiang","email":"yunhong.jiang@intel.com","username":"yunhong-jiang"},"change_message_id":"7ee6f63b89245bacb593786078a73c1a7b209535","unresolved":false,"context_lines":[{"line_number":47,"context_line":"- Non-Unique Index additions"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Schema changes that will be automatically performed during migrate:"},{"line_number":50,"context_line":"- Unique Index additions/removals"},{"line_number":51,"context_line":"- Foreign Key additions/removals"},{"line_number":52,"context_line":"- Column nullability"},{"line_number":53,"context_line":"- Column server default"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_1f6763eb","line":50,"updated":"2014-06-25 23:49:28.000000000","message":"Unique Index is something need more consideration because the unique index may fail if new column added and a unique index for it is also added.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":47,"context_line":"- Non-Unique Index additions"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Schema changes that will be automatically performed during migrate:"},{"line_number":50,"context_line":"- Unique Index additions/removals"},{"line_number":51,"context_line":"- Foreign Key additions/removals"},{"line_number":52,"context_line":"- Column nullability"},{"line_number":53,"context_line":"- Column server default"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_1ada11e9","line":50,"in_reply_to":"1ae5cdf2_1f6763eb","updated":"2014-06-26 01:43:24.000000000","message":"My plan was to try adding the unique index and if it fails, it gives a graceful error.\n\nThe user can then use a tool to find duplicate rows and manually fix them (or perhaps the tool can help clean those up too).\n\nThere is precedent for this strategy in the instances.uuid uniqueness migration:\n\nhttps://review.openstack.org/#/c/97300/","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"e3d0d0c754fc85b53d662fbca0589a8c0eb7eb8f","unresolved":false,"context_lines":[{"line_number":50,"context_line":"- Unique Index additions/removals"},{"line_number":51,"context_line":"- Foreign Key additions/removals"},{"line_number":52,"context_line":"- Column nullability"},{"line_number":53,"context_line":"- Column server default"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Schema changes that will be automatically performed during contract:"},{"line_number":56,"context_line":"- Table drops"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_8cc60567","line":53,"updated":"2014-06-25 15:37:52.000000000","message":"might be good to add the why bit here.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":50,"context_line":"- Unique Index additions/removals"},{"line_number":51,"context_line":"- Foreign Key additions/removals"},{"line_number":52,"context_line":"- Column nullability"},{"line_number":53,"context_line":"- Column server default"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Schema changes that will be automatically performed during contract:"},{"line_number":56,"context_line":"- Table drops"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_ba991d21","line":53,"in_reply_to":"1ae5cdf2_8cc60567","updated":"2014-06-26 01:43:24.000000000","message":"Sure, I\u0027ll expand on this a bit.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7d99d5eb73ecc3cefc0bec115c20daca898346f2","unresolved":false,"context_lines":[{"line_number":75,"context_line":"easily automate in testing was the \u0027deleted\u0027 column type change."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Each of the \u0027expand\u0027, \u0027migrate\u0027 and \u0027contract\u0027 phases would verify the"},{"line_number":78,"context_line":"previous phases were executed before continuing."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"The existing \u0027db sync\u0027 command would be reimplemented to run the \u0027expand\u0027"},{"line_number":81,"context_line":"\u0027migrate\u0027 and \u0027contract\u0027 phases. This would provide a backwards compatible"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_495cef4c","line":78,"updated":"2014-06-25 15:28:20.000000000","message":"How? Are we going to store information about the phase each schema version has been through in the database?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":75,"context_line":"easily automate in testing was the \u0027deleted\u0027 column type change."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Each of the \u0027expand\u0027, \u0027migrate\u0027 and \u0027contract\u0027 phases would verify the"},{"line_number":78,"context_line":"previous phases were executed before continuing."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"The existing \u0027db sync\u0027 command would be reimplemented to run the \u0027expand\u0027"},{"line_number":81,"context_line":"\u0027migrate\u0027 and \u0027contract\u0027 phases. This would provide a backwards compatible"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_bac03de1","line":78,"in_reply_to":"1ae5cdf2_495cef4c","updated":"2014-06-26 01:43:24.000000000","message":"Before doing a \u0027migrate\u0027, the code runs the same path as \u0027expand\u0027 phase and generates a list of changes (but doesn\u0027t run them). If it\u0027s not-empty, then the \u0027expand\u0027 phase hasn\u0027t been run.\n\nSame thing with the \u0027contract\u0027 phase, except it checks both the \u0027expand\u0027 and \u0027migrate\u0027 phases.\n\nSince running the sqlalchemy-migrate migrations still get run as part of the \u0027migrate\u0027 phase, that code also checks the version that sqlalchemy-migrate is currently at (using the existing logic that nova already has).","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7d99d5eb73ecc3cefc0bec115c20daca898346f2","unresolved":false,"context_lines":[{"line_number":80,"context_line":"The existing \u0027db sync\u0027 command would be reimplemented to run the \u0027expand\u0027"},{"line_number":81,"context_line":"\u0027migrate\u0027 and \u0027contract\u0027 phases. This would provide a backwards compatible"},{"line_number":82,"context_line":"and simpler way to upgrade the database for those that don\u0027t wish to run"},{"line_number":83,"context_line":"migrations online."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"A new \u0027--dryrun\u0027 argument would print, instead of execute, each generated"},{"line_number":86,"context_line":"DDL statement. This could be used by database administrators to see what"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_0943c76c","line":83,"updated":"2014-06-25 15:28:20.000000000","message":"Does this mean we call db_sync with an argument now, or are you proposing this be automated in some way?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":80,"context_line":"The existing \u0027db sync\u0027 command would be reimplemented to run the \u0027expand\u0027"},{"line_number":81,"context_line":"\u0027migrate\u0027 and \u0027contract\u0027 phases. This would provide a backwards compatible"},{"line_number":82,"context_line":"and simpler way to upgrade the database for those that don\u0027t wish to run"},{"line_number":83,"context_line":"migrations online."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"A new \u0027--dryrun\u0027 argument would print, instead of execute, each generated"},{"line_number":86,"context_line":"DDL statement. This could be used by database administrators to see what"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_3a734de8","line":83,"in_reply_to":"1ae5cdf2_0943c76c","updated":"2014-06-26 01:43:24.000000000","message":"Your comment has me confused :)\n\nThis paragraph effectively means this:\n\n    def db_sync(self):\n        self.db_expand()\n        self.db_migrate()\n        self.db_contract()\n\n\u0027db sync\u0027 will be reimplemented in terms of \u0027expand\u0027, \u0027migrate\u0027 and \u0027contract\u0027. This means deployers don\u0027t have to change their deployment method unless they want to take advantage of the online schema changes. (If they do, they run the \u0027expand\u0027, \u0027migrate\u0027 and \u0027contract\u0027 commands but not the \u0027sync\u0027)","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"e3d0d0c754fc85b53d662fbca0589a8c0eb7eb8f","unresolved":false,"context_lines":[{"line_number":137,"context_line":""},{"line_number":138,"context_line":"The expand and contract phases are optional. If not explicitly run"},{"line_number":139,"context_line":"otherwise, the existing call to \u0027nova-manage db sync\u0027 would execute all"},{"line_number":140,"context_line":"necessary schema changes, as is the existing behavior."},{"line_number":141,"context_line":""},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"Performance Impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_ec830953","line":140,"updated":"2014-06-25 15:37:52.000000000","message":"This is not end user impact, its \"deployer\" impact really.\n\nHere end user means people access the external REST API and starting servers, etc.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":137,"context_line":""},{"line_number":138,"context_line":"The expand and contract phases are optional. If not explicitly run"},{"line_number":139,"context_line":"otherwise, the existing call to \u0027nova-manage db sync\u0027 would execute all"},{"line_number":140,"context_line":"necessary schema changes, as is the existing behavior."},{"line_number":141,"context_line":""},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"Performance Impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_ba4e7d94","line":140,"in_reply_to":"1ae5cdf2_ec830953","updated":"2014-06-26 01:43:24.000000000","message":"I\u0027ll move this to the right location.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":9060,"name":"Mike Dorman","email":"dorm@dorm.org","username":"gd-mdorman"},"change_message_id":"a6e40093cfd6d1efdd19db1274f382d7da7dc277","unresolved":false,"context_lines":[{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Those deployers that want to take advantage of the online schema changes"},{"line_number":154,"context_line":"will need to run the \u0027expand\u0027, \u0027migrate\u0027 and \u0027contract\u0027 commands at the"},{"line_number":155,"context_line":"appropriate steps in their deployment process."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Deployers that have made local schema changes (extra indexes, columns,"},{"line_number":158,"context_line":"tables, etc) will need to update the model to ensure those additions"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_3d499932","line":155,"updated":"2014-06-25 16:43:36.000000000","message":"Are these commands run with dbsync for online changes?  Reading above left me with the impression that dbsync would only be used for offline migrations (as now), and that online schema changes would be done automatically by the nova itself.  Maybe clarify this a little bit?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"d8ecf8b0c913024d521d0de0fc3ba71b9e770a67","unresolved":false,"context_lines":[{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Those deployers that want to take advantage of the online schema changes"},{"line_number":154,"context_line":"will need to run the \u0027expand\u0027, \u0027migrate\u0027 and \u0027contract\u0027 commands at the"},{"line_number":155,"context_line":"appropriate steps in their deployment process."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Deployers that have made local schema changes (extra indexes, columns,"},{"line_number":158,"context_line":"tables, etc) will need to update the model to ensure those additions"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_7afeb507","line":155,"in_reply_to":"1ae5cdf2_3d499932","updated":"2014-06-26 01:43:24.000000000","message":"Hopefully the comment I added above helps? If not, let me know.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"e3d0d0c754fc85b53d662fbca0589a8c0eb7eb8f","unresolved":false,"context_lines":[{"line_number":163,"context_line":"----------------"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"Keeping the model updated with database migrations will be much more"},{"line_number":166,"context_line":"important. A new unit test will enforce this."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_ecea290d","line":166,"updated":"2014-06-25 15:37:52.000000000","message":"Cool.","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":4573,"name":"yunhong-jiang","email":"yunhong.jiang@intel.com","username":"yunhong-jiang"},"change_message_id":"632b1393d830a3ac379e620fb805fdbcf38abae7","unresolved":false,"context_lines":[{"line_number":201,"context_line":""},{"line_number":202,"context_line":"No tempest tests will be added since tempest does not do any upgrade"},{"line_number":203,"context_line":"testing."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"turbo-hipster will be leveraged to ensure that each of the expand,"},{"line_number":206,"context_line":"migrate and contract phases operate correctly on production database"},{"line_number":207,"context_line":"snapshots."}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ae5cdf2_48ae8ee8","line":204,"updated":"2014-07-02 04:37:44.000000000","message":"I think we need the upgrade test, possibly as part of grenade?","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"cec6af1e9246b167542dcbb4a87e8cc46fbeb262","unresolved":false,"context_lines":[{"line_number":201,"context_line":""},{"line_number":202,"context_line":"No tempest tests will be added since tempest does not do any upgrade"},{"line_number":203,"context_line":"testing."},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"turbo-hipster will be leveraged to ensure that each of the expand,"},{"line_number":206,"context_line":"migrate and contract phases operate correctly on production database"},{"line_number":207,"context_line":"snapshots."}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_fe554efa","line":204,"in_reply_to":"1ae5cdf2_48ae8ee8","updated":"2014-07-08 14:22:17.000000000","message":"I can add something about grenade (which is nothing will be changing as long as it continues to run \u0027db sync\u0027)","commit_id":"1c5e5216f04e28064004a9b6aef198f008388e4a"}],"specs/kilo/approved/online-schema-changes.rst":[{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"31bb6f9e06c442c4efcab02347dccce4679403f3","unresolved":false,"context_lines":[{"line_number":77,"context_line":"Schema changes that will be automatically performed during contract:"},{"line_number":78,"context_line":"- Table drops"},{"line_number":79,"context_line":"- Column drops"},{"line_number":80,"context_line":"- Non-Unique Index drops"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Those schema changes that are not listed will need to be manually made"},{"line_number":83,"context_line":"as part of an sqlalchemy-migrate migration. As an example, column type"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a890539_dd797b83","line":80,"updated":"2014-11-24 09:56:57.000000000","message":"The slight issue is, if we use objects to perform the data migration, then need access to both the old and new column.\n\nSo if you remove the old column from the model, you would need to select that using raw SQL.\n\nMaybe thats fine, but I have a feeling we need to describe that here?","commit_id":"b466fbc6eded27066a2f910138f27b1aa1bf5400"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"ee9c84830b8de273a84b5aa54d5a6ae2cd937855","unresolved":false,"context_lines":[{"line_number":77,"context_line":"Schema changes that will be automatically performed during contract:"},{"line_number":78,"context_line":"- Table drops"},{"line_number":79,"context_line":"- Column drops"},{"line_number":80,"context_line":"- Non-Unique Index drops"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Those schema changes that are not listed will need to be manually made"},{"line_number":83,"context_line":"as part of an sqlalchemy-migrate migration. As an example, column type"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a890539_753ddc8e","line":80,"in_reply_to":"5a890539_dd797b83","updated":"2014-11-24 10:50:46.000000000","message":"I\u0027m currently considering how to properly achieve this for substractive changes like column removal.\nIf we consider that a scheme change will remove a table col, then the object will get a dict from SQLA not containing that field. That\u0027s up to the _from_db_object() to carry that logic and consider that if the field is not present in the object (because objects will be bumped accordingly to the DB scheme), it should failback to something else.","commit_id":"b466fbc6eded27066a2f910138f27b1aa1bf5400"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"31bb6f9e06c442c4efcab02347dccce4679403f3","unresolved":false,"context_lines":[{"line_number":127,"context_line":""},{"line_number":128,"context_line":"This would also require changes to sqlalchemy-migrate to be able to"},{"line_number":129,"context_line":"manage seperate streams of migrations."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"Data model impact"},{"line_number":133,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a890539_e73762e1","line":130,"updated":"2014-11-24 09:56:57.000000000","message":"If we reject keeping the old an new in parallel, thats OK with me, but we should probably put something in here about that.","commit_id":"b466fbc6eded27066a2f910138f27b1aa1bf5400"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"31bb6f9e06c442c4efcab02347dccce4679403f3","unresolved":false,"context_lines":[{"line_number":218,"context_line":"- Implement schema synchronizer"},{"line_number":219,"context_line":"- Implement new \u0027expand\u0027, \u0027migrate\u0027 and \u0027contract\u0027 commands to"},{"line_number":220,"context_line":"  \u0027nova-manage db\u0027"},{"line_number":221,"context_line":"- Reimplement \u0027db sync\u0027 to use the new expand, migrate and contract calls"},{"line_number":222,"context_line":"- Drop schema changes from existing migrations that are now done"},{"line_number":223,"context_line":"  automatically as part of the schema synchronizer"},{"line_number":224,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a890539_11b3ece9","line":221,"updated":"2014-11-24 09:56:57.000000000","message":"Is it possible to drop this for now, so we keep the old and new systems at the same time?","commit_id":"b466fbc6eded27066a2f910138f27b1aa1bf5400"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"31bb6f9e06c442c4efcab02347dccce4679403f3","unresolved":false,"context_lines":[{"line_number":243,"context_line":""},{"line_number":244,"context_line":"turbo-hipster tests upgrades using production database snapshots. No"},{"line_number":245,"context_line":"changes are necessary since it currently uses \u0027nova-manage db sync\u0027 as"},{"line_number":246,"context_line":"well."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a890539_87069e1a","line":246,"updated":"2014-11-24 09:56:57.000000000","message":"If we keep the old and new approaches both working, we end up need to add tests to explicitly call the new methods, and compare the old an new methods.\n\nThere was talk of in tree tests to ensure the existing migrations and model are in sync. Are these covered elsewhere?","commit_id":"b466fbc6eded27066a2f910138f27b1aa1bf5400"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"31bb6f9e06c442c4efcab02347dccce4679403f3","unresolved":false,"context_lines":[{"line_number":244,"context_line":"turbo-hipster tests upgrades using production database snapshots. No"},{"line_number":245,"context_line":"changes are necessary since it currently uses \u0027nova-manage db sync\u0027 as"},{"line_number":246,"context_line":"well."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"Documentation Impact"},{"line_number":250,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a890539_0748ee5a","line":247,"updated":"2014-11-24 09:56:57.000000000","message":"There was talk of tests that work the set of migrations we have, we were wondering about how we prove any commit to any commit works with the expand contract? It might be good to talk about previous release to current commit, maybe? Not sure if we need to think about anything inbetween, probably not I guess?","commit_id":"b466fbc6eded27066a2f910138f27b1aa1bf5400"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"1bbcb8502aa56db5be45a2197c1183b4c874c560","unresolved":false,"context_lines":[{"line_number":61,"context_line":"The three new commands would be built by dynamically executing alembic\u0027s"},{"line_number":62,"context_line":"autogenerate and DDL generating features. A list of differences would"},{"line_number":63,"context_line":"be generated by alembic and then DDL statements would be generated using"},{"line_number":64,"context_line":"a separate feature of alembic."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"The set of DDL statements that can be run in each phase would be dictated"},{"line_number":67,"context_line":"by the database software used (eg MySQL, PostgreSQL, etc), the version of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_567ec1e5","line":64,"range":{"start_line":64,"start_character":22,"end_line":64,"end_character":29},"updated":"2014-12-07 20:10:07.000000000","message":"We don\u0027t use alembic in nova at the moment. I assume that this isn\u0027t saying we need to port the sqlalchemy-migrate migrations to alembic? Would new migrations continue to be written in sqlalchemy-migrate?","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"1fc776ec7ac709bee6ec8dc22678f75377c57a48","unresolved":false,"context_lines":[{"line_number":88,"context_line":"Those schema changes that cannot be generated automatically will be"},{"line_number":89,"context_line":"forbidden. As an example, column type changes will no longer be allowed"},{"line_number":90,"context_line":"because it is generally not safe to be executed online and it is"},{"line_number":91,"context_line":"difficult to automate on all databases."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"The migrate and contract phases would verify that the previous phases"},{"line_number":94,"context_line":"(expand in the case of migrate, expand and migrate in the case of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"5a890539_16727d11","line":91,"updated":"2014-11-26 19:12:58.000000000","message":"This essentially means that we are stuck with the pretty awful column type choices that infect the current Nova schema (VARCHAR(255) for lots of things, unnecessarily). Is there a plan to allow column type changes some time in the future? There are certainly ways to reduce the majority of downtime when changing column types, including creating a new column with the new type, batch updating the new column values to contain the (converted) values from the old column, and then dropping the old column and renaming the new column to the old column (a very fast operation). Only the final step (drop and rename column) would need to be done offline.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"f62dbf8784bdcdb51f12330b71526166f66ba528","unresolved":false,"context_lines":[{"line_number":88,"context_line":"Those schema changes that cannot be generated automatically will be"},{"line_number":89,"context_line":"forbidden. As an example, column type changes will no longer be allowed"},{"line_number":90,"context_line":"because it is generally not safe to be executed online and it is"},{"line_number":91,"context_line":"difficult to automate on all databases."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"The migrate and contract phases would verify that the previous phases"},{"line_number":94,"context_line":"(expand in the case of migrate, expand and migrate in the case of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_42ffb867","line":91,"in_reply_to":"3a961159_b66c4538","updated":"2014-12-08 23:26:10.000000000","message":"I left this strong because it\u0027s a complicated subject and the database engines right now don\u0027t handle it well.\n\nFor instance, I don\u0027t think any database engines we support can change the type of a column with blocking reads/writes.\n\nThere are also a lot of situations where column type changes won\u0027t be compatible with running code (which is the eventual end goal, every schema change is run online).\n\nI didn\u0027t want to complicate this spec by diving into this nuanced subject too much.\n\nThat said, I can see loosening this up. Perhaps by saying that initially column type changes won\u0027t be handled, but this may change in the future?","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"1bbcb8502aa56db5be45a2197c1183b4c874c560","unresolved":false,"context_lines":[{"line_number":88,"context_line":"Those schema changes that cannot be generated automatically will be"},{"line_number":89,"context_line":"forbidden. As an example, column type changes will no longer be allowed"},{"line_number":90,"context_line":"because it is generally not safe to be executed online and it is"},{"line_number":91,"context_line":"difficult to automate on all databases."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"The migrate and contract phases would verify that the previous phases"},{"line_number":94,"context_line":"(expand in the case of migrate, expand and migrate in the case of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_b66c4538","line":91,"in_reply_to":"5a890539_16727d11","updated":"2014-12-07 20:10:07.000000000","message":"@Jay -- I think the config drive column in the instances table is the canonical example of this. We use a varchar(255) to store a boolean, because the value used to be a UUID. We\u0027ve never been willing to take the downtime to clean that up.\n\nI\u0027d like to see that sort of thing addressed too, but I feel it should be done in a later spec. Let\u0027s implement the minimum viable set of features here for now, and then grow it over time instead of blocking on perfection.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"1fc776ec7ac709bee6ec8dc22678f75377c57a48","unresolved":false,"context_lines":[{"line_number":103,"context_line":"to see what would be executed for a particular phase. These can be"},{"line_number":104,"context_line":"optionally executed manually if desired. The schema synchronizer will"},{"line_number":105,"context_line":"not generate that DDL statement since the running schema does not"},{"line_number":106,"context_line":"have that difference anymore."},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"When \u0027db contract\u0027 is finally run, the version in the migrate_version"},{"line_number":109,"context_line":"table would be updated to the latest version. This would maintain"}],"source_content_type":"text/x-rst","patch_set":7,"id":"5a890539_365e997a","line":106,"updated":"2014-11-26 19:12:58.000000000","message":"++ nice.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"1bbcb8502aa56db5be45a2197c1183b4c874c560","unresolved":false,"context_lines":[{"line_number":110,"context_line":"compatibility with the existing sqlalchemy-migrate based migrations."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"It is intended that eventually the use of sqlalchemy-migrate would"},{"line_number":113,"context_line":"be dropped in the future."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_167439fc","line":113,"range":{"start_line":113,"start_character":18,"end_line":113,"end_character":24},"updated":"2014-12-07 20:10:07.000000000","message":"I think these two paras address my question on line 64 somewhat, but I still think we should clarify that migrations continue to be written in sqlalchemy-migrate for now.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"f62dbf8784bdcdb51f12330b71526166f66ba528","unresolved":false,"context_lines":[{"line_number":110,"context_line":"compatibility with the existing sqlalchemy-migrate based migrations."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"It is intended that eventually the use of sqlalchemy-migrate would"},{"line_number":113,"context_line":"be dropped in the future."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_823fe024","line":113,"in_reply_to":"3a961159_167439fc","updated":"2014-12-08 23:26:10.000000000","message":"I\u0027ll make this clearer","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"1bbcb8502aa56db5be45a2197c1183b4c874c560","unresolved":false,"context_lines":[{"line_number":181,"context_line":""},{"line_number":182,"context_line":"The expand and contract phases are optional. If not explicitly run"},{"line_number":183,"context_line":"otherwise, the existing call to \u0027nova-manage db sync\u0027 would execute all"},{"line_number":184,"context_line":"necessary schema changes, as is the existing behavior."},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"Those deployers that want to take advantage of the online schema changes"},{"line_number":187,"context_line":"will need to run the \u0027db expand\u0027, \u0027db migrate\u0027 and \u0027db contract\u0027 commands"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_36605535","line":184,"updated":"2014-12-07 20:10:07.000000000","message":"Should we require contract to be run ever? For example, I think that perhaps we should require it to be run at least once per release cycle.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"f62dbf8784bdcdb51f12330b71526166f66ba528","unresolved":false,"context_lines":[{"line_number":181,"context_line":""},{"line_number":182,"context_line":"The expand and contract phases are optional. If not explicitly run"},{"line_number":183,"context_line":"otherwise, the existing call to \u0027nova-manage db sync\u0027 would execute all"},{"line_number":184,"context_line":"necessary schema changes, as is the existing behavior."},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"Those deployers that want to take advantage of the online schema changes"},{"line_number":187,"context_line":"will need to run the \u0027db expand\u0027, \u0027db migrate\u0027 and \u0027db contract\u0027 commands"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_2218ecd2","line":184,"in_reply_to":"3a961159_36605535","updated":"2014-12-08 23:26:10.000000000","message":"I think we should probably require it. If we don\u0027t, then old column names can never be reused in the future.\n\nOnce per release cycle seems reasonable.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"1fc776ec7ac709bee6ec8dc22678f75377c57a48","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"Deployers that have made local schema changes (extra indexes, columns,"},{"line_number":191,"context_line":"tables, etc) will need to update the model to ensure those additions"},{"line_number":192,"context_line":"aren\u0027t dropped during the contract phase."},{"line_number":193,"context_line":""},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"Developer impact"}],"source_content_type":"text/x-rst","patch_set":7,"id":"5a890539_56632541","line":192,"updated":"2014-11-26 19:12:58.000000000","message":"I think it\u0027s probably worth mentioning here that this technique won\u0027t work for deployments that use the same node for running `nova-manage db sync` as the one running nova-conductor. Or at least, deployments that used the same node to run db sync as are running nova-api would need to be extremely careful not to restart the nova-api service before nova-manage db expand was run. Otherwise, the nova-conductor code would be using a set of SQlalchemy models that did not correspond to the database schema. I bring this up because up until this blueprint, the typical upgrade procedures are to have the Puppet manifest or Chef recipe pull the latest Git or package sources for nova-conductor, stop the nova-conductor service, run nova-manage db sync, and then start nova-conductor again. With this \"online schema migration\" strategy, there should be no stopping of the nova-conductor service before running nova-manage db expand. If the deployer forgets to NOT stop nova-conductor before running db expand, and keeps using their old scripts that did the stop/sync/start stuff, simply replacing \"sync\" with \"expand\", then BOOM. :) Probably worth noting this in the release notes at the very least.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"f62dbf8784bdcdb51f12330b71526166f66ba528","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"Deployers that have made local schema changes (extra indexes, columns,"},{"line_number":191,"context_line":"tables, etc) will need to update the model to ensure those additions"},{"line_number":192,"context_line":"aren\u0027t dropped during the contract phase."},{"line_number":193,"context_line":""},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"Developer impact"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_22370c01","line":192,"in_reply_to":"5a890539_56632541","updated":"2014-12-08 23:26:10.000000000","message":"I\u0027ll make sure to add a paragraph addressing the deploy flow to make it clearer what the two possible deploy paths are now.\n\nHowever, I\u0027m not sure I understand the problem you\u0027re describing.\n\nThe deployer can stick with \u0027db sync\u0027 if they would like. That would still use the sqlalchemy-migrate path. However, they can alternatively use the \u0027db {expand,migrate,contract}\u0027 path. That does not use sqlalchemy-migrate at all. Unfortunately, you can\u0027t mix them (within a deploy).\n\nSo, replacing \u0027db sync\u0027 with \u0027db expand\u0027 will potentially leave them in a midstate with some schema changes, but not all (because \u0027db migrate\u0027 wasn\u0027t run).\n\nAlso, running \u0027db expand\u0027 and then \u0027db sync\u0027 won\u0027t work since sqlalchemy-migrate is unaware of the schema changes that \u0027db expand\u0027 has done.\n\nMy original version of the spec changes \u0027db sync\u0027 to call \u0027db {expand,migrate,contract}\u0027 but it was requested to be conservative and maintain the old sqlalchemy-migrate migrations in parallel for now.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"1fc776ec7ac709bee6ec8dc22678f75377c57a48","unresolved":false,"context_lines":[{"line_number":254,"context_line":""},{"line_number":255,"context_line":"turbo-hipster tests upgrades using production database snapshots. No"},{"line_number":256,"context_line":"changes are necessary since it currently uses \u0027nova-manage db sync\u0027 as"},{"line_number":257,"context_line":"well."},{"line_number":258,"context_line":""},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":7,"id":"5a890539_7658218b","line":257,"updated":"2014-11-26 19:12:58.000000000","message":"Would we at least want to have a turbo-hipster job that ran expand/migrate/contract instead of sync?","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"f62dbf8784bdcdb51f12330b71526166f66ba528","unresolved":false,"context_lines":[{"line_number":254,"context_line":""},{"line_number":255,"context_line":"turbo-hipster tests upgrades using production database snapshots. No"},{"line_number":256,"context_line":"changes are necessary since it currently uses \u0027nova-manage db sync\u0027 as"},{"line_number":257,"context_line":"well."},{"line_number":258,"context_line":""},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_e22c24ae","line":257,"in_reply_to":"3a961159_765add84","updated":"2014-12-08 23:26:10.000000000","message":"I\u0027ll clean this up to ensure both deploy paths are run by turbo-hipster.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"1bbcb8502aa56db5be45a2197c1183b4c874c560","unresolved":false,"context_lines":[{"line_number":254,"context_line":""},{"line_number":255,"context_line":"turbo-hipster tests upgrades using production database snapshots. No"},{"line_number":256,"context_line":"changes are necessary since it currently uses \u0027nova-manage db sync\u0027 as"},{"line_number":257,"context_line":"well."},{"line_number":258,"context_line":""},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_765add84","line":257,"in_reply_to":"5a890539_7658218b","updated":"2014-12-07 20:10:07.000000000","message":"We can certainly add those tests if people would like them.","commit_id":"b0b56109e3e0de1181b428340ac03107220db43b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"adac12494d95b4e42d00adcdd90d64f765b4f775","unresolved":false,"context_lines":[{"line_number":113,"context_line":"to see what would be executed for a particular phase. These can be"},{"line_number":114,"context_line":"optionally executed manually if desired. The schema synchronizer will"},{"line_number":115,"context_line":"not generate that DDL statement since the running schema does not"},{"line_number":116,"context_line":"have that difference anymore."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"When \u0027db contract\u0027 is finally run and the running schema has been"},{"line_number":119,"context_line":"verified to match the models, the version in the migrate_version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_cf5b48f4","line":116,"updated":"2014-12-12 09:22:14.000000000","message":"FYI, that\u0027s out-of-box when using Alembic \nhttp://alembic.readthedocs.org/en/latest/offline.html","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"42563f81432a22519f392f8e8abd0d6deeba1c52","unresolved":false,"context_lines":[{"line_number":113,"context_line":"to see what would be executed for a particular phase. These can be"},{"line_number":114,"context_line":"optionally executed manually if desired. The schema synchronizer will"},{"line_number":115,"context_line":"not generate that DDL statement since the running schema does not"},{"line_number":116,"context_line":"have that difference anymore."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"When \u0027db contract\u0027 is finally run and the running schema has been"},{"line_number":119,"context_line":"verified to match the models, the version in the migrate_version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_5171aba6","line":116,"in_reply_to":"3a961159_23121c62","updated":"2014-12-12 17:12:59.000000000","message":"That\u0027s nice, I missed it in L69. Could you plase mark the setup of Alembic as a first work item so it would be clearier ?","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"82c6ed7307d28d2e46ef208b6aebc361ad39bcdf","unresolved":false,"context_lines":[{"line_number":113,"context_line":"to see what would be executed for a particular phase. These can be"},{"line_number":114,"context_line":"optionally executed manually if desired. The schema synchronizer will"},{"line_number":115,"context_line":"not generate that DDL statement since the running schema does not"},{"line_number":116,"context_line":"have that difference anymore."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"When \u0027db contract\u0027 is finally run and the running schema has been"},{"line_number":119,"context_line":"verified to match the models, the version in the migrate_version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_6f44021b","line":116,"in_reply_to":"3a961159_5171aba6","updated":"2014-12-12 18:00:26.000000000","message":"No setup of Alembic is necessary in this spec since a dependent spec is already starting to use some of Alembic:\n\nhttps://review.openstack.org/#/c/134984/\n\nHere are the remaining patches to be merged:\n\nhttps://review.openstack.org/#/q/status:open+project:openstack/nova+branch:master+topic:bp/validate-migrations-and-model,n,z","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"08e4b16c44172bf5d7bf04145450fb532068a304","unresolved":false,"context_lines":[{"line_number":113,"context_line":"to see what would be executed for a particular phase. These can be"},{"line_number":114,"context_line":"optionally executed manually if desired. The schema synchronizer will"},{"line_number":115,"context_line":"not generate that DDL statement since the running schema does not"},{"line_number":116,"context_line":"have that difference anymore."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"When \u0027db contract\u0027 is finally run and the running schema has been"},{"line_number":119,"context_line":"verified to match the models, the version in the migrate_version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_23121c62","line":116,"in_reply_to":"3a961159_cf5b48f4","updated":"2014-12-12 15:31:12.000000000","message":"I\u0027m not sure where the confusion is coming from, but I tried to make it clear that this implementation uses Alembic under the hood. Of course it leverages Alembic to provide this.","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"ea570bf1ee5d526995c90994960f7e7dc586c41b","unresolved":false,"context_lines":[{"line_number":121,"context_line":"migration version. This would maintain compatibility with the"},{"line_number":122,"context_line":"existing sqlalchemy-migrate workflow."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Alternatives"},{"line_number":126,"context_line":"------------"},{"line_number":127,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_2fbccc05","line":124,"updated":"2014-12-12 09:19:28.000000000","message":"I know that\u0027s an honey pot, but could we maybe envisage moving to Alembic as part of this spec ? That sounds a good opportunity IMHO.","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"08e4b16c44172bf5d7bf04145450fb532068a304","unresolved":false,"context_lines":[{"line_number":121,"context_line":"migration version. This would maintain compatibility with the"},{"line_number":122,"context_line":"existing sqlalchemy-migrate workflow."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Alternatives"},{"line_number":126,"context_line":"------------"},{"line_number":127,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_8e94d78b","line":124,"in_reply_to":"3a961159_2fbccc05","updated":"2014-12-12 15:31:12.000000000","message":"Porting all of the sqlalchemy-migrate migration scripts we currently have over to Alembic only to delete them seems like a waste of time.","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"82c6ed7307d28d2e46ef208b6aebc361ad39bcdf","unresolved":false,"context_lines":[{"line_number":121,"context_line":"migration version. This would maintain compatibility with the"},{"line_number":122,"context_line":"existing sqlalchemy-migrate workflow."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Alternatives"},{"line_number":126,"context_line":"------------"},{"line_number":127,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_ef4392ac","line":124,"in_reply_to":"3a961159_3157bf03","updated":"2014-12-12 18:00:26.000000000","message":"I still think you are misunderstanding what this spec does.\n\nThe expand and contract commands use no scripts. All changes are dynamically made by comparing the model to the running schema and then generating DDL to make the running schema match the model.\n\nThe current version of this spec proposes that there will be two parallel paths to perform the same thing:\n\n1) The existing sqlalchemy-migrate migrations using \u0027db sync\u0027\n\n2) The new expand/contract migrations using the new \u0027db expand\u0027, \u0027db migrate\u0027 and \u0027db contract\u0027 commands.\n\nYou can use either of them, but not both within the same code deploy. They are semantically identical in that the results will be the same for both.\n\nHowever, how they perform the schema changes is completely different.\n\nThis is different than how this spec was originally proposed. That proposal completely dropped sqlalchemy-migrate since technically there is no need for two different ways to perform the same goal.\n\nHowever, there was some concern considering this is a non-trivial change, and it was fairly simple to maintain reasonable levels of compatibility with sqlalchemy-migrate in the short term to be a little conservative.","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"42563f81432a22519f392f8e8abd0d6deeba1c52","unresolved":false,"context_lines":[{"line_number":121,"context_line":"migration version. This would maintain compatibility with the"},{"line_number":122,"context_line":"existing sqlalchemy-migrate workflow."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Alternatives"},{"line_number":126,"context_line":"------------"},{"line_number":127,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3a961159_3157bf03","line":124,"in_reply_to":"3a961159_8e94d78b","updated":"2014-12-12 17:12:59.000000000","message":"I was not saying we should backport the current scripts to Alembic, but rather was proposing to provide Alembic for the *new* scripts using the expand and contract commands :-)","commit_id":"d92924937b5f4b6d37b65f476af671274df5978b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"7bb7115e0060f44b28b787793469b40fcf3b32ea","unresolved":false,"context_lines":[{"line_number":53,"context_line":"The new expand/contract workflow will not utilize any migration scripts,"},{"line_number":54,"context_line":"instead it will dynamically compare the running schema against the"},{"line_number":55,"context_line":"database model defined in Nova. DDL statements will be generated, and"},{"line_number":56,"context_line":"optionally executed, to make the running schema match the model."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"The existing schema management workflow is the \u0027db sync\u0027 command to"},{"line_number":59,"context_line":"nova-manage. This is managed by sqlalchemy-migrate and uses individual"}],"source_content_type":"text/x-rst","patch_set":9,"id":"3a961159_81926549","line":56,"updated":"2014-12-17 14:22:07.000000000","message":"thanks for rewriting this paragraph, much more understandable by me","commit_id":"d39c87cc9a84077126213ab3c4432d74f121382f"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"42ae86b4d40808fb999586015684adb1c14d6dac","unresolved":false,"context_lines":[{"line_number":70,"context_line":"   code."},{"line_number":71,"context_line":"#. migrate. This would apply changes that are necessary to be run offline."},{"line_number":72,"context_line":"#. contract. This would apply changes that are compatible with new"},{"line_number":73,"context_line":"   running code."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"Those schema changes that can be safely and semantically applied while"},{"line_number":76,"context_line":"running online will be applied during the expand and contract phases."}],"source_content_type":"text/x-rst","patch_set":9,"id":"3a961159_20fcf6b4","line":73,"updated":"2015-01-06 15:07:17.000000000","message":"This is a small detail, but we probably want to have some way to prevent (or discourage) people from mixing the two methods. Like, running expand and then db-sync.","commit_id":"d39c87cc9a84077126213ab3c4432d74f121382f"},{"author":{"_account_id":4573,"name":"yunhong-jiang","email":"yunhong.jiang@intel.com","username":"yunhong-jiang"},"change_message_id":"085d081facd547e296dc25ad48b25cb0e5888a12","unresolved":false,"context_lines":[{"line_number":91,"context_line":""},{"line_number":92,"context_line":"As an example, index additions can be executed online in MySQL 5.5, but"},{"line_number":93,"context_line":"not 5.1. An index addition would be run during the expand phase for"},{"line_number":94,"context_line":"MySQL 5.5 or higher, but during the migrate phase for MySQL 5.1."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"It is intended that the initial set that will run online will be"},{"line_number":97,"context_line":"conservative at first and a subset of what is possible to run safely."}],"source_content_type":"text/x-rst","patch_set":9,"id":"3a961159_7c38b73d","line":94,"updated":"2014-12-17 18:30:18.000000000","message":"I\u0027m a bit concerned by this statement. Does that mean the developer has to be aware of this difference, or will the wrapped by alembic? I didn\u0027t find such information on alembic\u0027s document.","commit_id":"d39c87cc9a84077126213ab3c4432d74f121382f"},{"author":{"_account_id":100,"name":"Johannes Erdfelt","email":"johannes@erdfelt.com","username":"johannes.erdfelt"},"change_message_id":"6a0272216b121116cba82e943984db55d2227631","unresolved":false,"context_lines":[{"line_number":91,"context_line":""},{"line_number":92,"context_line":"As an example, index additions can be executed online in MySQL 5.5, but"},{"line_number":93,"context_line":"not 5.1. An index addition would be run during the expand phase for"},{"line_number":94,"context_line":"MySQL 5.5 or higher, but during the migrate phase for MySQL 5.1."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"It is intended that the initial set that will run online will be"},{"line_number":97,"context_line":"conservative at first and a subset of what is possible to run safely."}],"source_content_type":"text/x-rst","patch_set":9,"id":"3a961159_3f4d419a","line":94,"in_reply_to":"3a961159_7c38b73d","updated":"2014-12-17 19:13:43.000000000","message":"Alembic does not know what DDL operations will require locks in the various database engines. For our purposes, alembic only generates the DDL statements we need.\n\nCode in this patch would be aware of the restrictions that various database engines have and schedule the DDL statements to be executed in the appropriate phase given the guarantees of the phase. This may be conservative at first, but it will expand over time.\n\nSince the expand phase is intended to be run online, then only DDL statements that can be run online (ie without blocking other readers/writers of the database) will be executed during the expand phase. If it\u0027s not appropriate for the expand phase, it moves to the migrate phase.\n\nIn that respect, all of this is hidden in the implementation. Developers who only make changes to the model need not be aware of what happens and when.","commit_id":"d39c87cc9a84077126213ab3c4432d74f121382f"}]}