)]}'
{"specs/mitaka/online-schema-migration.rst":[{"author":{"_account_id":6482,"name":"Steve Martinelli","email":"s.martinelli@gmail.com","username":"stevemar"},"change_message_id":"8e8a27580b9b82380c780dc03ea57a78a779be57","unresolved":false,"context_lines":[{"line_number":4,"context_line":""},{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Online schema migration"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"ba8a016a_8fc2aa51","line":7,"updated":"2015-11-19 23:44:59.000000000","message":"clean up the \u003d\u0027s to match title length","commit_id":"7d5752ef46365aeaf488258e3ec09e31a7269ab7"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"53fa0eb57622eda675a2c56ddd8f1b6dd2591b0a","unresolved":false,"context_lines":[{"line_number":31,"context_line":"Keystone was mentioned at the summit in the ops live upgrades session as one"},{"line_number":32,"context_line":"of the projects which are the closest to being zero-downtime [2]_."},{"line_number":33,"context_line":"Even though one of the operators stated that he upgraded keystone without"},{"line_number":34,"context_line":"downtime, this is impossible across major releases due to schema changes which"},{"line_number":35,"context_line":"cause incompatibilities between versions. Incompatibilities are introduced in"},{"line_number":36,"context_line":"part due to lack of commitment of reviewers to the \"cause\" of rolling upgrades,"},{"line_number":37,"context_line":"but even more so, due to lack of tests which could catch those issues."}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_c278642e","line":34,"updated":"2015-11-20 15:16:54.000000000","message":"Just out of curiosity, did this operator explain how?","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":31,"context_line":"Keystone was mentioned at the summit in the ops live upgrades session as one"},{"line_number":32,"context_line":"of the projects which are the closest to being zero-downtime [2]_."},{"line_number":33,"context_line":"Even though one of the operators stated that he upgraded keystone without"},{"line_number":34,"context_line":"downtime, this is impossible across major releases due to schema changes which"},{"line_number":35,"context_line":"cause incompatibilities between versions. Incompatibilities are introduced in"},{"line_number":36,"context_line":"part due to lack of commitment of reviewers to the \"cause\" of rolling upgrades,"},{"line_number":37,"context_line":"but even more so, due to lack of tests which could catch those issues."}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_009e24b8","line":34,"in_reply_to":"ba8a016a_c278642e","updated":"2015-11-23 15:19:37.000000000","message":"He stated that he upgraded instances (which were behind haproxy) one-by-one on a running system, but I suspect there was some downtime that he didn\u0027t account for. He was not concerned with a \"little\" downtime.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":13063,"name":"WEI CHEN","email":"dave.jungler@gmail.com","username":"wei.d.chen"},"change_message_id":"edc9f4ea9d73d724be2d8b5743833e9840ebe768","unresolved":false,"context_lines":[{"line_number":39,"context_line":"This spec could be a statement of commitment by the reviewers of Keystone to"},{"line_number":40,"context_line":"support zero-downtime upgrades."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Keystone is less susceptible to upgrade issues then some of the other projects"},{"line_number":43,"context_line":"in OpenStack, because it doesn\u0027t have cross-service RPC communication. The"},{"line_number":44,"context_line":"only other issue which can cause upgrade problems is cross-version API"},{"line_number":45,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_525bbe54","line":42,"updated":"2015-11-22 06:57:13.000000000","message":"nit: s/then/than","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":39,"context_line":"This spec could be a statement of commitment by the reviewers of Keystone to"},{"line_number":40,"context_line":"support zero-downtime upgrades."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Keystone is less susceptible to upgrade issues then some of the other projects"},{"line_number":43,"context_line":"in OpenStack, because it doesn\u0027t have cross-service RPC communication. The"},{"line_number":44,"context_line":"only other issue which can cause upgrade problems is cross-version API"},{"line_number":45,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_e081a04e","line":42,"in_reply_to":"ba8a016a_525bbe54","updated":"2015-11-23 15:19:37.000000000","message":"Done","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":13063,"name":"WEI CHEN","email":"dave.jungler@gmail.com","username":"wei.d.chen"},"change_message_id":"edc9f4ea9d73d724be2d8b5743833e9840ebe768","unresolved":false,"context_lines":[{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Keystone is less susceptible to upgrade issues then some of the other projects"},{"line_number":43,"context_line":"in OpenStack, because it doesn\u0027t have cross-service RPC communication. The"},{"line_number":44,"context_line":"only other issue which can cause upgrade problems is cross-version API"},{"line_number":45,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":46,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":47,"context_line":"compatibility between those is tested and retained."}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_2d725f7d","line":44,"updated":"2015-11-22 06:57:13.000000000","message":"del/only/ according to my following comments.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Keystone is less susceptible to upgrade issues then some of the other projects"},{"line_number":43,"context_line":"in OpenStack, because it doesn\u0027t have cross-service RPC communication. The"},{"line_number":44,"context_line":"only other issue which can cause upgrade problems is cross-version API"},{"line_number":45,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":46,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":47,"context_line":"compatibility between those is tested and retained."}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_40150ce3","line":44,"in_reply_to":"ba8a016a_2d725f7d","updated":"2015-11-23 15:19:37.000000000","message":"Done","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":13063,"name":"WEI CHEN","email":"dave.jungler@gmail.com","username":"wei.d.chen"},"change_message_id":"edc9f4ea9d73d724be2d8b5743833e9840ebe768","unresolved":false,"context_lines":[{"line_number":44,"context_line":"only other issue which can cause upgrade problems is cross-version API"},{"line_number":45,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":46,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":47,"context_line":"compatibility between those is tested and retained."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Proposed Change"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_8d5cabee","line":47,"updated":"2015-11-22 06:57:13.000000000","message":"Basically, per my understanding there are three factors we should consider for rolling upgrade.\n\n- cross-service RPC communication. (keystone not affected)\n- API compatibility (not covered in this spec, and not necessary to be mentioned in this spec.)\n- DB schema incompatibilities (covered in this spec, which should be talked here.)","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":44,"context_line":"only other issue which can cause upgrade problems is cross-version API"},{"line_number":45,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":46,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":47,"context_line":"compatibility between those is tested and retained."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Proposed Change"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_00d62427","line":47,"in_reply_to":"ba8a016a_8d5cabee","updated":"2015-11-23 15:19:37.000000000","message":"That\u0027s correct. I\u0027m not sure if you are suggesting removing/restructuring this paragraph? For now I will add a following sentence stating that API compatibility will not be furthered discussed in this spec.\n\nI think there should be some mention of it here, so that someone reading it would have an idea of how complete the rolling-upgrade capability of Keystone is.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"53fa0eb57622eda675a2c56ddd8f1b6dd2591b0a","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":53,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":54,"context_line":"alters. There is already a solution which is present in Nova [3]_."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"A patch introducing this unit test is under review in Keystone [4]_."},{"line_number":57,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_22037078","line":54,"updated":"2015-11-20 15:16:54.000000000","message":"*Just* `drops` and `alters` or is a definitive list we can go off of? If that list isn\u0027t included in this document, then I think we should at least point to is so that we have a reference to go off of.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"8dbbd2446140dae23f2398de54319d33ad39f4e7","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":53,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":54,"context_line":"alters. There is already a solution which is present in Nova [3]_."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"A patch introducing this unit test is under review in Keystone [4]_."},{"line_number":57,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_b48516d0","line":54,"in_reply_to":"ba8a016a_22037078","updated":"2015-11-23 14:26:36.000000000","message":"Also, how are we going to manage changes that effect referential integrity?\n\nLet\u0027s say that you have two tables that store similar information in an upgrade scenario (e.g. you want to make a schema change to the project table). What if the old project table has foreign key constraints to other tables in the keystone schema? How do we ensure other tables that are referencing the old project table get updated properly in the migration to the new project table?\n\nI\u0027m not sure if this is an implementation detail but I think it might be worth discussing.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":53,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":54,"context_line":"alters. There is already a solution which is present in Nova [3]_."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"A patch introducing this unit test is under review in Keystone [4]_."},{"line_number":57,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_437c36b5","line":54,"in_reply_to":"ba8a016a_22037078","updated":"2015-11-23 15:19:37.000000000","message":"Yes, these are just drops and alters.\n\nI heard there may be some databases, like old MySQL versions which have problems keeping up while applying other schema changes (f.ex adding indices), but I think filtering drops and alters (which is what Nova does) is good enough. It closes the gap on all obvious incompatibilities.\n\nAs per your second comment, I added a paragraph explaining a simpler case in which data migration is needed. Hope that answers some of your concerns. There are also examples of migrations in Nova, which we can use when writing developer documentation.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"53fa0eb57622eda675a2c56ddd8f1b6dd2591b0a","unresolved":false,"context_lines":[{"line_number":87,"context_line":"executed after all the services are running with the new version."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"A similar approach for two-phase migrations, taken by Nova [6]_ was determined"},{"line_number":90,"context_line":"experimental and will be removed in Mitaka [7]_."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"We have yet to decide whether we (or OpenStack in general) want to follow with"},{"line_number":93,"context_line":"our implementation of two-phase migrations. In the meantime, to address the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_82a6dc51","line":90,"updated":"2015-11-20 15:16:54.000000000","message":"Out of curiosity, is nova replacing this two-phase migration with anything else?","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":87,"context_line":"executed after all the services are running with the new version."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"A similar approach for two-phase migrations, taken by Nova [6]_ was determined"},{"line_number":90,"context_line":"experimental and will be removed in Mitaka [7]_."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"We have yet to decide whether we (or OpenStack in general) want to follow with"},{"line_number":93,"context_line":"our implementation of two-phase migrations. In the meantime, to address the"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_2eb9d915","line":90,"in_reply_to":"ba8a016a_82a6dc51","updated":"2015-11-23 15:19:37.000000000","message":"No. For now they are happy with their current process, although it takes 3 releases to remove things (deprecate, remove code, drop columns/tables).","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":13063,"name":"WEI CHEN","email":"dave.jungler@gmail.com","username":"wei.d.chen"},"change_message_id":"edc9f4ea9d73d724be2d8b5743833e9840ebe768","unresolved":false,"context_lines":[{"line_number":117,"context_line":"down."},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"In case of more complicated two-phase migrations the time period of this"},{"line_number":120,"context_line":"performance impact could be limited."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"Other Deployer Impact"},{"line_number":123,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_ed4a1755","line":120,"updated":"2015-11-22 06:57:13.000000000","message":"Not sure if we are talking about the performance impact of the *unit testcase* which is all you need for this spec.\n\nLooks like you are talking schema migrations, I am okay with as is.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":117,"context_line":"down."},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"In case of more complicated two-phase migrations the time period of this"},{"line_number":120,"context_line":"performance impact could be limited."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"Other Deployer Impact"},{"line_number":123,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_e9074bc7","line":120,"in_reply_to":"ba8a016a_ed4a1755","updated":"2015-11-23 15:19:37.000000000","message":"Sorry, I meant to mention the impact of data migrations here.\n\nI added a paragraph about it in the \"Proposed change\" section.\n\nYour comment made me think. The two-phase migrations doesn\u0027t shorten the time period in which data migrations are performed, because there still would be an intermediate release, which supports both versions and f.ex. writes data to two columns in two formats, so the old version can read data produced by the new version. So I\u0027m removing this sentence.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":13063,"name":"WEI CHEN","email":"dave.jungler@gmail.com","username":"wei.d.chen"},"change_message_id":"edc9f4ea9d73d724be2d8b5743833e9840ebe768","unresolved":false,"context_lines":[{"line_number":123,"context_line":"---------------------"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Ability to perform rolling upgrades will have a large and positive impact on"},{"line_number":126,"context_line":"deployment."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Developer Impact"},{"line_number":129,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_ad73afb3","line":126,"updated":"2015-11-22 06:57:13.000000000","message":"To be more specific, talking anything about the impact from \"online schema migration\".","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":123,"context_line":"---------------------"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Ability to perform rolling upgrades will have a large and positive impact on"},{"line_number":126,"context_line":"deployment."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Developer Impact"},{"line_number":129,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_497777d4","line":126,"in_reply_to":"ba8a016a_ad73afb3","updated":"2015-11-23 15:19:37.000000000","message":"Done","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"53fa0eb57622eda675a2c56ddd8f1b6dd2591b0a","unresolved":false,"context_lines":[{"line_number":157,"context_line":"Documentation Impact"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"None"},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"References"},{"line_number":163,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_45031e0e","line":160,"updated":"2015-11-20 15:16:54.000000000","message":"I would expect some level of developer documentation added, at least. \n\nAre we going to go back and fix all migration to be \"online\" enabled? If not, then new developers are going to have multiple examples in the current code-base of how to do a migration, and no real explanation of why except a failing unit test. Without documentation, this just raises the barrier to entry for new developers to the project.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":13063,"name":"WEI CHEN","email":"dave.jungler@gmail.com","username":"wei.d.chen"},"change_message_id":"edc9f4ea9d73d724be2d8b5743833e9840ebe768","unresolved":false,"context_lines":[{"line_number":157,"context_line":"Documentation Impact"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"None"},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"References"},{"line_number":163,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_ad816f4f","line":160,"in_reply_to":"ba8a016a_45031e0e","updated":"2015-11-22 06:57:13.000000000","message":"++\n\nWe should document this somewhere.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"775ad10d578d275332941564c9dc01c1be56dd76","unresolved":false,"context_lines":[{"line_number":157,"context_line":"Documentation Impact"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":159,"context_line":""},{"line_number":160,"context_line":"None"},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"References"},{"line_number":163,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba8a016a_0e03b5b9","line":160,"in_reply_to":"ba8a016a_ad816f4f","updated":"2015-11-23 15:19:37.000000000","message":"Ok, I\u0027m adding information here and to the work items section.","commit_id":"7c5d86681d2c2fe4c353b7afbeef3f630ac4eec7"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Future incompatible changes in sqlalchemy migrations, like removing,"},{"line_number":15,"context_line":"renaming columns and tables can break rolling upgrades (upgrades when"},{"line_number":16,"context_line":"multiple Keystone instances are run at different versions)."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Support for rolling upgrades is a key theme for the Mitaka cycle [1]_."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_474517b1","line":16,"updated":"2015-11-25 18:32:19.000000000","message":"(nit) keystone","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"b00fef9daa4322d4e7334ba1c7a6c3c85a68c109","unresolved":false,"context_lines":[{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Future incompatible changes in sqlalchemy migrations, like removing,"},{"line_number":15,"context_line":"renaming columns and tables can break rolling upgrades (upgrades when"},{"line_number":16,"context_line":"multiple Keystone instances are run at different versions)."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Support for rolling upgrades is a key theme for the Mitaka cycle [1]_."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_931bee72","line":16,"in_reply_to":"9a8ffd7b_474517b1","updated":"2015-11-27 09:36:21.000000000","message":"Done","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":33,"context_line":"Even though one of the operators stated that he upgraded keystone without"},{"line_number":34,"context_line":"downtime, this is impossible across major releases due to schema changes which"},{"line_number":35,"context_line":"cause incompatibilities between versions. Incompatibilities are introduced in"},{"line_number":36,"context_line":"part due to lack of commitment of reviewers to the \"cause\" of rolling upgrades,"},{"line_number":37,"context_line":"but even more so, due to lack of tests which could catch those issues."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This spec could be a statement of commitment by the reviewers of Keystone to"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_c7a2674b","line":36,"updated":"2015-11-25 18:32:19.000000000","message":"I woudln\u0027t say this is a lack of commitment of reviewers, maybe this isn\u0027t just a factor we (reviewers) were taking into consideration so far.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"d543fb68637821cec96314f340db31000dbac01e","unresolved":false,"context_lines":[{"line_number":33,"context_line":"Even though one of the operators stated that he upgraded keystone without"},{"line_number":34,"context_line":"downtime, this is impossible across major releases due to schema changes which"},{"line_number":35,"context_line":"cause incompatibilities between versions. Incompatibilities are introduced in"},{"line_number":36,"context_line":"part due to lack of commitment of reviewers to the \"cause\" of rolling upgrades,"},{"line_number":37,"context_line":"but even more so, due to lack of tests which could catch those issues."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This spec could be a statement of commitment by the reviewers of Keystone to"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_88a20bfc","line":36,"in_reply_to":"9a8ffd7b_c7a2674b","updated":"2015-11-26 11:20:31.000000000","message":"You\u0027re right.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":36,"context_line":"part due to lack of commitment of reviewers to the \"cause\" of rolling upgrades,"},{"line_number":37,"context_line":"but even more so, due to lack of tests which could catch those issues."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This spec could be a statement of commitment by the reviewers of Keystone to"},{"line_number":40,"context_line":"support zero-downtime upgrades."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Keystone is less susceptible to upgrade issues than some of the other projects"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_e7bc4b6e","line":39,"updated":"2015-11-25 18:32:19.000000000","message":"(nit) keystone","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"b00fef9daa4322d4e7334ba1c7a6c3c85a68c109","unresolved":false,"context_lines":[{"line_number":36,"context_line":"part due to lack of commitment of reviewers to the \"cause\" of rolling upgrades,"},{"line_number":37,"context_line":"but even more so, due to lack of tests which could catch those issues."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"This spec could be a statement of commitment by the reviewers of Keystone to"},{"line_number":40,"context_line":"support zero-downtime upgrades."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"Keystone is less susceptible to upgrade issues than some of the other projects"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_b3187276","line":39,"in_reply_to":"9a8ffd7b_e7bc4b6e","updated":"2015-11-27 09:36:21.000000000","message":"Done","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":54,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":55,"context_line":"alters. There is already a solution which is present in Nova [3]_."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"A patch introducing this unit test is under review in Keystone [4]_."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"The test is surrounded by comments with direct wording that encourages"},{"line_number":60,"context_line":"reviewers to take into consideration the impact on upgradeability."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_27301301","line":57,"updated":"2015-11-25 18:32:19.000000000","message":"(nit) keystone","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"b00fef9daa4322d4e7334ba1c7a6c3c85a68c109","unresolved":false,"context_lines":[{"line_number":54,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":55,"context_line":"alters. There is already a solution which is present in Nova [3]_."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"A patch introducing this unit test is under review in Keystone [4]_."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"The test is surrounded by comments with direct wording that encourages"},{"line_number":60,"context_line":"reviewers to take into consideration the impact on upgradeability."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_53282686","line":57,"in_reply_to":"9a8ffd7b_27301301","updated":"2015-11-27 09:36:21.000000000","message":"Done","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":60,"context_line":"reviewers to take into consideration the impact on upgradeability."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"The unit test blocks all alters and drops, but also contains a list of"},{"line_number":63,"context_line":"migrations where we allow dropping things."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The rules for adding exceptions are very specific:"},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_c32a5a7e","line":63,"updated":"2015-11-25 18:32:19.000000000","message":"s/allow dropping/allow altering and dropping/","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"b00fef9daa4322d4e7334ba1c7a6c3c85a68c109","unresolved":false,"context_lines":[{"line_number":60,"context_line":"reviewers to take into consideration the impact on upgradeability."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"The unit test blocks all alters and drops, but also contains a list of"},{"line_number":63,"context_line":"migrations where we allow dropping things."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The rules for adding exceptions are very specific:"},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_131e1e5f","line":63,"in_reply_to":"9a8ffd7b_c32a5a7e","updated":"2015-11-27 09:36:21.000000000","message":"Done","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":69,"context_line":"2) Migrations removing structures not used in the previous version are allowed"},{"line_number":70,"context_line":"   (we keep compatibility between releases), ex.:"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   a) feature is deprecated in Kilo,"},{"line_number":73,"context_line":"   b) code supporting the feature is removed in Liberty,"},{"line_number":74,"context_line":"   c) table can be dropped only after liberty i.e. in Mitaka."},{"line_number":75,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_43ac2aed","line":72,"updated":"2015-11-25 18:32:19.000000000","message":"I\u0027d say \"feature is deprecated according to the deprecation policies\".\n\nSaying Kilo makes this very specific to this release and to the change; as perhaps there may be deprecations that can take longer than 2 cycles ?","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"d543fb68637821cec96314f340db31000dbac01e","unresolved":false,"context_lines":[{"line_number":69,"context_line":"2) Migrations removing structures not used in the previous version are allowed"},{"line_number":70,"context_line":"   (we keep compatibility between releases), ex.:"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   a) feature is deprecated in Kilo,"},{"line_number":73,"context_line":"   b) code supporting the feature is removed in Liberty,"},{"line_number":74,"context_line":"   c) table can be dropped only after liberty i.e. in Mitaka."},{"line_number":75,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_39a32f02","line":72,"in_reply_to":"9a8ffd7b_43ac2aed","updated":"2015-11-26 11:20:31.000000000","message":"Maybe instead of using names, numbers 1-3 can be used, because it can get confusing when we are talking about changes taking 3 or more releases.\n\nI will make a change in the next patch, taking your comments into account.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":70,"context_line":"   (we keep compatibility between releases), ex.:"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   a) feature is deprecated in Kilo,"},{"line_number":73,"context_line":"   b) code supporting the feature is removed in Liberty,"},{"line_number":74,"context_line":"   c) table can be dropped only after liberty i.e. in Mitaka."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3) Any other changes which don\u0027t pass this test are disallowed."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_8380d247","line":73,"updated":"2015-11-25 18:32:19.000000000","message":"Similarly here, perhaps \"is removed in the previous release\"","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   a) feature is deprecated in Kilo,"},{"line_number":73,"context_line":"   b) code supporting the feature is removed in Liberty,"},{"line_number":74,"context_line":"   c) table can be dropped only after liberty i.e. in Mitaka."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3) Any other changes which don\u0027t pass this test are disallowed."},{"line_number":77,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_63592efe","line":74,"updated":"2015-11-25 18:32:19.000000000","message":"Same here, something like: \"table can be dropped a release after the code has been removed.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":72,"context_line":"   a) feature is deprecated in Kilo,"},{"line_number":73,"context_line":"   b) code supporting the feature is removed in Liberty,"},{"line_number":74,"context_line":"   c) table can be dropped only after liberty i.e. in Mitaka."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3) Any other changes which don\u0027t pass this test are disallowed."},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"We start requiring things to be additive in Mitaka, so we ignore all migrations"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_8313f20c","line":75,"updated":"2015-11-25 18:32:19.000000000","message":"Is this an exhaustive list already ? What about altering the tables ? e.g the type of a column, etc ?","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"d543fb68637821cec96314f340db31000dbac01e","unresolved":false,"context_lines":[{"line_number":72,"context_line":"   a) feature is deprecated in Kilo,"},{"line_number":73,"context_line":"   b) code supporting the feature is removed in Liberty,"},{"line_number":74,"context_line":"   c) table can be dropped only after liberty i.e. in Mitaka."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3) Any other changes which don\u0027t pass this test are disallowed."},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"We start requiring things to be additive in Mitaka, so we ignore all migrations"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_6e656f1c","line":75,"in_reply_to":"9a8ffd7b_8313f20c","updated":"2015-11-26 11:20:31.000000000","message":"The unit test will only check for alters and drops. The idea is to keep (and update) structures which are used by the previous version.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"effe73b2dafa226a981e5aee68ef3f2a3631c89e","unresolved":false,"context_lines":[{"line_number":88,"context_line":""},{"line_number":89,"context_line":"a) A new column, with a new format is created in a schema upgrade script."},{"line_number":90,"context_line":"   Because the old version doesn\u0027t know about it, the old column is retained"},{"line_number":91,"context_line":"   and data is written to both columns. The old column is also checked in cases"},{"line_number":92,"context_line":"   when the new column doesn\u0027t contain current data."},{"line_number":93,"context_line":"b) Next version is released, it contains an upgrade script which migrates the"},{"line_number":94,"context_line":"   rest of data from the old and into the new column. It migrates any data"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_81001cac","line":91,"updated":"2015-11-24 15:23:06.000000000","message":"Do we have to write to both columns? Can logic exist that reads from the old column but also reads and writes to the new column? By the time we need to drop the old column, it should only consist of stale and old data, right?","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":88,"context_line":""},{"line_number":89,"context_line":"a) A new column, with a new format is created in a schema upgrade script."},{"line_number":90,"context_line":"   Because the old version doesn\u0027t know about it, the old column is retained"},{"line_number":91,"context_line":"   and data is written to both columns. The old column is also checked in cases"},{"line_number":92,"context_line":"   when the new column doesn\u0027t contain current data."},{"line_number":93,"context_line":"b) Next version is released, it contains an upgrade script which migrates the"},{"line_number":94,"context_line":"   rest of data from the old and into the new column. It migrates any data"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_c3bedad3","line":91,"in_reply_to":"9a8ffd7b_301df473","updated":"2015-11-25 18:32:19.000000000","message":"I think this is really going to complicate things, not sure it\u0027s worth it the effort. Still more as you said in the introduction that keystone is one of the closest projects near to zero downtime.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"8bd4cec267812b6a3e549dbbce1486519bfe84e7","unresolved":false,"context_lines":[{"line_number":88,"context_line":""},{"line_number":89,"context_line":"a) A new column, with a new format is created in a schema upgrade script."},{"line_number":90,"context_line":"   Because the old version doesn\u0027t know about it, the old column is retained"},{"line_number":91,"context_line":"   and data is written to both columns. The old column is also checked in cases"},{"line_number":92,"context_line":"   when the new column doesn\u0027t contain current data."},{"line_number":93,"context_line":"b) Next version is released, it contains an upgrade script which migrates the"},{"line_number":94,"context_line":"   rest of data from the old and into the new column. It migrates any data"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_5c96f33f","line":91,"in_reply_to":"9a8ffd7b_6e97afec","updated":"2015-11-27 10:53:00.000000000","message":"I just noticed this can get simpler. At this point, data can be read only from the old place... It actually should be read only from the old column, if the row is updated by the old code which can be still running. I think this is what Lance may have meant. I will change this sentence in a following patch.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"33a5c3f0bd210a57944bee87b74b0a0fb9d4531b","unresolved":false,"context_lines":[{"line_number":88,"context_line":""},{"line_number":89,"context_line":"a) A new column, with a new format is created in a schema upgrade script."},{"line_number":90,"context_line":"   Because the old version doesn\u0027t know about it, the old column is retained"},{"line_number":91,"context_line":"   and data is written to both columns. The old column is also checked in cases"},{"line_number":92,"context_line":"   when the new column doesn\u0027t contain current data."},{"line_number":93,"context_line":"b) Next version is released, it contains an upgrade script which migrates the"},{"line_number":94,"context_line":"   rest of data from the old and into the new column. It migrates any data"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_301df473","line":91,"in_reply_to":"9a8ffd7b_81001cac","updated":"2015-11-24 17:19:47.000000000","message":"The old code should understand data written by the new version, we also don\u0027t know how long the old version will be running, so I would be careful about assuming this. But of course it would depend on the case and what inconsistencies we are good with. I\u0027m trying to portray a generic and safe way to do it here.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"d543fb68637821cec96314f340db31000dbac01e","unresolved":false,"context_lines":[{"line_number":88,"context_line":""},{"line_number":89,"context_line":"a) A new column, with a new format is created in a schema upgrade script."},{"line_number":90,"context_line":"   Because the old version doesn\u0027t know about it, the old column is retained"},{"line_number":91,"context_line":"   and data is written to both columns. The old column is also checked in cases"},{"line_number":92,"context_line":"   when the new column doesn\u0027t contain current data."},{"line_number":93,"context_line":"b) Next version is released, it contains an upgrade script which migrates the"},{"line_number":94,"context_line":"   rest of data from the old and into the new column. It migrates any data"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_6e97afec","line":91,"in_reply_to":"9a8ffd7b_c3bedad3","updated":"2015-11-26 11:20:31.000000000","message":"Well, in this example we would have to check both places... on the other hand, the patches introducing the schema changes will get simpler to review, because they will be divided into smaller, incremental changes which don\u0027t cause incompatibilities.\n\nAlso, the amount of patches that make incompatible changes is very low. For instance, in Heat and Cinder, there were no incompatible changes in Liberty, even though there was no unit test checking this (Heat added this test in Mitaka and I believe guys from Cinder are also getting to it).","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"effe73b2dafa226a981e5aee68ef3f2a3631c89e","unresolved":false,"context_lines":[{"line_number":89,"context_line":"a) A new column, with a new format is created in a schema upgrade script."},{"line_number":90,"context_line":"   Because the old version doesn\u0027t know about it, the old column is retained"},{"line_number":91,"context_line":"   and data is written to both columns. The old column is also checked in cases"},{"line_number":92,"context_line":"   when the new column doesn\u0027t contain current data."},{"line_number":93,"context_line":"b) Next version is released, it contains an upgrade script which migrates the"},{"line_number":94,"context_line":"   rest of data from the old and into the new column. It migrates any data"},{"line_number":95,"context_line":"   which wasn\u0027t migrated in the process of normally running the service. The"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_a1968092","line":92,"updated":"2015-11-24 15:23:06.000000000","message":"Does adding a new column to a table use the alter statement? If so, doesn\u0027t the alter statement introduce a table lock? Are we worried about table locks and the availability of the table?\n\n\nhttp://docs.sqlalchemy.org/en/rel_1_0/core/metadata.html#sqlalchemy.schema.Table.append_column","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"33a5c3f0bd210a57944bee87b74b0a0fb9d4531b","unresolved":false,"context_lines":[{"line_number":89,"context_line":"a) A new column, with a new format is created in a schema upgrade script."},{"line_number":90,"context_line":"   Because the old version doesn\u0027t know about it, the old column is retained"},{"line_number":91,"context_line":"   and data is written to both columns. The old column is also checked in cases"},{"line_number":92,"context_line":"   when the new column doesn\u0027t contain current data."},{"line_number":93,"context_line":"b) Next version is released, it contains an upgrade script which migrates the"},{"line_number":94,"context_line":"   rest of data from the old and into the new column. It migrates any data"},{"line_number":95,"context_line":"   which wasn\u0027t migrated in the process of normally running the service. The"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_ed48f22f","line":92,"in_reply_to":"9a8ffd7b_a1968092","updated":"2015-11-24 17:19:47.000000000","message":"I know that in case of PostgreSQL the lock for adding columns is brief, regardless of the table size, because it doesn\u0027t require a full scan of the table. I\u0027m not sure how MySQL handles these cases...\n\nIn any case, if the schema is compatible across two versions, the operator may cancel the schema alter operation at any moment and try another approach.\n\nHere is a good explanation, which also suggests replacing an alter column statement with a set of add column, migrate data, drop old column - like we would do to support rolling upgrades, but for a different end result - avoiding locks:\n\nhttps://dba.stackexchange.com/questions/27153/alter-table-on-live-production-databases#27156","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"effe73b2dafa226a981e5aee68ef3f2a3631c89e","unresolved":false,"context_lines":[{"line_number":95,"context_line":"   which wasn\u0027t migrated in the process of normally running the service. The"},{"line_number":96,"context_line":"   old column is also removed in this release from SQLAlchemy models and is no"},{"line_number":97,"context_line":"   longer used."},{"line_number":98,"context_line":"c) In the third release, the upgrade script removes the old column."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_01e2ec03","line":98,"updated":"2015-11-24 15:23:06.000000000","message":"This assumes the column isn\u0027t going to be renamed to the original column name. If we want to rename it back, that would require another release, I think?","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"33a5c3f0bd210a57944bee87b74b0a0fb9d4531b","unresolved":false,"context_lines":[{"line_number":95,"context_line":"   which wasn\u0027t migrated in the process of normally running the service. The"},{"line_number":96,"context_line":"   old column is also removed in this release from SQLAlchemy models and is no"},{"line_number":97,"context_line":"   longer used."},{"line_number":98,"context_line":"c) In the third release, the upgrade script removes the old column."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_b06a4419","line":98,"in_reply_to":"9a8ffd7b_01e2ec03","updated":"2015-11-24 17:19:47.000000000","message":"Yes. I don\u0027t think doing this work just to change a column name would be worth it.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"accd53013605501f207a4d464fc3ac904ba98e88","unresolved":false,"context_lines":[{"line_number":113,"context_line":"We have yet to decide whether we (or OpenStack in general) want to follow with"},{"line_number":114,"context_line":"our implementation of two-phase migrations. In the meantime, to address the"},{"line_number":115,"context_line":"problem, we can ban schema changes which cause incompatibilities (see above),"},{"line_number":116,"context_line":"which is the approach that works in Nova."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Security Impact"},{"line_number":119,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_c3081a94","line":116,"updated":"2015-11-25 18:32:19.000000000","message":"If we ban and we don\u0027t have a policy on how to do it, how do we deal with migrations in the meantime, if they won\u0027t pass in the tests ?\n\nAlso, if we (as OpenStack) have to decide the implementation of this, why not decide after implmeenting in keystone? This wouldn\u0027t be too bad, as keystone is already one of the closest projects near to zero downtime.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"d543fb68637821cec96314f340db31000dbac01e","unresolved":false,"context_lines":[{"line_number":113,"context_line":"We have yet to decide whether we (or OpenStack in general) want to follow with"},{"line_number":114,"context_line":"our implementation of two-phase migrations. In the meantime, to address the"},{"line_number":115,"context_line":"problem, we can ban schema changes which cause incompatibilities (see above),"},{"line_number":116,"context_line":"which is the approach that works in Nova."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Security Impact"},{"line_number":119,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a8ffd7b_53eb7478","line":116,"in_reply_to":"9a8ffd7b_c3081a94","updated":"2015-11-26 11:20:31.000000000","message":"The migrations, in any case, have to be divided into two or more schema migration scripts. The difference is, that in two-phase migrations, they could be applied in one release (operators can run these additional actions before and after the upgrade).\n\nNeutron is already experimenting with this, but Nova chose to not go down this path (at least not yet). I think the best option for Keystone would be to follow Neutron, but only after it turns out it\u0027s a good solution. Right now they don\u0027t have it all figured out yet.\n\nI believe I outlined the policy on how to do migrations - the unit test will be just a signal for code reviewers. Drops and alters, which cause incompatibilities will not be allowed (by our policy), but drops of tables and columns, which are no longer used would be allowed and added to the exception list of the unit test.","commit_id":"9caf2e5ef9f8ba7cedcc1a18f7cbc000513f93ab"},{"author":{"_account_id":6486,"name":"Brant Knudson","email":"blk@acm.org","username":"blk-u"},"change_message_id":"10dd6870ba30c3389043b1f763c43cdf0e443d67","unresolved":false,"context_lines":[{"line_number":42,"context_line":"that could be added to keystone [3]_."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Keystone is less susceptible to upgrade issues than some of the other projects"},{"line_number":45,"context_line":"in OpenStack, because it doesn\u0027t have cross-service RPC communication. The"},{"line_number":46,"context_line":"other issue which can cause upgrade problems is cross-version API"},{"line_number":47,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":48,"context_line":"because the CLI interface is maintained as a separate project and so the"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_ed21d2b5","line":45,"updated":"2015-11-30 20:36:09.000000000","message":"keystone provides notifications over rpc for other services, so this is inaccurate.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":42,"context_line":"that could be added to keystone [3]_."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Keystone is less susceptible to upgrade issues than some of the other projects"},{"line_number":45,"context_line":"in OpenStack, because it doesn\u0027t have cross-service RPC communication. The"},{"line_number":46,"context_line":"other issue which can cause upgrade problems is cross-version API"},{"line_number":47,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":48,"context_line":"because the CLI interface is maintained as a separate project and so the"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_86b32e0a","line":45,"in_reply_to":"9a8ffd7b_ed21d2b5","updated":"2015-12-01 11:12:45.000000000","message":"fixing it to specify that keystone services don\u0027t talk with each other via RPC","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":6486,"name":"Brant Knudson","email":"blk@acm.org","username":"blk-u"},"change_message_id":"10dd6870ba30c3389043b1f763c43cdf0e443d67","unresolved":false,"context_lines":[{"line_number":46,"context_line":"other issue which can cause upgrade problems is cross-version API"},{"line_number":47,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":48,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":49,"context_line":"compatibility between those is tested and retained. API compatibility will not"},{"line_number":50,"context_line":"be discussed in this spec."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_801c8d6b","line":49,"updated":"2015-11-30 20:36:09.000000000","message":"There\u0027s no testing of the CLIs that I know of.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":46,"context_line":"other issue which can cause upgrade problems is cross-version API"},{"line_number":47,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":48,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":49,"context_line":"compatibility between those is tested and retained. API compatibility will not"},{"line_number":50,"context_line":"be discussed in this spec."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_a6867211","line":49,"in_reply_to":"9a8ffd7b_801c8d6b","updated":"2015-12-01 11:12:45.000000000","message":"I have seen some tests in keystone/tests/unit/test_v2_keystoneclient.py","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":6486,"name":"Brant Knudson","email":"blk@acm.org","username":"blk-u"},"change_message_id":"10dd6870ba30c3389043b1f763c43cdf0e443d67","unresolved":false,"context_lines":[{"line_number":47,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":48,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":49,"context_line":"compatibility between those is tested and retained. API compatibility will not"},{"line_number":50,"context_line":"be discussed in this spec."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Proposed Change"},{"line_number":53,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_402545ac","line":50,"updated":"2015-11-30 20:36:09.000000000","message":"If this spec isn\u0027t about API compatbility then why mention it? Remove this paragraph.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":1916,"name":"Guang Yee","email":"gyee@suse.com","username":"guang-yee"},"change_message_id":"90379369c0f8e5d785030c52d0a1a9440bf8198a","unresolved":false,"context_lines":[{"line_number":47,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":48,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":49,"context_line":"compatibility between those is tested and retained. API compatibility will not"},{"line_number":50,"context_line":"be discussed in this spec."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Proposed Change"},{"line_number":53,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_ed8540ad","line":50,"in_reply_to":"9a8ffd7b_402545ac","updated":"2015-12-01 06:45:23.000000000","message":"++","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":47,"context_line":"compatibility. Issues of API compatibility are already mostly addressed,"},{"line_number":48,"context_line":"because the CLI interface is maintained as a separate project and so the"},{"line_number":49,"context_line":"compatibility between those is tested and retained. API compatibility will not"},{"line_number":50,"context_line":"be discussed in this spec."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Proposed Change"},{"line_number":53,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_86980e6c","line":50,"in_reply_to":"9a8ffd7b_ed8540ad","updated":"2015-12-01 11:12:45.000000000","message":"Done","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":1916,"name":"Guang Yee","email":"gyee@suse.com","username":"guang-yee"},"change_message_id":"90379369c0f8e5d785030c52d0a1a9440bf8198a","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Proposed Change"},{"line_number":53,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"The key concept is to keep (and update) structures which are used by the"},{"line_number":56,"context_line":"previous version. They can only be removed when no longer used by both current"},{"line_number":57,"context_line":"and the previous release."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_6887fe12","line":54,"updated":"2015-12-01 06:45:23.000000000","message":"It would be nice to explain *how* rolling upgrade is achieved so reviewers that are less SQL DB savvy can comprehend the proposed changes. For example, how is rolling upgrade done in the case when we have multiple regions with DB clusters at each region?","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Proposed Change"},{"line_number":53,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"The key concept is to keep (and update) structures which are used by the"},{"line_number":56,"context_line":"previous version. They can only be removed when no longer used by both current"},{"line_number":57,"context_line":"and the previous release."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_0cc51d5a","line":54,"in_reply_to":"9a8ffd7b_6887fe12","updated":"2015-12-01 11:12:45.000000000","message":"I added a better explanation of what happens inside a cluster during a rolling upgrade from a developers perspective.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":1916,"name":"Guang Yee","email":"gyee@suse.com","username":"guang-yee"},"change_message_id":"90379369c0f8e5d785030c52d0a1a9440bf8198a","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"The key concept is to keep (and update) structures which are used by the"},{"line_number":56,"context_line":"previous version. They can only be removed when no longer used by both current"},{"line_number":57,"context_line":"and the previous release."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":60,"context_line":"schema changes which cause those incompatibilities, specifically drops and"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_ad2bb893","line":57,"updated":"2015-12-01 06:45:23.000000000","message":"We only care about +1 upgrades only? What about upgrade from (say) Juno to Liberty?","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"The key concept is to keep (and update) structures which are used by the"},{"line_number":56,"context_line":"previous version. They can only be removed when no longer used by both current"},{"line_number":57,"context_line":"and the previous release."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":60,"context_line":"schema changes which cause those incompatibilities, specifically drops and"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_46546653","line":57,"in_reply_to":"9a8ffd7b_ad2bb893","updated":"2015-12-01 11:12:45.000000000","message":"Yes, to upgrade to Liberty, the operator would have to upgrade incrementaly through Kilo.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"13f9e9b337f3a5abeeb9299ae2722d1b3efb0d0e","unresolved":false,"context_lines":[{"line_number":58,"context_line":""},{"line_number":59,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":60,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":61,"context_line":"alters. There is already a solution which is present in Nova [4]_."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"A patch introducing this unit test is under review in keystone [5]_."},{"line_number":64,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_ca5ab950","line":61,"updated":"2015-11-30 20:07:28.000000000","message":"nit: nova*","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":58,"context_line":""},{"line_number":59,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":60,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":61,"context_line":"alters. There is already a solution which is present in Nova [4]_."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"A patch introducing this unit test is under review in keystone [5]_."},{"line_number":64,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_97a916e2","line":61,"in_reply_to":"9a8ffd7b_ca5ab950","updated":"2015-12-01 11:12:45.000000000","message":"Done","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":6486,"name":"Brant Knudson","email":"blk@acm.org","username":"blk-u"},"change_message_id":"10dd6870ba30c3389043b1f763c43cdf0e443d67","unresolved":false,"context_lines":[{"line_number":60,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":61,"context_line":"alters. There is already a solution which is present in Nova [4]_."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"A patch introducing this unit test is under review in keystone [5]_."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The test is surrounded by comments with direct wording that signals code"},{"line_number":66,"context_line":"reviewers to take into consideration the impact on upgradeability."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_2024c1ad","line":63,"updated":"2015-11-30 20:36:09.000000000","message":"Remove this paragraph since the patch isn\u0027t going to be under review for as long as this doc is published. Somebody reading this can easily find the patches for the change since it\u0027ll be linked via the blueprint.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":1916,"name":"Guang Yee","email":"gyee@suse.com","username":"guang-yee"},"change_message_id":"90379369c0f8e5d785030c52d0a1a9440bf8198a","unresolved":false,"context_lines":[{"line_number":60,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":61,"context_line":"alters. There is already a solution which is present in Nova [4]_."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"A patch introducing this unit test is under review in keystone [5]_."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The test is surrounded by comments with direct wording that signals code"},{"line_number":66,"context_line":"reviewers to take into consideration the impact on upgradeability."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_cd763cd8","line":63,"in_reply_to":"9a8ffd7b_2024c1ad","updated":"2015-12-01 06:45:23.000000000","message":"++","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":60,"context_line":"schema changes which cause those incompatibilities, specifically drops and"},{"line_number":61,"context_line":"alters. There is already a solution which is present in Nova [4]_."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"A patch introducing this unit test is under review in keystone [5]_."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The test is surrounded by comments with direct wording that signals code"},{"line_number":66,"context_line":"reviewers to take into consideration the impact on upgradeability."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_b7cf7ab2","line":63,"in_reply_to":"9a8ffd7b_cd763cd8","updated":"2015-12-01 11:12:45.000000000","message":"Done","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":1916,"name":"Guang Yee","email":"gyee@suse.com","username":"guang-yee"},"change_message_id":"90379369c0f8e5d785030c52d0a1a9440bf8198a","unresolved":false,"context_lines":[{"line_number":63,"context_line":"A patch introducing this unit test is under review in keystone [5]_."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The test is surrounded by comments with direct wording that signals code"},{"line_number":66,"context_line":"reviewers to take into consideration the impact on upgradeability."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"The unit test blocks all alters and drops, but also contains a list of"},{"line_number":69,"context_line":"migrations where we allow altering dropping things."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_0810b24d","line":66,"updated":"2015-12-01 06:45:23.000000000","message":"Shoudn\u0027t we be testing upgrades in Grenade?","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":63,"context_line":"A patch introducing this unit test is under review in keystone [5]_."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The test is surrounded by comments with direct wording that signals code"},{"line_number":66,"context_line":"reviewers to take into consideration the impact on upgradeability."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"The unit test blocks all alters and drops, but also contains a list of"},{"line_number":69,"context_line":"migrations where we allow altering dropping things."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_174be6b1","line":66,"in_reply_to":"9a8ffd7b_0810b24d","updated":"2015-12-01 11:12:45.000000000","message":"Yes, I also want to work on that (in this cycle), but detecting those issues earlier goes a long way in explaining what is wrong with the approach that a developer took.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"13f9e9b337f3a5abeeb9299ae2722d1b3efb0d0e","unresolved":false,"context_lines":[{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The rules for adding exceptions are very specific:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1) Migrations which don\u0027t cause incompatibilities are allowed f.ex. dropping"},{"line_number":74,"context_line":"   an index or constraint."},{"line_number":75,"context_line":"2) Migrations removing structures not used in the previous version are allowed"},{"line_number":76,"context_line":"   (we keep compatibility between releases), ex.:"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_6a2e2da7","line":73,"updated":"2015-11-30 20:07:28.000000000","message":"nit: for example*","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The rules for adding exceptions are very specific:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"1) Migrations which don\u0027t cause incompatibilities are allowed f.ex. dropping"},{"line_number":74,"context_line":"   an index or constraint."},{"line_number":75,"context_line":"2) Migrations removing structures not used in the previous version are allowed"},{"line_number":76,"context_line":"   (we keep compatibility between releases), ex.:"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_f7a98225","line":73,"in_reply_to":"9a8ffd7b_6a2e2da7","updated":"2015-12-01 11:12:45.000000000","message":"Done","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":1916,"name":"Guang Yee","email":"gyee@suse.com","username":"guang-yee"},"change_message_id":"90379369c0f8e5d785030c52d0a1a9440bf8198a","unresolved":false,"context_lines":[{"line_number":78,"context_line":"   a) feature is deprecated according to the deprecation policies (release 1),"},{"line_number":79,"context_line":"   b) code supporting the feature is removed the following release (release 2),"},{"line_number":80,"context_line":"   c) table can be dropped a release after the code has been removed (i.e. in"},{"line_number":81,"context_line":"      release 3)."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"3) Any other changes which don\u0027t pass this test are disallowed."},{"line_number":84,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_8893424f","line":81,"updated":"2015-12-01 06:45:23.000000000","message":"Why can\u0027t b) and c) be done together? If code is removed, then what else is accessing the old tables/columns?","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":78,"context_line":"   a) feature is deprecated according to the deprecation policies (release 1),"},{"line_number":79,"context_line":"   b) code supporting the feature is removed the following release (release 2),"},{"line_number":80,"context_line":"   c) table can be dropped a release after the code has been removed (i.e. in"},{"line_number":81,"context_line":"      release 3)."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"3) Any other changes which don\u0027t pass this test are disallowed."},{"line_number":84,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_125f740f","line":81,"in_reply_to":"9a8ffd7b_8893424f","updated":"2015-12-01 11:12:45.000000000","message":"Currently, sql migrations are executed before running the new code. So if we want to drop columns, we have to do it in the next release. I\u0027m adding this observation to the first paragraph to clarify things.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":1916,"name":"Guang Yee","email":"gyee@suse.com","username":"guang-yee"},"change_message_id":"90379369c0f8e5d785030c52d0a1a9440bf8198a","unresolved":false,"context_lines":[{"line_number":90,"context_line":"up to the developer to decide how this is achieved. The data migration may be"},{"line_number":91,"context_line":"started automatically, when the new version is introduced, while an old version"},{"line_number":92,"context_line":"is still running, and finished up with a migration script in the next (third)"},{"line_number":93,"context_line":"release. It could also be scripted, with information on how to do it in the"},{"line_number":94,"context_line":"release notes. A suggested migration would happen in three phases:"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"a) A new column, with a new format is created in a schema upgrade script."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_08ce725a","line":93,"updated":"2015-12-01 06:45:23.000000000","message":"wow, can this be really be done, especially if we have write operations during that time? And how does the code check for conflicts in both columns?","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":90,"context_line":"up to the developer to decide how this is achieved. The data migration may be"},{"line_number":91,"context_line":"started automatically, when the new version is introduced, while an old version"},{"line_number":92,"context_line":"is still running, and finished up with a migration script in the next (third)"},{"line_number":93,"context_line":"release. It could also be scripted, with information on how to do it in the"},{"line_number":94,"context_line":"release notes. A suggested migration would happen in three phases:"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"a) A new column, with a new format is created in a schema upgrade script."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_12c1b42e","line":93,"in_reply_to":"9a8ffd7b_08ce725a","updated":"2015-12-01 11:12:45.000000000","message":"In case there can be write conflicts, the simplest approach would be to only read from the old column, while the old version is still running (step \"a)\" below). In other cases, this is entirely up to the developer implementing the migration.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"13f9e9b337f3a5abeeb9299ae2722d1b3efb0d0e","unresolved":false,"context_lines":[{"line_number":102,"context_line":"   from the old and into the new column. It migrates any data which wasn\u0027t"},{"line_number":103,"context_line":"   migrated in the process of normally running the service. If the data was"},{"line_number":104,"context_line":"   read only from the old column, this release should read data from the new"},{"line_number":105,"context_line":"   column, and, as before, write to both (to be backwards compatible)."},{"line_number":106,"context_line":"c) In the following release (in case when the previous release reads data from"},{"line_number":107,"context_line":"   the new column), the old column can be removed from SQLAlchemy models and"},{"line_number":108,"context_line":"   is no longer used."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_ea221801","line":105,"updated":"2015-11-30 20:07:28.000000000","message":"Can\u0027t the first two steps be done in one? \n\nA new column is created and with that column comes new logic that knows how to deal with the data in that column. The new logic should also know how to understand the old column. The logic should know what to do if only one is populated and so forth. The new logic should write to both columns. A migration script can be provided to migrate the data from one type to another and store the new type in the new column. Is there any reason why this can\u0027t be done in a single release?\n\nIn the next release, the logic should be changed to stop writing to the old column. The column can be dropped.\n\n\nExample:\n\n- (mitaka) we upgrade all keystone api nodes from liberty to mitaka\n- (mitaka) each api node understands the old schema\n- (mitaka) migrate the database, which results in the creation of the new column and the migration of data from the old column to the new column\n- (n-release) we upgrade all keystone api nodes from mitaka to n\n- (n-release) drop the old column that is no longer used\n\nI think we should also clarify that in the process of upgrading, the all application nodes must be updated before updating any of the database nodes. This ensures that each application node knows how to handle both schemas.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":102,"context_line":"   from the old and into the new column. It migrates any data which wasn\u0027t"},{"line_number":103,"context_line":"   migrated in the process of normally running the service. If the data was"},{"line_number":104,"context_line":"   read only from the old column, this release should read data from the new"},{"line_number":105,"context_line":"   column, and, as before, write to both (to be backwards compatible)."},{"line_number":106,"context_line":"c) In the following release (in case when the previous release reads data from"},{"line_number":107,"context_line":"   the new column), the old column can be removed from SQLAlchemy models and"},{"line_number":108,"context_line":"   is no longer used."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_e6e6b543","line":105,"in_reply_to":"9a8ffd7b_ea221801","updated":"2015-12-01 11:12:45.000000000","message":"I understand your point. I still think there may be cases in which we can\u0027t distinguish which column is most up to date. I\u0027m adding a paragraph below to make it clear, that the first two steps may be squashed. The dropping currently requires a new release, because schema migrations are run before starting the new code.\n\nThe schema is run in the first step of upgrading, that\u0027s why we are banning drops and alters, so that the SQLAlchemy models from the previous version can still be used.\n\nI enhanced the first paragraph in this section to clarify this.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"13f9e9b337f3a5abeeb9299ae2722d1b3efb0d0e","unresolved":false,"context_lines":[{"line_number":105,"context_line":"   column, and, as before, write to both (to be backwards compatible)."},{"line_number":106,"context_line":"c) In the following release (in case when the previous release reads data from"},{"line_number":107,"context_line":"   the new column), the old column can be removed from SQLAlchemy models and"},{"line_number":108,"context_line":"   is no longer used."},{"line_number":109,"context_line":"d) After it is no longer used, a migrate script can remove the old column."},{"line_number":110,"context_line":""},{"line_number":111,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_ade3ca32","line":108,"updated":"2015-11-30 20:07:28.000000000","message":"Also, is there a way to optionally load a sql schema in sqlalchemy? If a keystone api nodes are updated before the schema, they should *attempt* to register the new schema, and fall back to the old one if possible. Is that possible in sqlalchemy?","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":105,"context_line":"   column, and, as before, write to both (to be backwards compatible)."},{"line_number":106,"context_line":"c) In the following release (in case when the previous release reads data from"},{"line_number":107,"context_line":"   the new column), the old column can be removed from SQLAlchemy models and"},{"line_number":108,"context_line":"   is no longer used."},{"line_number":109,"context_line":"d) After it is no longer used, a migrate script can remove the old column."},{"line_number":110,"context_line":""},{"line_number":111,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_0603b910","line":108,"in_reply_to":"9a8ffd7b_ade3ca32","updated":"2015-12-01 11:12:45.000000000","message":"I don\u0027t think this is possible right now, although someone may be working on it. I talked with the Neutron guys involved with online schema migration, but I think they still haven\u0027t figured out how to do all of this in one release, because of the need to release a new set of SQLAlchemy models. The Neutron spec which was implemented only briefly mentioned the problem:\n\nhttps://specs.openstack.org/openstack/neutron-specs/specs/liberty/online-schema-migrations.html\n\n(the second bullet point)\n\nFYI, Mike Bayer is the author of SQLAlchemy.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":6486,"name":"Brant Knudson","email":"blk@acm.org","username":"blk-u"},"change_message_id":"10dd6870ba30c3389043b1f763c43cdf0e443d67","unresolved":false,"context_lines":[{"line_number":129,"context_line":"Security Impact"},{"line_number":130,"context_line":"---------------"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"None"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"Notifications Impact"},{"line_number":135,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_60864967","line":132,"updated":"2015-11-30 20:36:09.000000000","message":"The security impact is that the code is more complicated. We\u0027ll have to ensure that updates are correctly made to both columns when we have to maintain compatibility.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":129,"context_line":"Security Impact"},{"line_number":130,"context_line":"---------------"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"None"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"Notifications Impact"},{"line_number":135,"context_line":"--------------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_69a4d607","line":132,"in_reply_to":"9a8ffd7b_60864967","updated":"2015-12-01 11:12:45.000000000","message":"I agree that this may be the case in some instances, I\u0027m adding this warning.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":6486,"name":"Brant Knudson","email":"blk@acm.org","username":"blk-u"},"change_message_id":"10dd6870ba30c3389043b1f763c43cdf0e443d67","unresolved":false,"context_lines":[{"line_number":146,"context_line":""},{"line_number":147,"context_line":"The new way of doing schema changes entails that some data migrations would"},{"line_number":148,"context_line":"have to be done online, which could impact performance. On the other hand the"},{"line_number":149,"context_line":"performance is impacted infinitely when the service is down."},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"Other Deployer Impact"},{"line_number":152,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_c0dcd576","line":149,"updated":"2015-11-30 20:36:09.000000000","message":"Another performance impact is when we have to maintain the data in 2 columns.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"102ef6d622158a3e097c2d8c18bbfd8c5d343f10","unresolved":false,"context_lines":[{"line_number":146,"context_line":""},{"line_number":147,"context_line":"The new way of doing schema changes entails that some data migrations would"},{"line_number":148,"context_line":"have to be done online, which could impact performance. On the other hand the"},{"line_number":149,"context_line":"performance is impacted infinitely when the service is down."},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"Other Deployer Impact"},{"line_number":152,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a8ffd7b_1c68e6b9","line":149,"in_reply_to":"9a8ffd7b_c0dcd576","updated":"2015-12-01 11:12:45.000000000","message":"This is actually what I meant by doing online migrations. I added it as an example.","commit_id":"e54192edaeee36d7f99c53ba4f7e9ec9c142b874"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"da4440f80b56c23e87d53f21f544664d6184e0f0","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Currently, we don\u0027t limit developer\u0027s choice on what schema updates are"},{"line_number":25,"context_line":"permissible. However, certain operations, like table/column drops, name changes"},{"line_number":26,"context_line":"can make the database inaccessible for a period of time. This period may be"},{"line_number":27,"context_line":"very long in cases when the table contains thousands or millions of entries,"},{"line_number":28,"context_line":"making the upgrade time consuming, which can become not acceptable, even for"},{"line_number":29,"context_line":"offline upgrades."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9a8ffd7b_dcf25106","line":27,"updated":"2015-12-01 14:28:08.000000000","message":"So I thought the real goal of this was for rolling upgrades, not to simply cut down migration time of any one node.  While I agree that the later is always good....it seems odd that we lead on this in the problem description.  Surely the key is....allow multiple keystone versions to be run against a newly upgraded database....","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":9500,"name":"Matt Fischer","email":"matt@mattfischer.com","username":"mfisch"},"change_message_id":"aab08ee2162d6a49a5c6eba171aeeb4cd65a0333","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Currently, we don\u0027t limit developer\u0027s choice on what schema updates are"},{"line_number":25,"context_line":"permissible. However, certain operations, like table/column drops, name changes"},{"line_number":26,"context_line":"can make the database inaccessible for a period of time. This period may be"},{"line_number":27,"context_line":"very long in cases when the table contains thousands or millions of entries,"},{"line_number":28,"context_line":"making the upgrade time consuming, which can become not acceptable, even for"},{"line_number":29,"context_line":"offline upgrades."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_2274c89a","line":27,"in_reply_to":"9a8ffd7b_dcf25106","updated":"2015-12-03 19:37:17.000000000","message":"perhaps the goal is not only can you run multiple versions but you can also keep running while the migrations occur and thats what the author is getting at? Although I agree that this is not the main point.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"ea95038f29176cf927b98d4561240ab994648870","unresolved":false,"context_lines":[{"line_number":33,"context_line":"Even though one of the operators stated that he upgraded keystone without"},{"line_number":34,"context_line":"downtime, this is impossible across major releases due to schema changes which"},{"line_number":35,"context_line":"cause incompatibilities between versions. Incompatibilities are introduced in"},{"line_number":36,"context_line":"part due to lack of consideration by code reviewers of the need to perform"},{"line_number":37,"context_line":"rolling upgrades, but even more so, due to lack of tests which could catch"},{"line_number":38,"context_line":"those issues."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_4f0d02e7","line":36,"updated":"2015-12-07 15:50:23.000000000","message":"I\u0027m not sure this is fair because it\u0027s not something we look for because it\u0027s not something we have promised to support yet.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bbc202182347fc7535ff257df6b9ea4244563d9f","unresolved":false,"context_lines":[{"line_number":33,"context_line":"Even though one of the operators stated that he upgraded keystone without"},{"line_number":34,"context_line":"downtime, this is impossible across major releases due to schema changes which"},{"line_number":35,"context_line":"cause incompatibilities between versions. Incompatibilities are introduced in"},{"line_number":36,"context_line":"part due to lack of consideration by code reviewers of the need to perform"},{"line_number":37,"context_line":"rolling upgrades, but even more so, due to lack of tests which could catch"},{"line_number":38,"context_line":"those issues."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_3516c44c","line":36,"in_reply_to":"7a740942_0fc05729","updated":"2015-12-07 16:55:18.000000000","message":"Since this was never a requirement then I\u0027d say just remove it. You can say about any feature X that the reviews have not been enforcing the feature X rules.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"9c6c431eebbddca1526cff4caa6224acabb875c8","unresolved":false,"context_lines":[{"line_number":33,"context_line":"Even though one of the operators stated that he upgraded keystone without"},{"line_number":34,"context_line":"downtime, this is impossible across major releases due to schema changes which"},{"line_number":35,"context_line":"cause incompatibilities between versions. Incompatibilities are introduced in"},{"line_number":36,"context_line":"part due to lack of consideration by code reviewers of the need to perform"},{"line_number":37,"context_line":"rolling upgrades, but even more so, due to lack of tests which could catch"},{"line_number":38,"context_line":"those issues."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_0fc05729","line":36,"in_reply_to":"7a740942_4f0d02e7","updated":"2015-12-07 16:29:46.000000000","message":"Do you have a suggestion on how to word it? If not, I will remove this sentence in the next iteration.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"ea95038f29176cf927b98d4561240ab994648870","unresolved":false,"context_lines":[{"line_number":38,"context_line":"those issues."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"This spec could be a statement of commitment by the reviewers of keystone to"},{"line_number":41,"context_line":"support zero-downtime upgrades. Furthermore, there are also new \"assert\" tags,"},{"line_number":42,"context_line":"that could be added to keystone [3]_."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_0fd13a23","line":41,"updated":"2015-12-07 15:50:23.000000000","message":"Is this about 0-downtime or rolling keystone versions? I seem them as two separate issues. Rolling migrations shouldn\u0027t be significantly difficult since we can easily handle it in code if we used the right abstractions. 0-downtime upgrades are much harder since we don\u0027t control the persistence layer itself.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"4b31d158d0b8d9d9aadd52ef0c9f9ec6d9433c48","unresolved":false,"context_lines":[{"line_number":38,"context_line":"those issues."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"This spec could be a statement of commitment by the reviewers of keystone to"},{"line_number":41,"context_line":"support zero-downtime upgrades. Furthermore, there are also new \"assert\" tags,"},{"line_number":42,"context_line":"that could be added to keystone [3]_."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_ea44dcb7","line":41,"in_reply_to":"7a740942_0fd13a23","updated":"2015-12-07 16:06:19.000000000","message":"I\u0027m not sure we use the same definitions of a rolling upgrade. I understand it as the situation when we roll the upgrade across the cluster, so we upgrade each node one-by-one, without stopping them all at once.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bbc202182347fc7535ff257df6b9ea4244563d9f","unresolved":false,"context_lines":[{"line_number":38,"context_line":"those issues."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"This spec could be a statement of commitment by the reviewers of keystone to"},{"line_number":41,"context_line":"support zero-downtime upgrades. Furthermore, there are also new \"assert\" tags,"},{"line_number":42,"context_line":"that could be added to keystone [3]_."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_35448448","line":41,"in_reply_to":"7a740942_ea44dcb7","updated":"2015-12-07 16:55:18.000000000","message":"That\u0027s exactly what I mean by rolling upgrade. A rolling upgrade, to me, doesn\u0027t mean 0 downtime. It means likely that you are at least shrinking the downtime window.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"ea95038f29176cf927b98d4561240ab994648870","unresolved":false,"context_lines":[{"line_number":47,"context_line":"Queries generated by SQLAlchemy expect fields defined there to be present"},{"line_number":48,"context_line":"inside the DB. To keep the database in sync, schema migration is executed"},{"line_number":49,"context_line":"before new version of code is run. The key concept of a rolling upgrade is that"},{"line_number":50,"context_line":"two versions may run at the same time, so we want to keep structures which are"},{"line_number":51,"context_line":"used by the previous version. New structures are ignored by this old version,"},{"line_number":52,"context_line":"containing old SQLAlchemy models, while the new code can support new structures"},{"line_number":53,"context_line":"of any shape. Old columns and tables can only be removed when no longer used"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_8fdf6a0c","line":50,"updated":"2015-12-07 15:50:23.000000000","message":"Can we explicitly state that we are only supporting migrating 1 version back? You kind of say it below at the end of this paragraph, but I think it should be front and center.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"ea95038f29176cf927b98d4561240ab994648870","unresolved":false,"context_lines":[{"line_number":53,"context_line":"of any shape. Old columns and tables can only be removed when no longer used"},{"line_number":54,"context_line":"by both current and the previous release (which may still be running)."},{"line_number":55,"context_line":"Upgrades have to be applied incrementally, one release at a time, with only two"},{"line_number":56,"context_line":"adjacent versions running simultaneously."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":59,"context_line":"schema changes which cause those incompatibilities, specifically drops and"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_6f86bedc","line":56,"updated":"2015-12-07 15:50:23.000000000","message":"So what is the prescribed way to upgrade? It seems based on this paragraph that the keystone code should be rolled out to all of the nodes and then the database schema gets upgraded. Can we describe that somewhere?","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"4b31d158d0b8d9d9aadd52ef0c9f9ec6d9433c48","unresolved":false,"context_lines":[{"line_number":53,"context_line":"of any shape. Old columns and tables can only be removed when no longer used"},{"line_number":54,"context_line":"by both current and the previous release (which may still be running)."},{"line_number":55,"context_line":"Upgrades have to be applied incrementally, one release at a time, with only two"},{"line_number":56,"context_line":"adjacent versions running simultaneously."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":59,"context_line":"schema changes which cause those incompatibilities, specifically drops and"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_ea8e1ccf","line":56,"in_reply_to":"7a740942_6f86bedc","updated":"2015-12-07 16:06:19.000000000","message":"I wouldn\u0027t roll-out the code to all nodes before upgrading the schema. I would take one node out of the cluster, stop the service, upgrade it, upgrade the schema and start it.\n\nThen do the same for all other nodes (where upgrading the schema will be a no-op).","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bbc202182347fc7535ff257df6b9ea4244563d9f","unresolved":false,"context_lines":[{"line_number":53,"context_line":"of any shape. Old columns and tables can only be removed when no longer used"},{"line_number":54,"context_line":"by both current and the previous release (which may still be running)."},{"line_number":55,"context_line":"Upgrades have to be applied incrementally, one release at a time, with only two"},{"line_number":56,"context_line":"adjacent versions running simultaneously."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"To address the problem of schema incompatibilities between versions, we can ban"},{"line_number":59,"context_line":"schema changes which cause those incompatibilities, specifically drops and"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_f5603cd1","line":56,"in_reply_to":"7a740942_ea8e1ccf","updated":"2015-12-07 16:55:18.000000000","message":"This doesn\u0027t make sense to me. You\u0027ll only update the schema once. When will that be?","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"804a02c5d2aba93d5dbaeee09504db4df6d3ce75","unresolved":false,"context_lines":[{"line_number":65,"context_line":"The unit test blocks all alters and drops, but also contains a list of"},{"line_number":66,"context_line":"migrations where we allow altering dropping things."},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"The rules for adding exceptions are very specific:"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"1) Migrations which don\u0027t cause incompatibilities are allowed, for example"},{"line_number":71,"context_line":"   dropping an index or constraint."}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_f574bc36","line":68,"updated":"2015-12-07 17:00:20.000000000","message":"We should also call out that adding required columns that don\u0027t have a sane default will not be allowed.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"959b6c6b7803c8338adc5b704ed98fa8ac8ff42d","unresolved":false,"context_lines":[{"line_number":82,"context_line":"We start requiring things to be additive in Mitaka, so we ignore all migrations"},{"line_number":83,"context_line":"before that point."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Some types of changes, like changing the format in which data is stored, may"},{"line_number":86,"context_line":"require live migration of data. Since there are no ready-made solutions, it is"},{"line_number":87,"context_line":"up to the developer to decide how this is achieved. The data migration may be"},{"line_number":88,"context_line":"started automatically, when the new version is introduced, while an old version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_e55df515","line":85,"updated":"2015-12-07 14:17:57.000000000","message":"This paragraph describes one approach to data migration. If you agree, I will remove it and propose a new spec for data migration.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"9c6c431eebbddca1526cff4caa6224acabb875c8","unresolved":false,"context_lines":[{"line_number":82,"context_line":"We start requiring things to be additive in Mitaka, so we ignore all migrations"},{"line_number":83,"context_line":"before that point."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Some types of changes, like changing the format in which data is stored, may"},{"line_number":86,"context_line":"require live migration of data. Since there are no ready-made solutions, it is"},{"line_number":87,"context_line":"up to the developer to decide how this is achieved. The data migration may be"},{"line_number":88,"context_line":"started automatically, when the new version is introduced, while an old version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_4fe73f11","line":85,"in_reply_to":"7a740942_0f0ffa37","updated":"2015-12-07 16:29:46.000000000","message":"This spec would pertain only to schema migration, so I would allow data migrations in the same way as they are allowed right now.\n\nThe introduced unit test would still disallow dropping and altering (when they cause schema incompatibilities).\n\nAn example would be, that to change a column type, you couldn\u0027t do that in place, you would have to add a new column, then migrate data to it, and refrain from removing the old column. It would have to be removed in the next release, when it\u0027s no longer used by the previous version.\n\nBefore introducing online _data_ migration to keystone, the operator could decide if he upgrades the service with downtime, achieving full consistency, or without downtime, introducing inconsistencies. He could also work around those inconsistencies, if they are explained in the release notes.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"82a48df73e217a36d272eb3103c86e278f8f694e","unresolved":false,"context_lines":[{"line_number":82,"context_line":"We start requiring things to be additive in Mitaka, so we ignore all migrations"},{"line_number":83,"context_line":"before that point."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Some types of changes, like changing the format in which data is stored, may"},{"line_number":86,"context_line":"require live migration of data. Since there are no ready-made solutions, it is"},{"line_number":87,"context_line":"up to the developer to decide how this is achieved. The data migration may be"},{"line_number":88,"context_line":"started automatically, when the new version is introduced, while an old version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_5eb93a78","line":85,"in_reply_to":"7a740942_1e965293","updated":"2015-12-07 15:03:31.000000000","message":"I want to divide this spec into schema migration and data migration.\n\nSolve schema migration problem in this release, next gather more feedback from the operators, and then follow up with solving the data migration problem, with which some of the reviewers (mainly Brand Knudson I think) had concerns.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"ea95038f29176cf927b98d4561240ab994648870","unresolved":false,"context_lines":[{"line_number":82,"context_line":"We start requiring things to be additive in Mitaka, so we ignore all migrations"},{"line_number":83,"context_line":"before that point."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Some types of changes, like changing the format in which data is stored, may"},{"line_number":86,"context_line":"require live migration of data. Since there are no ready-made solutions, it is"},{"line_number":87,"context_line":"up to the developer to decide how this is achieved. The data migration may be"},{"line_number":88,"context_line":"started automatically, when the new version is introduced, while an old version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_0f0ffa37","line":85,"in_reply_to":"7a740942_5eb93a78","updated":"2015-12-07 15:50:23.000000000","message":"I don\u0027t think you can actually split the two unless you are saying that this particular spec in about not allowing certain types of migrations that would cause a data migration. This spec is a bit all over the map so I\u0027m not entirely sure what it\u0027s trying to do.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"cb96923ac38629a4ea02e0b664e596ebf0c5b10b","unresolved":false,"context_lines":[{"line_number":82,"context_line":"We start requiring things to be additive in Mitaka, so we ignore all migrations"},{"line_number":83,"context_line":"before that point."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Some types of changes, like changing the format in which data is stored, may"},{"line_number":86,"context_line":"require live migration of data. Since there are no ready-made solutions, it is"},{"line_number":87,"context_line":"up to the developer to decide how this is achieved. The data migration may be"},{"line_number":88,"context_line":"started automatically, when the new version is introduced, while an old version"}],"source_content_type":"text/x-rst","patch_set":8,"id":"7a740942_1e965293","line":85,"in_reply_to":"7a740942_e55df515","updated":"2015-12-07 14:52:49.000000000","message":"Why put that in another schema? This one should contain all of the details about online schema migrations.","commit_id":"36864f14f30a973a22430b8d6525f9ffd0d37faf"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"f78633a1c9ef960dcf86a8d3393cd05930b3dccf","unresolved":false,"context_lines":[{"line_number":132,"context_line":"configuration, which switches the place from which the data is read. In cases"},{"line_number":133,"context_line":"when a table is no longer used it can be dropped in the next release."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"Before removing columns and tables, a sanity check should be performed,"},{"line_number":136,"context_line":"ensuring all data was migrated."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"7a740942_18c4145d","line":135,"updated":"2015-12-07 22:01:24.000000000","message":"This sanity check can be done in the form of another migration script. The migration will fail unless all data in the table is updated to NULL or migrated. This migration script should be prior to the migration that drops the old table/column.","commit_id":"7fcdc8625f274da76b2383e2ddcd2ad00fefdb90"},{"author":{"_account_id":6482,"name":"Steve Martinelli","email":"s.martinelli@gmail.com","username":"stevemar"},"change_message_id":"cfb6ebc97044bd6a98d0165148206b52a56d6d75","unresolved":false,"context_lines":[{"line_number":217,"context_line":"* Prepare a unit test blocking alters and drops in SQL migrations."},{"line_number":218,"context_line":"* Prepare developer documentation with examples."},{"line_number":219,"context_line":"* Help review patches which fall over on this test."},{"line_number":220,"context_line":"* Add grenade CI test, with altering requests sent to two keystone instances"},{"line_number":221,"context_line":"  at different versions"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"7a740942_852de2a9","line":220,"updated":"2015-12-08 09:04:44.000000000","message":"i\u0027m OK with making this a stretch goal","commit_id":"7fcdc8625f274da76b2383e2ddcd2ad00fefdb90"}]}