)]}'
{"specs/keystone/newton/manage-migation.rst":[{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"989e48cd756763027019556476325fdd64c68674","unresolved":false,"context_lines":[{"line_number":72,"context_line":"*db_sync_migration_complete*"},{"line_number":73,"context_line":"*db_sync* (still supported, but deprecated, and will execute"},{"line_number":74,"context_line":"db_sync_migration_prepare followed by printing a warning message reminding the"},{"line_number":75,"context_line":"operator to run db_sync_migration_complete once the migration is over)"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"In terms of implementation, it is proposed that we will continue to use a"},{"line_number":78,"context_line":"single migration repository to handle the two db_sync commands, with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1aa78d24_dc4c8b44","line":75,"updated":"2016-07-05 16:51:37.000000000","message":"I think we shouldn\u0027t change the behavior of the old command. It should run all migration steps, to stay compatible with any old upgrade scripts which may run it in an offline upgrade scenario.","commit_id":"78efaf72ca535e29750683ee06ce31e12d9bbad6"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"26a1b759f7c5107f62834edb04a78715ebe2c0e3","unresolved":false,"context_lines":[{"line_number":72,"context_line":"*db_sync_migration_complete*"},{"line_number":73,"context_line":"*db_sync* (still supported, but deprecated, and will execute"},{"line_number":74,"context_line":"db_sync_migration_prepare followed by printing a warning message reminding the"},{"line_number":75,"context_line":"operator to run db_sync_migration_complete once the migration is over)"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"In terms of implementation, it is proposed that we will continue to use a"},{"line_number":78,"context_line":"single migration repository to handle the two db_sync commands, with"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1aa78d24_3cf2ef6f","line":75,"in_reply_to":"1aa78d24_dc4c8b44","updated":"2016-07-05 17:00:50.000000000","message":"The problem with that is, since we do not support downgrades, if an operator does this due to habit, they have now prevented rolling upgrades, since the database has been migrated too far.","commit_id":"78efaf72ca535e29750683ee06ce31e12d9bbad6"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"85da21b48879f9ff40c7a734e1c7d449e88c27b8","unresolved":false,"context_lines":[{"line_number":18,"context_line":"Problem Description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Openstack services are moving towards support for rolling upgrades, with"},{"line_number":22,"context_line":"today, nova, neutron and swift claiming such support. Other services are"},{"line_number":23,"context_line":"working on this (e.g. glance). See the References section for more"},{"line_number":24,"context_line":"information on support in other services. Prior to Newton, keystone did not"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_489575e2","line":21,"updated":"2016-07-11 14:36:48.000000000","message":"OpenStack*","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"4abd5722b5530876a952f31d01d1fbd9108d415b","unresolved":false,"context_lines":[{"line_number":18,"context_line":"Problem Description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Openstack services are moving towards support for rolling upgrades, with"},{"line_number":22,"context_line":"today, nova, neutron and swift claiming such support. Other services are"},{"line_number":23,"context_line":"working on this (e.g. glance). See the References section for more"},{"line_number":24,"context_line":"information on support in other services. Prior to Newton, keystone did not"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_de58c833","line":21,"in_reply_to":"1aa78d24_489575e2","updated":"2016-07-12 11:07:36.000000000","message":"Done","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"85da21b48879f9ff40c7a734e1c7d449e88c27b8","unresolved":false,"context_lines":[{"line_number":36,"context_line":"users during large database migrations while carrying out a)."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"In Newton, we plan to support a), but not b), although will attempt to"},{"line_number":39,"context_line":"minimize any temporary lockout. We initiated support for rolling upgrades by"},{"line_number":40,"context_line":"only allowing additive changes to the database (this is protected via tests)."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"To support just the rolling upgrade part, keystone would allow (given we have"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_c85e8544","line":39,"updated":"2016-07-11 14:36:48.000000000","message":"By temporary lockout do you mean downtime?","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"4abd5722b5530876a952f31d01d1fbd9108d415b","unresolved":false,"context_lines":[{"line_number":36,"context_line":"users during large database migrations while carrying out a)."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"In Newton, we plan to support a), but not b), although will attempt to"},{"line_number":39,"context_line":"minimize any temporary lockout. We initiated support for rolling upgrades by"},{"line_number":40,"context_line":"only allowing additive changes to the database (this is protected via tests)."},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"To support just the rolling upgrade part, keystone would allow (given we have"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_5e5cd818","line":39,"in_reply_to":"1aa78d24_c85e8544","updated":"2016-07-12 11:07:36.000000000","message":"Well not formal downtime...but the database might be unavailable for, say, write actions for a while...I\u0027ll clarify","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"85da21b48879f9ff40c7a734e1c7d449e88c27b8","unresolved":false,"context_lines":[{"line_number":74,"context_line":"the existing rolling upgrade problem, and yet fits within the context of the"},{"line_number":75,"context_line":"full rolling upgrades and zero down-time cross project initiative. This"},{"line_number":76,"context_line":"\"migration complete\" command, which an operator will run once all nodes have"},{"line_number":77,"context_line":"been migrated, and will tidy up the database (e.g. write defaults to any new"},{"line_number":78,"context_line":"attributes that need them)."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"There seems little or no commonality of the actual commands used by other"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_f3fed2ac","line":77,"updated":"2016-07-11 14:36:48.000000000","message":"By migrated do you mean the code? Once all keystone nodes have been updated with the latest application code?","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"4abd5722b5530876a952f31d01d1fbd9108d415b","unresolved":false,"context_lines":[{"line_number":74,"context_line":"the existing rolling upgrade problem, and yet fits within the context of the"},{"line_number":75,"context_line":"full rolling upgrades and zero down-time cross project initiative. This"},{"line_number":76,"context_line":"\"migration complete\" command, which an operator will run once all nodes have"},{"line_number":77,"context_line":"been migrated, and will tidy up the database (e.g. write defaults to any new"},{"line_number":78,"context_line":"attributes that need them)."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"There seems little or no commonality of the actual commands used by other"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_5eb3384d","line":77,"in_reply_to":"1aa78d24_f3fed2ac","updated":"2016-07-12 11:07:36.000000000","message":"yes, I\u0027ll clarify","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"85da21b48879f9ff40c7a734e1c7d449e88c27b8","unresolved":false,"context_lines":[{"line_number":75,"context_line":"full rolling upgrades and zero down-time cross project initiative. This"},{"line_number":76,"context_line":"\"migration complete\" command, which an operator will run once all nodes have"},{"line_number":77,"context_line":"been migrated, and will tidy up the database (e.g. write defaults to any new"},{"line_number":78,"context_line":"attributes that need them)."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"There seems little or no commonality of the actual commands used by other"},{"line_number":81,"context_line":"services\u0027 manage utility for the full zero down-time support, but"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_d36d9633","line":78,"updated":"2016-07-11 14:36:48.000000000","message":"I believe nova does something like using blocking migrations. For example, say migration X does something that introduces a default and migration X + 1 cannot be run until all data has been migrated. I think there is a way to make it so that X + 1 will fail unless all data is migrated. Then the flow would be something like: \n\n    - deployer runs db_sync\n    - migrates up to migration X but fails to migrate X + 1\n    - deployer uses additional migration helpers to manually migrate the data in batches\n    - deployer reruns the migration which successfully migrates past X + 1 since all data was migrated","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"4abd5722b5530876a952f31d01d1fbd9108d415b","unresolved":false,"context_lines":[{"line_number":75,"context_line":"full rolling upgrades and zero down-time cross project initiative. This"},{"line_number":76,"context_line":"\"migration complete\" command, which an operator will run once all nodes have"},{"line_number":77,"context_line":"been migrated, and will tidy up the database (e.g. write defaults to any new"},{"line_number":78,"context_line":"attributes that need them)."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"There seems little or no commonality of the actual commands used by other"},{"line_number":81,"context_line":"services\u0027 manage utility for the full zero down-time support, but"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_9e63e0b0","line":78,"in_reply_to":"1aa78d24_d36d9633","updated":"2016-07-12 11:07:36.000000000","message":"OK, so I guess that is a different kind of \"tidy up\" requirement...I assume, for nova, one where it needs to wait for all the on-fly-migrations to happen - but if some data doesn\u0027t get accesses (and hence migrated) they need a way for forcing the rest over the line.  I think this is typically required before N+2 is run.","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"change_message_id":"85da21b48879f9ff40c7a734e1c7d449e88c27b8","unresolved":false,"context_lines":[{"line_number":103,"context_line":"will remove any columns/tables that are no longer referenced."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"For keystone in this release, it is proposed that we support two of the"},{"line_number":106,"context_line":"above commands, that allow us to deliver out needs:"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"*keystone-manage upgrade --expand* - this will, for keystone, both expand the"},{"line_number":109,"context_line":"data and do the migration (i.e. steps 1-4 above). We would need to document"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_d35e3644","line":106,"updated":"2016-07-11 14:36:48.000000000","message":"s/out/our/ ?","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"4abd5722b5530876a952f31d01d1fbd9108d415b","unresolved":false,"context_lines":[{"line_number":103,"context_line":"will remove any columns/tables that are no longer referenced."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"For keystone in this release, it is proposed that we support two of the"},{"line_number":106,"context_line":"above commands, that allow us to deliver out needs:"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"*keystone-manage upgrade --expand* - this will, for keystone, both expand the"},{"line_number":109,"context_line":"data and do the migration (i.e. steps 1-4 above). We would need to document"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_def448f2","line":106,"in_reply_to":"1aa78d24_d35e3644","updated":"2016-07-12 11:07:36.000000000","message":"Done","commit_id":"a28f6bc86020d6e4396307e25f54f6eee0cff7d8"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":15,"context_line":"database tidy-ups."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem Description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"OpenStack services are moving towards support for rolling upgrades, with"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_23dcc245","line":18,"range":{"start_line":18,"start_character":0,"end_line":18,"end_character":19},"updated":"2016-07-14 19:02:12.000000000","message":"Overall, I just think the problem description could be much more succinct.  Please rewrite.","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":18,"context_line":"Problem Description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"OpenStack services are moving towards support for rolling upgrades, with"},{"line_number":22,"context_line":"today, nova, neutron and swift claiming such support. Other services are"},{"line_number":23,"context_line":"working on this (e.g. glance). See the References section for more"},{"line_number":24,"context_line":"information on support in other services. Prior to Newton, keystone did not"},{"line_number":25,"context_line":"claim any such support, but we have decided to add this."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"First, it is important to state that there are actually two intertwined"},{"line_number":28,"context_line":"features when people talk about rolling upgrades;"},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"a) The ability, in a multi-node service, to roll upgrades to nodes one at a"},{"line_number":31,"context_line":"time while the overall services remains in operation - so by definition"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_cf2082b3","line":28,"range":{"start_line":21,"start_character":0,"end_line":28,"end_character":49},"updated":"2016-07-14 19:02:12.000000000","message":"This is more like background/history information, as opposed to the problem description.","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":27,"context_line":"First, it is important to state that there are actually two intertwined"},{"line_number":28,"context_line":"features when people talk about rolling upgrades;"},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"a) The ability, in a multi-node service, to roll upgrades to nodes one at a"},{"line_number":31,"context_line":"time while the overall services remains in operation - so by definition"},{"line_number":32,"context_line":"allowing a mix of old and new nodes to run together at the same time using"},{"line_number":33,"context_line":"the same database."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"b) Provide zero down-time for the service, including no temporary lockout of"},{"line_number":36,"context_line":"users during large database migrations while carrying out a)."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"In Newton, we plan to support a), but not b), although will attempt to"},{"line_number":39,"context_line":"minimize any temporary lockout (e.g. when database writes might be blocked"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_cf59a25d","line":36,"range":{"start_line":30,"start_character":0,"end_line":36,"end_character":61},"updated":"2016-07-14 19:02:12.000000000","message":"Love a and b ++","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":39,"context_line":"minimize any temporary lockout (e.g. when database writes might be blocked"},{"line_number":40,"context_line":"while data migration is happening). We initiated support for rolling upgrades"},{"line_number":41,"context_line":"by only allowing additive changes to the database (this is protected via"},{"line_number":42,"context_line":"tests)."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"To support just the rolling upgrade part, keystone would allow (given we have"},{"line_number":45,"context_line":"only permitted additive database changes) the following sequence:"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_433bcec8","line":42,"range":{"start_line":42,"start_character":5,"end_line":42,"end_character":7},"updated":"2016-07-14 19:02:12.000000000","message":"You could reference the tests:\nhttps://github.com/openstack/keystone/blob/master/keystone/tests/unit/test_sql_banned_operations.py#L33","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":44,"context_line":"To support just the rolling upgrade part, keystone would allow (given we have"},{"line_number":45,"context_line":"only permitted additive database changes) the following sequence:"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"1) Take a keystone out of service and replace keystone with the new version,"},{"line_number":48,"context_line":"but don\u0027t run it yet. Execute the keystone-manage command to sync the database."},{"line_number":49,"context_line":"Wait and ensure all older nodes run fine. Only when you are happy with the"},{"line_number":50,"context_line":"situation, start the new keystone and bring it online."}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_63dc0a75","line":47,"range":{"start_line":47,"start_character":3,"end_line":47,"end_character":33},"updated":"2016-07-14 19:02:12.000000000","message":"This makes it sounds like we are taking keystone down entirely. May be instead say something like, \"install new version, one at a time on each node...","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":44,"context_line":"To support just the rolling upgrade part, keystone would allow (given we have"},{"line_number":45,"context_line":"only permitted additive database changes) the following sequence:"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"1) Take a keystone out of service and replace keystone with the new version,"},{"line_number":48,"context_line":"but don\u0027t run it yet. Execute the keystone-manage command to sync the database."},{"line_number":49,"context_line":"Wait and ensure all older nodes run fine. Only when you are happy with the"},{"line_number":50,"context_line":"situation, start the new keystone and bring it online."}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_6a5fec0e","line":47,"range":{"start_line":47,"start_character":3,"end_line":47,"end_character":33},"in_reply_to":"1aa78d24_63dc0a75","updated":"2016-07-15 20:23:00.000000000","message":"+1\n\"a keystone node\" maybe?","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":45,"context_line":"only permitted additive database changes) the following sequence:"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"1) Take a keystone out of service and replace keystone with the new version,"},{"line_number":48,"context_line":"but don\u0027t run it yet. Execute the keystone-manage command to sync the database."},{"line_number":49,"context_line":"Wait and ensure all older nodes run fine. Only when you are happy with the"},{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_6a8d2c7b","line":48,"range":{"start_line":48,"start_character":61,"end_line":48,"end_character":65},"updated":"2016-07-15 20:23:00.000000000","message":"Is it better to call it expand here?","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":45,"context_line":"only permitted additive database changes) the following sequence:"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"1) Take a keystone out of service and replace keystone with the new version,"},{"line_number":48,"context_line":"but don\u0027t run it yet. Execute the keystone-manage command to sync the database."},{"line_number":49,"context_line":"Wait and ensure all older nodes run fine. Only when you are happy with the"},{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_63c36a8b","line":48,"range":{"start_line":48,"start_character":4,"end_line":48,"end_character":20},"updated":"2016-07-14 19:02:12.000000000","message":"You should be able to run both versions","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":47,"context_line":"1) Take a keystone out of service and replace keystone with the new version,"},{"line_number":48,"context_line":"but don\u0027t run it yet. Execute the keystone-manage command to sync the database."},{"line_number":49,"context_line":"Wait and ensure all older nodes run fine. Only when you are happy with the"},{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2) Upgrade the other nodes one at a time, by taking them out of whatever proxy"},{"line_number":53,"context_line":"configuration is being rnn, upgrading keystone and bringing them back on line."}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_e3359aa8","line":50,"range":{"start_line":50,"start_character":11,"end_line":50,"end_character":54},"updated":"2016-07-14 19:02:12.000000000","message":"They should never be offline, that\u0027s the point of rolling upgrades, right?","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":47,"context_line":"1) Take a keystone out of service and replace keystone with the new version,"},{"line_number":48,"context_line":"but don\u0027t run it yet. Execute the keystone-manage command to sync the database."},{"line_number":49,"context_line":"Wait and ensure all older nodes run fine. Only when you are happy with the"},{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2) Upgrade the other nodes one at a time, by taking them out of whatever proxy"},{"line_number":53,"context_line":"configuration is being rnn, upgrading keystone and bringing them back on line."}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_5172a37d","line":50,"range":{"start_line":50,"start_character":11,"end_line":50,"end_character":54},"in_reply_to":"1aa78d24_e3359aa8","updated":"2016-07-15 20:23:00.000000000","message":"If I understand this correctly, it\u0027s just the one node that we take offline to expand the database. The rest continue to function as-is.","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2) Upgrade the other nodes one at a time, by taking them out of whatever proxy"},{"line_number":53,"context_line":"configuration is being rnn, upgrading keystone and bringing them back on line."},{"line_number":54,"context_line":"No database changes will take place during these steps since this was already"},{"line_number":55,"context_line":"done in 1)."},{"line_number":56,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_eaf75cf3","line":53,"range":{"start_line":53,"start_character":23,"end_line":53,"end_character":26},"updated":"2016-07-15 20:23:00.000000000","message":"Nit: run","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":49,"context_line":"Wait and ensure all older nodes run fine. Only when you are happy with the"},{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2) Upgrade the other nodes one at a time, by taking them out of whatever proxy"},{"line_number":53,"context_line":"configuration is being rnn, upgrading keystone and bringing them back on line."},{"line_number":54,"context_line":"No database changes will take place during these steps since this was already"},{"line_number":55,"context_line":"done in 1)."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"In order to support such rolling upgrades, some migration scripts"},{"line_number":58,"context_line":"cannot leave the database in the state they would ideally like. An example"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_c35fbee2","line":55,"range":{"start_line":52,"start_character":0,"end_line":55,"end_character":10},"updated":"2016-07-14 19:02:12.000000000","message":"I\u0027m missing something here...","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":60,"context_line":"to be written manually by code (rather than a server default). If, during a"},{"line_number":61,"context_line":"rolling upgrade, new entities are created via un-migrated nodes, such a new"},{"line_number":62,"context_line":"attribute will not get the default. See:"},{"line_number":63,"context_line":"\u003chttps://bugs.launchpad.net/keystone/+bug/1596500\u003e`_"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"Support for zero down-time, would require keystone to support the gradual"},{"line_number":66,"context_line":"on-the-fly migration of data as they are accessed, with support for a final"}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_4adb08ee","line":63,"range":{"start_line":63,"start_character":0,"end_line":63,"end_character":52},"updated":"2016-07-15 20:23:00.000000000","message":"We\u0027ll probably run into the same issue with Glance as well.","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":63,"context_line":"\u003chttps://bugs.launchpad.net/keystone/+bug/1596500\u003e`_"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"Support for zero down-time, would require keystone to support the gradual"},{"line_number":66,"context_line":"on-the-fly migration of data as they are accessed, with support for a final"},{"line_number":67,"context_line":"manual \"migrate anything left\" action. Such on-the-fly support is usually"},{"line_number":68,"context_line":"implemented in the database object layer, which has support for looking in"},{"line_number":69,"context_line":"both the old and new locations for data. It is anticipated that keystone will"},{"line_number":70,"context_line":"eventually require this level of support."}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_250c0594","line":67,"range":{"start_line":66,"start_character":50,"end_line":67,"end_character":37},"updated":"2016-07-15 20:23:00.000000000","message":"Is this going to migrate everything at one go? could this be done in smaller batches (a few rows at once) to minimize database load and hence any service disruption?","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"It is proposed that we add to keystone-manage a new capability that just solves"},{"line_number":76,"context_line":"the existing rolling upgrade problem, and yet fits within the context of the"},{"line_number":77,"context_line":"full rolling upgrades and zero down-time cross project initiative. This"},{"line_number":78,"context_line":"\"migration complete\" command, which an operator will run once all nodes have"},{"line_number":79,"context_line":"had their code upgrade to the new version, and will tidy up the database"},{"line_number":80,"context_line":"(e.g. write defaults to any new attributes that need them)."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"There seems little or no commonality of the actual commands used by other"},{"line_number":83,"context_line":"services\u0027 manage utility for the full zero down-time support, but"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_c3bb3ee3","line":80,"range":{"start_line":77,"start_character":67,"end_line":80,"end_character":59},"updated":"2016-07-14 19:02:12.000000000","message":"Love it! ++","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":84,"context_line":"conceptually, given release cycles of N, N+1 etc., the following actions are"},{"line_number":85,"context_line":"required given a multi-node configuration running code release N:"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"1) One node is upgraded to N+1 the *\u003cservice\u003e-manage upgrade --expand*"},{"line_number":88,"context_line":"command is run which creates new database structures, but does not migrate any"},{"line_number":89,"context_line":"data"},{"line_number":90,"context_line":"2) The nodes are then upgraded individually to N+1, which includes code that"},{"line_number":91,"context_line":"knows how to find data in both old and new locations (all the data is still"},{"line_number":92,"context_line":"in the old locations at this point). N+1 code must write to the old location"},{"line_number":93,"context_line":"(it is also allowable for N+1 code to duplicate the data to the new location,"},{"line_number":94,"context_line":"for data that cannot be updated by N code)."},{"line_number":95,"context_line":"3) Once all the nodes are migrated, then on-the-fly migration begins (triggered"},{"line_number":96,"context_line":"either by setting a config flag or a manage command). Once started, as data is"},{"line_number":97,"context_line":"accessed, it is migrated. N+1 code now only ever writes to the new location."},{"line_number":98,"context_line":"4) At some point in the future (and before an upgrade to N+2), the"},{"line_number":99,"context_line":"*\u003cservice\u003e-manage upgrade --migrate-force-complete* command is executed"},{"line_number":100,"context_line":"(perhaps with options for batching) that migrates any remaining data. (Note the"},{"line_number":101,"context_line":"actual command name for this phase varies wildly across services, the name"},{"line_number":102,"context_line":"given here is my own suggestion)."},{"line_number":103,"context_line":"5) The process above is repeated for N+2. Once all N+1 nodes have been upgraded"},{"line_number":104,"context_line":"then the *\u003cservice\u003e-manage upgrade --contract* command can be executed that"},{"line_number":105,"context_line":"will remove any columns/tables that are no longer referenced."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"For keystone in this release, it is proposed that we support two of the"},{"line_number":108,"context_line":"above commands, that allow us to deliver our needs:"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_a3ac328d","line":105,"range":{"start_line":87,"start_character":0,"end_line":105,"end_character":61},"updated":"2016-07-14 19:02:12.000000000","message":"Is this intended to be a sequence?  Isn\u0027t this as simple as, after all nodes have been upgraded, run keystone-manage upgrade -- to made database changes and migrate data.","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":84,"context_line":"conceptually, given release cycles of N, N+1 etc., the following actions are"},{"line_number":85,"context_line":"required given a multi-node configuration running code release N:"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"1) One node is upgraded to N+1 the *\u003cservice\u003e-manage upgrade --expand*"},{"line_number":88,"context_line":"command is run which creates new database structures, but does not migrate any"},{"line_number":89,"context_line":"data"},{"line_number":90,"context_line":"2) The nodes are then upgraded individually to N+1, which includes code that"},{"line_number":91,"context_line":"knows how to find data in both old and new locations (all the data is still"},{"line_number":92,"context_line":"in the old locations at this point). N+1 code must write to the old location"},{"line_number":93,"context_line":"(it is also allowable for N+1 code to duplicate the data to the new location,"},{"line_number":94,"context_line":"for data that cannot be updated by N code)."},{"line_number":95,"context_line":"3) Once all the nodes are migrated, then on-the-fly migration begins (triggered"},{"line_number":96,"context_line":"either by setting a config flag or a manage command). Once started, as data is"},{"line_number":97,"context_line":"accessed, it is migrated. N+1 code now only ever writes to the new location."},{"line_number":98,"context_line":"4) At some point in the future (and before an upgrade to N+2), the"},{"line_number":99,"context_line":"*\u003cservice\u003e-manage upgrade --migrate-force-complete* command is executed"},{"line_number":100,"context_line":"(perhaps with options for batching) that migrates any remaining data. (Note the"},{"line_number":101,"context_line":"actual command name for this phase varies wildly across services, the name"},{"line_number":102,"context_line":"given here is my own suggestion)."},{"line_number":103,"context_line":"5) The process above is repeated for N+2. Once all N+1 nodes have been upgraded"},{"line_number":104,"context_line":"then the *\u003cservice\u003e-manage upgrade --contract* command can be executed that"},{"line_number":105,"context_line":"will remove any columns/tables that are no longer referenced."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"For keystone in this release, it is proposed that we support two of the"},{"line_number":108,"context_line":"above commands, that allow us to deliver our needs:"}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_2099936f","line":105,"range":{"start_line":87,"start_character":0,"end_line":105,"end_character":61},"in_reply_to":"1aa78d24_a3ac328d","updated":"2016-07-15 20:23:00.000000000","message":"It usually is a sequence to get the intended result, which could be rolling upgrades or zero-downtime upgrades.\n\nGenerally it\u0027s done in the following sequence with room for variation specific to the projects:\n1. Expand the database \n2. Upgrade the code\n3. Migrate data\n4. Contract database","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":18338,"name":"Ron De Rose","email":"ronald.de.rose@intel.com","username":"derosenet"},"change_message_id":"e7993b0331e5941622cc2cbe5e0b3544ac014d71","unresolved":false,"context_lines":[{"line_number":107,"context_line":"For keystone in this release, it is proposed that we support two of the"},{"line_number":108,"context_line":"above commands, that allow us to deliver our needs:"},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"*keystone-manage db_sync --expand* - this will, for keystone, both expand the"},{"line_number":111,"context_line":"data and do the migration (i.e. steps 1-4 above). We would need to document"},{"line_number":112,"context_line":"that phase might take some time for large databases. This is in place of"},{"line_number":113,"context_line":"db_sync that an operator would have done on previous releases."}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_83ec96bc","line":110,"range":{"start_line":110,"start_character":17,"end_line":110,"end_character":24},"updated":"2016-07-14 19:02:12.000000000","message":"Can we leave db_sync out of this; make this spec scope smaller?","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":107,"context_line":"For keystone in this release, it is proposed that we support two of the"},{"line_number":108,"context_line":"above commands, that allow us to deliver our needs:"},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"*keystone-manage db_sync --expand* - this will, for keystone, both expand the"},{"line_number":111,"context_line":"data and do the migration (i.e. steps 1-4 above). We would need to document"},{"line_number":112,"context_line":"that phase might take some time for large databases. This is in place of"},{"line_number":113,"context_line":"db_sync that an operator would have done on previous releases."},{"line_number":114,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_d8ba2fc0","line":111,"range":{"start_line":110,"start_character":62,"end_line":111,"end_character":25},"updated":"2016-07-15 20:23:00.000000000","message":"Does this expand the database and also start the migrations that are not on-the-fly? \nHow does it know when to start the migrations? These migrations, I would imagine, should start only after N is completely out. If not, we may miss the updates made by N to the old column while the data is being migrated.\n\nAm I missing anything here?","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":112,"context_line":"that phase might take some time for large databases. This is in place of"},{"line_number":113,"context_line":"db_sync that an operator would have done on previous releases."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"*keystone-manage db_sync --migrate-force-complete* - this should be run once"},{"line_number":116,"context_line":"all the nodes have been migrated (i.e. equivalent to step 5 above). Note that,"},{"line_number":117,"context_line":"for consistency across installations, we want this to be run even if the"},{"line_number":118,"context_line":"upgrade was not executed in a rolling fashion (i.e. even if this is a single"},{"line_number":119,"context_line":"node configuration). Although not doing so would not cause any errors under"},{"line_number":120,"context_line":"normal circumstances, it would be better to have all deployed databases at the"},{"line_number":121,"context_line":"same migration level."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"The *db_sync* command (without options) will still be supported, but"},{"line_number":124,"context_line":"deprecated, and will execute *keystone-manage upgrade --expand* followed by"}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_188847e7","line":121,"range":{"start_line":115,"start_character":0,"end_line":121,"end_character":21},"updated":"2016-07-15 20:23:00.000000000","message":"#4 above mentions that \"--migrate-force-complete\" would migrate any remaining data. I was assuming this is the data left behind as a result of on-the-fly migrations not touching a certain set of rows. Is my understand correct here? If it is, how is it different from the migrations done by \"keystone-manage db_sync --expand\"?\n\nOr maybe in other words, what are the concrete actions taken this command?","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":8158,"name":"Hemanth Makkapati","email":"makstacker+osgerrit@gmail.com","username":"hemanth.makkapati"},"change_message_id":"17956b099986aa82f2443fa83eded532d03d2f98","unresolved":false,"context_lines":[{"line_number":136,"context_line":"database migrations that have been performed so far will be run upon execution"},{"line_number":137,"context_line":"of *keystone-manage upgrade --migrate-force-complete*. This approach allows"},{"line_number":138,"context_line":"support both for rolling upgrade migrations at major release as well as those"},{"line_number":139,"context_line":"deployers who are tracking master."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Alternatives"},{"line_number":142,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":6,"id":"fad55976_b84e5bbf","line":139,"updated":"2016-07-15 20:23:00.000000000","message":"Can you throw some more light on how you guys plan to maintain expand and contract migration streams separately? \nJust curious because this is something we are trying to tackle for Glance as well.","commit_id":"394af2c2e82b5f3871206f096244c3fd2408cf84"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"5fe4bb88142fcc52dcef6a3f084d6834126fb35f","unresolved":false,"context_lines":[{"line_number":103,"context_line":"   *\u003cservice\u003e-manage upgrade --rolling-upgrade-complete*. Each node is"},{"line_number":104,"context_line":"   re-cycled one at a time. All code (which by definition is X+1) can now"},{"line_number":105,"context_line":"   migrate data from the old column to the new column on-the-fly as data is"},{"line_number":106,"context_line":"   accessed. Writes only happen to the new columns."},{"line_number":107,"context_line":"6) At some point in the future (and before an upgrade to X+2), the"},{"line_number":108,"context_line":"   *\u003cservice\u003e-manage upgrade --force-migrate-complete* command is executed"},{"line_number":109,"context_line":"   (perhaps with options for batching) that migrates any remaining data. (Note"}],"source_content_type":"text/x-rst","patch_set":9,"id":"dada55a8_a3d84d54","line":106,"range":{"start_line":106,"start_character":13,"end_line":106,"end_character":51},"updated":"2016-07-26 18:19:57.000000000","message":"In 3) you still read data from the old column, even in the new code version, but here you start writing only to the new column.\n\nSo if a row is modified at this point by writing only to the new column, nodes that are performing reads with the old configuration won\u0027t get that update and will read outdated information.\n\nI think you need a step in between, where you start reading from the new column, but still update the old column.","commit_id":"5d7df83bf7263286e2fe156658771d031efd2c13"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bfb29c39065724d326c9f88ac1d6da2d2ec71add","unresolved":false,"context_lines":[{"line_number":28,"context_line":"features when people talk about rolling upgrades;"},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"a) The ability, in a multi-node service, to roll upgrades to nodes one at a"},{"line_number":31,"context_line":"time while the overall services remains in operation - so by definition"},{"line_number":32,"context_line":"allowing a mix of old and new nodes to run together at the same time using"},{"line_number":33,"context_line":"the same database."},{"line_number":34,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_c5cecdb6","line":31,"updated":"2016-08-02 14:40:32.000000000","message":"either \u0027service remains\u0027 or \u0027services remain\u0027 - i\u0027d prefer the former","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"change_message_id":"1df13bb53478f33c85564f3199d9d91aba5d5970","unresolved":false,"context_lines":[{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2) Upgrade the other nodes one at a time, by taking them out of whatever proxy"},{"line_number":53,"context_line":"configuration is being rnn, upgrading keystone and bringing them back on line."},{"line_number":54,"context_line":"No database changes will take place during these steps since this was already"},{"line_number":55,"context_line":"done in 1)."},{"line_number":56,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_b7615214","line":53,"range":{"start_line":53,"start_character":23,"end_line":53,"end_character":26},"updated":"2016-08-02 13:52:16.000000000","message":"run","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bfb29c39065724d326c9f88ac1d6da2d2ec71add","unresolved":false,"context_lines":[{"line_number":50,"context_line":"situation, start the new keystone and bring it online."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2) Upgrade the other nodes one at a time, by taking them out of whatever proxy"},{"line_number":53,"context_line":"configuration is being rnn, upgrading keystone and bringing them back on line."},{"line_number":54,"context_line":"No database changes will take place during these steps since this was already"},{"line_number":55,"context_line":"done in 1)."},{"line_number":56,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_0525855b","line":53,"range":{"start_line":53,"start_character":23,"end_line":53,"end_character":26},"in_reply_to":"bacf61ea_b7615214","updated":"2016-08-02 14:40:32.000000000","message":"also online","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bfb29c39065724d326c9f88ac1d6da2d2ec71add","unresolved":false,"context_lines":[{"line_number":60,"context_line":"to be written manually by code (rather than a server default). If, during a"},{"line_number":61,"context_line":"rolling upgrade, new entities are created via un-migrated nodes, such a new"},{"line_number":62,"context_line":"attribute will not get the default. See:"},{"line_number":63,"context_line":"\u003chttps://bugs.launchpad.net/keystone/+bug/1596500\u003e`_"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"Support for zero down-time, would require keystone to support the gradual"},{"line_number":66,"context_line":"on-the-fly migration of data as they are accessed, with support for a final"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_859e9561","line":63,"updated":"2016-08-02 14:40:32.000000000","message":"This link needs link text","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":7191,"name":"Jamie Lennox","email":"jamielennox@gmail.com","username":"jamielennox"},"change_message_id":"6119a43e4b588896d41ccabf5662466b4f573e44","unresolved":false,"context_lines":[{"line_number":89,"context_line":"1) One node has its code upgraded to X+1 the"},{"line_number":90,"context_line":"   *\u003cservice\u003e-manage upgrade --expand* command is run which creates new"},{"line_number":91,"context_line":"   database structures, but does not migrate any data. This command also sets"},{"line_number":92,"context_line":"   the database status flag \"migration_phase\" to \"Mixed X and X+1 nodes\""},{"line_number":93,"context_line":"2) This node is restarted and we are now running a mix of X and X+1. X+1 code"},{"line_number":94,"context_line":"   knows that if we are in a mix of X and X+1 nodes, then it must still read"},{"line_number":95,"context_line":"   and write data to the old column location. If there are also new columns"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_b772dd47","line":92,"range":{"start_line":92,"start_character":29,"end_line":92,"end_character":44},"updated":"2016-08-02 18:55:23.000000000","message":"so do we need this migration_phase flag? \n\nWe\u0027re going to have 2 SQL schema counter here. 1 tracks the latest migration that has been run and the second tracks the current \"cleaned up\" operations.\n\nWhen determining what to read from and when to write multiple columns can\u0027t we use those? \n\nif migration \u003e 100: \n   new_column \u003d x\nif cleanup_migration \u003c 5: \n   old_column \u003d x","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"3bb5cc8347eb8a43040c95a6c5f6f497f19674c9","unresolved":false,"context_lines":[{"line_number":89,"context_line":"1) One node has its code upgraded to X+1 the"},{"line_number":90,"context_line":"   *\u003cservice\u003e-manage upgrade --expand* command is run which creates new"},{"line_number":91,"context_line":"   database structures, but does not migrate any data. This command also sets"},{"line_number":92,"context_line":"   the database status flag \"migration_phase\" to \"Mixed X and X+1 nodes\""},{"line_number":93,"context_line":"2) This node is restarted and we are now running a mix of X and X+1. X+1 code"},{"line_number":94,"context_line":"   knows that if we are in a mix of X and X+1 nodes, then it must still read"},{"line_number":95,"context_line":"   and write data to the old column location. If there are also new columns"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_9a1f0009","line":92,"range":{"start_line":92,"start_character":29,"end_line":92,"end_character":44},"in_reply_to":"bacf61ea_b772dd47","updated":"2016-08-02 19:13:40.000000000","message":"So the problem is that how do I know that there other nodes has seen the fact that we have run the migration? They only read this on reboot, so how do I know they have restarted?","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bfb29c39065724d326c9f88ac1d6da2d2ec71add","unresolved":false,"context_lines":[{"line_number":96,"context_line":"   that have been introduced for new functionality (that X nodes know nothing"},{"line_number":97,"context_line":"   about), then X+1 can write to these new columns."},{"line_number":98,"context_line":"3) The other nodes have their code updated one at a time"},{"line_number":99,"context_line":"4) Once all the nodes are upgraded, set the \"migration_phase\" status to"},{"line_number":100,"context_line":"   \"X+1 nodes\", using the *\u003cservice\u003e-manage upgrade --rolling-upgrade-complete*"},{"line_number":101,"context_line":"   command. Each node is then re-cycled one at a time. All code is now X+1."},{"line_number":102,"context_line":"   In this phase X+1 code will write to both old and new columns, and when"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_a861f0aa","line":99,"updated":"2016-08-02 14:40:32.000000000","message":"In step #3 we updated all nodes to X+1 and I assume that means that they were restarted too. Since step #2 doesn\u0027t write to the new column (for a changed column) then this step doesn\u0027t seem necessary. Is this to get us ready for 0-downtime?","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"f67c99b33774d958fdc4c5ea25b0ecae8e967408","unresolved":false,"context_lines":[{"line_number":96,"context_line":"   that have been introduced for new functionality (that X nodes know nothing"},{"line_number":97,"context_line":"   about), then X+1 can write to these new columns."},{"line_number":98,"context_line":"3) The other nodes have their code updated one at a time"},{"line_number":99,"context_line":"4) Once all the nodes are upgraded, set the \"migration_phase\" status to"},{"line_number":100,"context_line":"   \"X+1 nodes\", using the *\u003cservice\u003e-manage upgrade --rolling-upgrade-complete*"},{"line_number":101,"context_line":"   command. Each node is then re-cycled one at a time. All code is now X+1."},{"line_number":102,"context_line":"   In this phase X+1 code will write to both old and new columns, and when"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_215d0fed","line":99,"in_reply_to":"bacf61ea_a861f0aa","updated":"2016-08-02 17:58:50.000000000","message":"(So yes this is only really needed when we will have on-the-fly migration support), but it the step that triggers the starting of writing to both locations, and ensures we know that every node has been restarted again fore the following step when we stop writing to the old location and start migrating. Without this intermediate step you might get an independent update (by nodes either side of a restart) written to both old and new location","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":7725,"name":"David Stanek","email":"dstanek@dstanek.com","username":"dstanek"},"change_message_id":"bfb29c39065724d326c9f88ac1d6da2d2ec71add","unresolved":false,"context_lines":[{"line_number":100,"context_line":"   \"X+1 nodes\", using the *\u003cservice\u003e-manage upgrade --rolling-upgrade-complete*"},{"line_number":101,"context_line":"   command. Each node is then re-cycled one at a time. All code is now X+1."},{"line_number":102,"context_line":"   In this phase X+1 code will write to both old and new columns, and when"},{"line_number":103,"context_line":"   reading will first look in the new column, then the old column, ensuring"},{"line_number":104,"context_line":"   that X+1 nodes that have not been restarted yet still see data in the old"},{"line_number":105,"context_line":"   locations."},{"line_number":106,"context_line":"5) Once all nodes have been restarted (so are aware of the \"upgrade complete\""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_a5663927","line":103,"updated":"2016-08-02 14:40:32.000000000","message":"If the new column is NULL then we look in the old column right? Might be good to clarify that here.","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":5707,"name":"Henry Nash","email":"henryn@linux.vnet.ibm.com","username":"henry-nash"},"change_message_id":"f67c99b33774d958fdc4c5ea25b0ecae8e967408","unresolved":false,"context_lines":[{"line_number":100,"context_line":"   \"X+1 nodes\", using the *\u003cservice\u003e-manage upgrade --rolling-upgrade-complete*"},{"line_number":101,"context_line":"   command. Each node is then re-cycled one at a time. All code is now X+1."},{"line_number":102,"context_line":"   In this phase X+1 code will write to both old and new columns, and when"},{"line_number":103,"context_line":"   reading will first look in the new column, then the old column, ensuring"},{"line_number":104,"context_line":"   that X+1 nodes that have not been restarted yet still see data in the old"},{"line_number":105,"context_line":"   locations."},{"line_number":106,"context_line":"5) Once all nodes have been restarted (so are aware of the \"upgrade complete\""}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_2f3a1be8","line":103,"in_reply_to":"bacf61ea_a5663927","updated":"2016-08-02 17:58:50.000000000","message":"Yes, that is correct.","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"},{"author":{"_account_id":7191,"name":"Jamie Lennox","email":"jamielennox@gmail.com","username":"jamielennox"},"change_message_id":"6119a43e4b588896d41ccabf5662466b4f573e44","unresolved":false,"context_lines":[{"line_number":199,"context_line":""},{"line_number":200,"context_line":"We could just use totally different keystone-manage commands, and not try"},{"line_number":201,"context_line":"and make this fit the general trend for now (given that we are not providing"},{"line_number":202,"context_line":"zero down-time support), e.g."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"*keystone-manage db_sync --initial-migration*"},{"line_number":205,"context_line":"*keystone-manage db_sync --complete-migration*"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bacf61ea_57a44107","line":202,"range":{"start_line":202,"start_character":26,"end_line":202,"end_character":27},"updated":"2016-08-02 18:55:23.000000000","message":"Still agree on seperate commands. If we expect people to actually run this then it would be nice not to have to remember all the flags. \n\nOn the other hand no one is going to be doing this manually anyway so you suffer through it once and automate it.","commit_id":"b3504fd1db69044c93d08f5b3754953b1712ac8e"}]}
