)]}'
{"doc/source/upgrade.rst":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":14,"context_line":"      License for the specific language governing permissions and limitations"},{"line_number":15,"context_line":"      under the License."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Upgrades"},{"line_number":18,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"Nova aims to provide upgrades with minimal downtime."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_b9a3f79d","side":"PARENT","line":17,"updated":"2015-10-02 18:41:58.000000000","message":"I kind of wish the \u0027future\u0027 things in here that you wanted to lift out would have just been removed from this doc and moved into a new \u0027future upgrades\u0027 doc, because it\u0027s hard to tell what is new between the 2 new docs and what is in this original one.  For two reasons:\n\n1. To make sure we aren\u0027t dropping something useful from the old doc.\n\n2. So that I don\u0027t need to review something that I\u0027ve already reviewed before - but this isn\u0027t so terrible because it\u0027s generally good to review these docs again from time to time to make sure they are still valid and understandable.","commit_id":"ae55af90d64fbc0447104c1fb9949501c834085c"}],"doc/source/upgrades.rst":[{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"a840b7ccda8dcb499dad9f594ec5edef00861e3c","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Once the upgrade process has completed, its also important to note:"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"* We try to ensure your deploy in release N works with a config file you"},{"line_number":60,"context_line":"  used with N-1. Where thats not possible, the release notes cover the"},{"line_number":61,"context_line":"  extra details. Log messages are usually present to guide deployers to"},{"line_number":62,"context_line":"  update their config at their own pace, before attempting the next upgrade"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3a50d1a3_0d2633ce","line":59,"updated":"2015-07-27 14:30:59.000000000","message":"due to the first capital letter being used to denote a realease (and N is the next next one) maybe use current and previous?","commit_id":"d91fe7122e52c36396487482dc44c85a17759eda"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"2e457ff6b5cdbaf6e2207a92f0b6d2e1adc72597","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Once the upgrade process has completed, its also important to note:"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"* We try to ensure your deploy in release N works with a config file you"},{"line_number":60,"context_line":"  used with N-1. Where thats not possible, the release notes cover the"},{"line_number":61,"context_line":"  extra details. Log messages are usually present to guide deployers to"},{"line_number":62,"context_line":"  update their config at their own pace, before attempting the next upgrade"}],"source_content_type":"text/x-rst","patch_set":3,"id":"3a50d1a3_1c5a9ff2","line":59,"in_reply_to":"3a50d1a3_0d2633ce","updated":"2015-07-28 17:18:42.000000000","message":"I feel like N and N-1 is a pretty standard terminology in industry.","commit_id":"d91fe7122e52c36396487482dc44c85a17759eda"},{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"a840b7ccda8dcb499dad9f594ec5edef00861e3c","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":"nova-conductor object backports"},{"line_number":221,"context_line":"    RPC pinning ensures new services can talk to the older service\u0027s method"},{"line_number":222,"context_line":"    signatures. But many of the parameters are objects that may well be too"},{"line_number":223,"context_line":"    new for the old service to understand, so you are able to send the object"},{"line_number":224,"context_line":"    back to the nova-conductor to be downgraded to a version the older service"},{"line_number":225,"context_line":"    can understand."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3a50d1a3_7070eadc","line":222,"updated":"2015-07-27 14:30:59.000000000","message":"nit: object is an overloaded term in this space in general - might be worth pointing out that we are talking about models that can be nested structures, and that specifically NovaObject framework provides this functionality - just for the sake of clarity","commit_id":"d91fe7122e52c36396487482dc44c85a17759eda"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"ed544226848e09f7af37ea5114b7107a5c68545e","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":"nova-conductor object backports"},{"line_number":221,"context_line":"    RPC pinning ensures new services can talk to the older service\u0027s method"},{"line_number":222,"context_line":"    signatures. But many of the parameters are objects that may well be too"},{"line_number":223,"context_line":"    new for the old service to understand, so you are able to send the object"},{"line_number":224,"context_line":"    back to the nova-conductor to be downgraded to a version the older service"},{"line_number":225,"context_line":"    can understand."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3a50d1a3_00fbf179","line":222,"in_reply_to":"3a50d1a3_7070eadc","updated":"2015-07-29 09:58:26.000000000","message":"This is actually just a move of existing text, really, we can some back and fix that I think?","commit_id":"d91fe7122e52c36396487482dc44c85a17759eda"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"091f5b5f7e2c888be324c72e55999ebf22ef90f0","unresolved":false,"context_lines":[{"line_number":236,"context_line":"    of singleton services, like nova-compute, some actions could be delayed"},{"line_number":237,"context_line":"    during the restart, but ideally no actions should fail due to the restart."},{"line_number":238,"context_line":"    NOTE: while this is true for the RabbitMQ RPC backend, we need to confirm"},{"line_number":239,"context_line":"    what happens for other RPC backends."},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"API load balancer draining"},{"line_number":242,"context_line":"    When upgrading API nodes, you can make your load balancer only send new"}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba15a1d1_077cfd70","line":239,"updated":"2015-09-22 00:06:41.000000000","message":"Do we support graceful shutdown for all services currently? I thought maybe we don\u0027t for nova-api, nova-scheduler (others?). This was confusing (at least for me) about what\u0027s current vs future, since this is in the \"current\" section.","commit_id":"85553bfde21ede4a8e23cdf1aa13ae552a8a795b"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"091f5b5f7e2c888be324c72e55999ebf22ef90f0","unresolved":false,"context_lines":[{"line_number":249,"context_line":"    changes by first adding the new structures, expanding. Then you can slowly"},{"line_number":250,"context_line":"    move all the data into a new location and format. Once that is complete,"},{"line_number":251,"context_line":"    you can drop bits of the scheme that are no long needed, i.e. contract."},{"line_number":252,"context_line":"    We have plans to implement this here:"},{"line_number":253,"context_line":"    https://review.openstack.org/#/c/102545/5/specs/juno/online-schema-changes.rst,cm"},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"Online Data Migrations using objects"},{"line_number":256,"context_line":"    In Kilo we are moving all data migration into the DB objects code."}],"source_content_type":"text/x-rst","patch_set":5,"id":"ba15a1d1_47e99519","line":253,"range":{"start_line":252,"start_character":4,"end_line":253,"end_character":85},"updated":"2015-09-22 00:06:41.000000000","message":"This this already implemented (expand and contract)? If so, this makes it sound like it\u0027s future \"we have plans to implement this\"","commit_id":"85553bfde21ede4a8e23cdf1aa13ae552a8a795b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":22,"context_line":"Please note this is a developer focused document, looking at how upgrade"},{"line_number":23,"context_line":"affects the Nova code base, rather than a guide for users on how to"},{"line_number":24,"context_line":"perform an upgrade."},{"line_number":25,"context_line":"Nova\u0027s commitment to supporting upgrades, is probably the biggest single"},{"line_number":26,"context_line":"driver of Nova\u0027s architecture and code structure."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"This document discusses the current state of Upgrades. For details on the"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_f35750c8","line":25,"updated":"2015-10-02 18:41:58.000000000","message":"s/upgrades, is/upgrades is/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":57,"context_line":"Once the upgrade process has completed, its also important to note:"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"* We try to ensure your deploy in release N works with a config file you"},{"line_number":60,"context_line":"  used with N-1. Where thats not possible, the release notes cover the"},{"line_number":61,"context_line":"  extra details. Log messages are usually present to guide deployers to"},{"line_number":62,"context_line":"  update their config at their own pace, before attempting the next upgrade"},{"line_number":63,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_332ce849","line":60,"updated":"2015-10-02 18:41:58.000000000","message":"s/thats/that\u0027s/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":64,"context_line":"* We continue to evolve, and sometimes that means feature and functionality"},{"line_number":65,"context_line":"  gets changed in a way that means you have to deploy things differently."},{"line_number":66,"context_line":"  In rare cases, we may actually remove a feature."},{"line_number":67,"context_line":"  As with configuration, were possible, we ensure you are able to run using"},{"line_number":68,"context_line":"  the old system for at least one release. This also gives us time to get"},{"line_number":69,"context_line":"  feedback on any deprecation and/or migration strategy."},{"line_number":70,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_731a6019","line":67,"updated":"2015-10-02 18:41:58.000000000","message":"s/were/where/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":91,"context_line":""},{"line_number":92,"context_line":"Also note that when upgrading from one stable release N to N+1, or to"},{"line_number":93,"context_line":"some point on master, the expectation is that you have moved to the"},{"line_number":94,"context_line":"latest release of N first. This is because we may have applied fixes"},{"line_number":95,"context_line":"to N that are required for a smooth transition forward. This is the"},{"line_number":96,"context_line":"logical extension of the same requirement we have when moving between"},{"line_number":97,"context_line":"arbitrary commits on master, where we have identified some specific"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_7377c002","line":94,"updated":"2015-10-02 18:41:58.000000000","message":"Suggest using an actual example for Kilo, like 2015.1.0 (Kilo GA) you should move to the latest available, let\u0027s say 2015.1.2, before upgrading to Liberty (12.0.0).  I find version numbers in an example more helpful than keeping track of my N\u0027s","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"e63f880023007b4369e56f077b0098944f153a1d","unresolved":false,"context_lines":[{"line_number":93,"context_line":"some point on master, the expectation is that you have moved to the"},{"line_number":94,"context_line":"latest release of N first. This is because we may have applied fixes"},{"line_number":95,"context_line":"to N that are required for a smooth transition forward. This is the"},{"line_number":96,"context_line":"logical extension of the same requirement we have when moving between"},{"line_number":97,"context_line":"arbitrary commits on master, where we have identified some specific"},{"line_number":98,"context_line":"ordering required through the use of the UpgradeImpact tag."},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_7845346e","line":96,"updated":"2015-09-28 14:04:28.000000000","message":"It might also be worth noting that this is also the scenario we have for CI testing (grenade) and we always expect that scenario to work since we test it (although that\u0027s the half-truth, since we don\u0027t actually run grenade from the latest tagged/released version on N).","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":93,"context_line":"some point on master, the expectation is that you have moved to the"},{"line_number":94,"context_line":"latest release of N first. This is because we may have applied fixes"},{"line_number":95,"context_line":"to N that are required for a smooth transition forward. This is the"},{"line_number":96,"context_line":"logical extension of the same requirement we have when moving between"},{"line_number":97,"context_line":"arbitrary commits on master, where we have identified some specific"},{"line_number":98,"context_line":"ordering required through the use of the UpgradeImpact tag."},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_f3e670e7","line":96,"in_reply_to":"9a1a9d01_7845346e","updated":"2015-10-02 18:41:58.000000000","message":"So if it\u0027s the half truth, maybe it\u0027s too confusing to call out...","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":101,"context_line":"-------------"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"Our upgrade and deprecation procedures are all designed so we are able to get"},{"line_number":104,"context_line":"timely feedback for our users on the direction of the project."},{"line_number":105,"context_line":"These procedures all came about from use feedback on our previous upgrade"},{"line_number":106,"context_line":"procedures and the problems they caused."},{"line_number":107,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_135d2479","line":104,"updated":"2015-10-02 18:41:58.000000000","message":"s/for our/from our/?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":102,"context_line":""},{"line_number":103,"context_line":"Our upgrade and deprecation procedures are all designed so we are able to get"},{"line_number":104,"context_line":"timely feedback for our users on the direction of the project."},{"line_number":105,"context_line":"These procedures all came about from use feedback on our previous upgrade"},{"line_number":106,"context_line":"procedures and the problems they caused."},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"Until recently, most Nova upgrades consisted of:"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_d34facbc","line":105,"updated":"2015-10-02 18:41:58.000000000","message":"s/use feedback/user feedback/?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":110,"context_line":"* turn off API"},{"line_number":111,"context_line":"* run database migrations"},{"line_number":112,"context_line":"* deploy new code and configurations"},{"line_number":113,"context_line":"* turn all services back on, including the API"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"Users shared two key pain points:"},{"line_number":116,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_33186884","line":113,"updated":"2015-10-02 18:41:58.000000000","message":"You don\u0027t mention anything about turning services off, like conductor/scheduler/nova-network. So the steps here are very high-level, which would be fine if this were just an intro, but I don\u0027t see anything in this doc about detailed upgrade steps.  I don\u0027t think we should probably skim over that.  Maybe we want to just punt and link off to some of the upgrades documentation in the operations guide?\n\nhttp://docs.openstack.org/openstack-ops/content/ch_ops_upgrades.html","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":115,"context_line":"Users shared two key pain points:"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"* long off line DB migrations causing lots of API downtime"},{"line_number":118,"context_line":"* there is no good time to have this downtime, as  users want to"},{"line_number":119,"context_line":"  create and delete servers at any time of day or night"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"This has fueled our drive to improve Nova\u0027s upgrade-ability."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_d307ac1b","line":118,"updated":"2015-10-02 18:41:58.000000000","message":"s/as  users/as users/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":118,"context_line":"* there is no good time to have this downtime, as  users want to"},{"line_number":119,"context_line":"  create and delete servers at any time of day or night"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"This has fueled our drive to improve Nova\u0027s upgrade-ability."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"REST API Backwards Compatibility"},{"line_number":124,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_0e5adb2a","line":121,"updated":"2015-10-02 18:41:58.000000000","message":"I think upgradeability is a word.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":124,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"When looking at the Nova REST API, it is important to know how many possible"},{"line_number":127,"context_line":"versions of Nova our in production and being distributed, and how end users"},{"line_number":128,"context_line":"are exposed to those differences."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"This discussion is out of scope for this look at upgrade. For the sake of this"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_4e77739f","line":127,"updated":"2015-10-02 18:41:58.000000000","message":"s/our/are/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":128,"context_line":"are exposed to those differences."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"This discussion is out of scope for this look at upgrade. For the sake of this"},{"line_number":131,"context_line":"document, we are assuming end users will be totally unaffected by and upgrade."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"For a detailed discussion of the API and its backwards compatibility, please"},{"line_number":134,"context_line":"see: :doc:`api_microversion_history`"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_6e6277da","line":131,"updated":"2015-10-02 18:41:58.000000000","message":"s/and/an/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":180,"context_line":"    \u003chttp://specs.openstack.org/openstack/nova-specs/specs/kilo/implemented/flavor-from-sysmeta-to-blob.html\u003e`_"},{"line_number":181,"context_line":"    for an example of data migrations in the object layer."},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"*Future* work -"},{"line_number":184,"context_line":"   #. Adding plumbing to enforce that relevant data migrations are completed"},{"line_number":185,"context_line":"      before running `contract` in the expand/migrate/contract schema migration"},{"line_number":186,"context_line":"      workflow.  A potential solution would be for `contract` to run a gating"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_ce91439c","line":183,"updated":"2015-10-02 18:41:58.000000000","message":"Shouldn\u0027t this be left for the separate future upgrades doc?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":190,"context_line":"Upgrade Testing"},{"line_number":191,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"We do test upgrades using `Grenade`_."},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"We also use `Turbo-hipster`_ to do additional testing of our database"},{"line_number":196,"context_line":"migration code that is used during an upgrade."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_0e12db09","line":193,"updated":"2015-10-02 18:41:58.000000000","message":"Back to my previous point about grenade, it maybe be worthwhile that we test 2 variants of grenade:\n\n1. Full N to N+1 (stable N to trunk N+1 (master)) upgrades (all services).\n\n2. Partial n-cpu (stable N to trunk N+1 (master) except for nova-compute stays on stable N).\n\nAnd the stable N is trunk of stable/N, it\u0027s not the latest released version, so like we don\u0027t test Kilo 2015.1.1 to trunk stable/liberty.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":211,"context_line":"upgrades:"},{"line_number":212,"context_line":""},{"line_number":213,"context_line":"RPC version pinning"},{"line_number":214,"context_line":"    Through careful RPC versioning, newer nodes are able to talk to older"},{"line_number":215,"context_line":"    nova-compute nodes. When upgrading control plane nodes, we can pin them"},{"line_number":216,"context_line":"    at an older version of the compute RPC API, until all the compute nodes"},{"line_number":217,"context_line":"    are able to be upgraded."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_6ef2779c","line":214,"updated":"2015-10-02 18:41:58.000000000","message":"newer (control) nodes?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":215,"context_line":"    nova-compute nodes. When upgrading control plane nodes, we can pin them"},{"line_number":216,"context_line":"    at an older version of the compute RPC API, until all the compute nodes"},{"line_number":217,"context_line":"    are able to be upgraded."},{"line_number":218,"context_line":"    https://wiki.openstack.org/wiki/RpcMajorVersionUpdates"},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"    .. note::"},{"line_number":221,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_6e2037f8","line":218,"updated":"2015-10-02 18:41:58.000000000","message":"We should probably move this in tree at some point.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":224,"context_line":"      upgraded in lockstep so n-1 cells compatibility does not work."},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Online Configuration Reload"},{"line_number":227,"context_line":"    During the upgrade, we pin new serves at the older RPC version. When all"},{"line_number":228,"context_line":"    services are updated to use newer code, we need to unpin them so we are"},{"line_number":229,"context_line":"    able to use any new functionality."},{"line_number":230,"context_line":"    To avoid having to restart the service, using the current SIGHUP signal"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_2e79dffc","line":227,"updated":"2015-10-02 18:41:58.000000000","message":"s/serves/services/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":228,"context_line":"    services are updated to use newer code, we need to unpin them so we are"},{"line_number":229,"context_line":"    able to use any new functionality."},{"line_number":230,"context_line":"    To avoid having to restart the service, using the current SIGHUP signal"},{"line_number":231,"context_line":"    handling, or otherwise, ideally we need a way to update the currently"},{"line_number":232,"context_line":"    running process to use the latest configuration."},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"Graceful service shutdown"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_0ed41bbb","line":231,"updated":"2015-10-02 18:41:58.000000000","message":"This reads like future work - is it?  If so, we should make that clear.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":238,"context_line":"    outstanding work, then terminates. During this process, messages can be"},{"line_number":239,"context_line":"    left on the queue for when the python process starts back up."},{"line_number":240,"context_line":"    This gives us a way to shutdown a service using older code, and start"},{"line_number":241,"context_line":"    up a service using newer code with minimal impact. If its a service that"},{"line_number":242,"context_line":"    can have multiple workers, like nova-conductor, you can usually add the"},{"line_number":243,"context_line":"    new workers before the graceful shutdown of the old workers. In the case"},{"line_number":244,"context_line":"    of singleton services, like nova-compute, some actions could be delayed"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_0eebbb79","line":241,"updated":"2015-10-02 18:41:58.000000000","message":"s/its/it\u0027s/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":239,"context_line":"    left on the queue for when the python process starts back up."},{"line_number":240,"context_line":"    This gives us a way to shutdown a service using older code, and start"},{"line_number":241,"context_line":"    up a service using newer code with minimal impact. If its a service that"},{"line_number":242,"context_line":"    can have multiple workers, like nova-conductor, you can usually add the"},{"line_number":243,"context_line":"    new workers before the graceful shutdown of the old workers. In the case"},{"line_number":244,"context_line":"    of singleton services, like nova-compute, some actions could be delayed"},{"line_number":245,"context_line":"    during the restart, but ideally no actions should fail due to the restart."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_ee9e8706","line":242,"updated":"2015-10-02 18:41:58.000000000","message":"How do you add new workers to a running conductor service?  Doesn\u0027t that require either a new separate conductor service added to the pool on another node or changing the conductor.workers value in nova.conf and restarting the service?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":247,"context_line":"    what happens for other RPC backends."},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"API load balancer draining"},{"line_number":250,"context_line":"    When upgrading API nodes, you can make your load balancer only send new"},{"line_number":251,"context_line":"    connections to the newer API nodes, allowing for a seamless update of your"},{"line_number":252,"context_line":"    API nodes."},{"line_number":253,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_ae896fcc","line":250,"updated":"2015-10-02 18:41:58.000000000","message":"Are there any reference docs to point to for this?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":251,"context_line":"    connections to the newer API nodes, allowing for a seamless update of your"},{"line_number":252,"context_line":"    API nodes."},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"Expand/Contract DB Migrations"},{"line_number":255,"context_line":"    Modern databases are able to make many schema changes while you are still"},{"line_number":256,"context_line":"    writing to the database. Taking this a step further, we can make all DB"},{"line_number":257,"context_line":"    changes by first adding the new structures, expanding. Then you can slowly"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_59297b88","line":254,"updated":"2015-10-02 18:41:58.000000000","message":"This should be in the future doc right?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":258,"context_line":"    move all the data into a new location and format. Once that is complete,"},{"line_number":259,"context_line":"    you can drop bits of the scheme that are no long needed, i.e. contract."},{"line_number":260,"context_line":"    We have plans to implement this here:"},{"line_number":261,"context_line":"    https://review.openstack.org/#/c/102545/5/specs/juno/online-schema-changes.rst,cm"},{"line_number":262,"context_line":""},{"line_number":263,"context_line":"Online Data Migrations using objects"},{"line_number":264,"context_line":"    In Kilo we are moving all data migration into the DB objects code."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_99b4f335","line":261,"updated":"2015-10-02 18:41:58.000000000","message":"Rather than linking to a code review of the spec, I\u0027d point at the formatted spec doc:\n\nhttp://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/online-schema-changes.html","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":264,"context_line":"    In Kilo we are moving all data migration into the DB objects code."},{"line_number":265,"context_line":"    When trying to migrate data in the database from the old format to the"},{"line_number":266,"context_line":"    new format, this is done in the object code when reading or saving things"},{"line_number":267,"context_line":"    that are in the old format. For records that are not updated, you need to"},{"line_number":268,"context_line":"    run a background process to convert those records into the newer format."},{"line_number":269,"context_line":"    This process must be completed before you contract the database schema."},{"line_number":270,"context_line":"    We have the first example of this happening here:"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_b941b73b","line":267,"updated":"2015-10-02 18:41:58.000000000","message":"\u0027need to run a background process\u0027 here is a bit confusing, it sounds like I need to actually run some daemon that does this, but really it\u0027s just conductor doing this as it detects data that needs to be migrated, right?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":283,"context_line":"    However, most people have a data retention policy, of say 30 days or 90"},{"line_number":284,"context_line":"    days after which they will want to delete those entries. Not deleting"},{"line_number":285,"context_line":"    those entries affects DB performance as indices grow very large and data"},{"line_number":286,"context_line":"    migrations take longer as there is more data to migrate."},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"nova-conductor object backports"},{"line_number":289,"context_line":"    RPC pinning ensures new services can talk to the older service\u0027s method"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_b9dd3772","line":286,"updated":"2015-10-02 18:41:58.000000000","message":"Maybe in the future doc we might want to note that there is work going into mitaka to add an instances purge command to nova-manage.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":286,"context_line":"    migrations take longer as there is more data to migrate."},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"nova-conductor object backports"},{"line_number":289,"context_line":"    RPC pinning ensures new services can talk to the older service\u0027s method"},{"line_number":290,"context_line":"    signatures. But many of the parameters are objects that may well be too"},{"line_number":291,"context_line":"    new for the old service to understand, so you are able to send the object"},{"line_number":292,"context_line":"    back to the nova-conductor to be downgraded to a version the older service"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_79bebf00","line":289,"updated":"2015-10-02 18:41:58.000000000","message":"This section is a bit vague, I wonder if adding a link to a review as an example would be helpful.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":298,"context_line":"A key observation is that there are many more nova-compute services than"},{"line_number":299,"context_line":"any other nova service. Also, if you restart that service it will likely"},{"line_number":300,"context_line":"interrupt any in flight operations. A graceful shutdown of that service helps"},{"line_number":301,"context_line":"avoid this, but that the cost of delaying any operations that get sent to the"},{"line_number":302,"context_line":"compute node."},{"line_number":303,"context_line":""},{"line_number":304,"context_line":"With all this in mind, we allow older nova-compute nodes to co-exist with"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_99b95318","line":301,"updated":"2015-10-02 18:41:58.000000000","message":"s/that the/at the/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":306,"context_line":"This is done by pinning the RPC versions to a set the older nova-compute"},{"line_number":307,"context_line":"nodes understand. The newer nova-conductor is able to service requests"},{"line_number":308,"context_line":"for the older object versions the nova-computes expect, while letting the"},{"line_number":309,"context_line":"newer services just a newer object version."},{"line_number":310,"context_line":"The versioning rules for RPC and object versions are all in place to make"},{"line_number":311,"context_line":"this possible."},{"line_number":312,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_59617ba7","line":309,"updated":"2015-10-02 18:41:58.000000000","message":"Something missing here, \"letting the newer services just a ...\" just what?  just send?","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":313,"context_line":"Understanding UpgradeImpact"},{"line_number":314,"context_line":"----------------------------"},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"We require all commit messages that impacts upgrade to have the following"},{"line_number":317,"context_line":"tag in its git commit: UpgradeImpact"},{"line_number":318,"context_line":"This raises the obvious question around when the code you are changing"},{"line_number":319,"context_line":"actually causes and impact to the upgrade?"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_f90e4fe4","line":316,"updated":"2015-10-02 18:41:58.000000000","message":"s/all commit messages/all changes/\n\ns/impacts upgrade/impact upgrades/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":314,"context_line":"----------------------------"},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"We require all commit messages that impacts upgrade to have the following"},{"line_number":317,"context_line":"tag in its git commit: UpgradeImpact"},{"line_number":318,"context_line":"This raises the obvious question around when the code you are changing"},{"line_number":319,"context_line":"actually causes and impact to the upgrade?"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_b9f9b7ca","line":317,"updated":"2015-10-02 18:41:58.000000000","message":"s/git commit/commit message/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":316,"context_line":"We require all commit messages that impacts upgrade to have the following"},{"line_number":317,"context_line":"tag in its git commit: UpgradeImpact"},{"line_number":318,"context_line":"This raises the obvious question around when the code you are changing"},{"line_number":319,"context_line":"actually causes and impact to the upgrade?"},{"line_number":320,"context_line":""},{"line_number":321,"context_line":"We need to focus on what changes for existing deployers or end users after"},{"line_number":322,"context_line":"our change has been applied. For example, if we change the default of a"}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_19b783f6","line":319,"updated":"2015-10-02 18:41:58.000000000","message":"This sentence is a bit confusing.  Maybe:\n\n\"This raises the obvious question of what about the code you are changing actually causes an impact on upgrades.\"\n\nIt also reminds us at the end of the release to scrape the UpgradeImpact tagged commits so we make sure they are documented in the release notes.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":327,"context_line":"change might impact end users."},{"line_number":328,"context_line":""},{"line_number":329,"context_line":"Please note that we expect any Nova commit to be in production. So we need"},{"line_number":330,"context_line":"to ensure that no commit causes an upgrade regression. Should we revert at"},{"line_number":331,"context_line":"commit we need to be vocal on the ML to ensure those upgrading from"},{"line_number":332,"context_line":"arbitrary git commits are aware of changes that might affect them."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_79c8ff74","line":330,"updated":"2015-10-02 18:41:58.000000000","message":"s/revert at/revert that/","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"0d32a95af20c44fc5ef7883a6d3e5591d2ff5c8d","unresolved":false,"context_lines":[{"line_number":328,"context_line":""},{"line_number":329,"context_line":"Please note that we expect any Nova commit to be in production. So we need"},{"line_number":330,"context_line":"to ensure that no commit causes an upgrade regression. Should we revert at"},{"line_number":331,"context_line":"commit we need to be vocal on the ML to ensure those upgrading from"},{"line_number":332,"context_line":"arbitrary git commits are aware of changes that might affect them."}],"source_content_type":"text/x-rst","patch_set":7,"id":"9a1a9d01_39a127aa","line":331,"updated":"2015-10-02 18:41:58.000000000","message":"s/ML/mailing list/\n\nProbably specify openstack-dev and openstack-operators.","commit_id":"fefc5f3df84f23b7d48530c8d883f5d1fb06fb54"}]}