)]}'
{"doc/source/upgrade.rst":[{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"ff4dea601e708cd4932c16785ba5493a357d6738","unresolved":false,"context_lines":[{"line_number":51,"context_line":"  additional upgrade related load."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Rolling upgrade process"},{"line_number":55,"context_line":"\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"To reduce downtime, the services can be upgraded in a rolling fashion. It"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a629dbe_737c0473","line":54,"updated":"2016-11-07 19:57:59.000000000","message":"I was really thinking about a new version of just list list really.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"eef1273b9b1b31d7fcd11d33a82269af69680055","unresolved":false,"context_lines":[{"line_number":134,"context_line":"     upgrade is performed."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Testing Zero Downtime Upgrade Process"},{"line_number":138,"context_line":"-------------------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a629dbe_77b843e6","line":137,"range":{"start_line":137,"start_character":0,"end_line":137,"end_character":37},"updated":"2016-11-08 16:21:00.000000000","message":"Change this to \"Zero Downtime Upgrades\"","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"eef1273b9b1b31d7fcd11d33a82269af69680055","unresolved":false,"context_lines":[{"line_number":138,"context_line":"-------------------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"For zero downtime upgrade, nova-api service should be upgraded one at a time."},{"line_number":142,"context_line":"This allows one api service to serve the requests while we upgrade the other."},{"line_number":143,"context_line":"This results in a condition where both old (N) and new (N+1) nova-api services"},{"line_number":144,"context_line":"co-exist for a certain time period."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"Testing the upgrade process is done on Mitaka to Newton."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"Environment:"},{"line_number":149,"context_line":"\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a629dbe_37a5ebb4","line":146,"range":{"start_line":141,"start_character":0,"end_line":146,"end_character":56},"updated":"2016-11-08 16:21:00.000000000","message":"In this section explain what zero downtime upgrade means. Definition. Also add a bit about how is this different than rolling upgrades.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"eef1273b9b1b31d7fcd11d33a82269af69680055","unresolved":false,"context_lines":[{"line_number":144,"context_line":"co-exist for a certain time period."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"Testing the upgrade process is done on Mitaka to Newton."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"Environment:"},{"line_number":149,"context_line":"\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027\u0027"},{"line_number":150,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a629dbe_77020371","line":147,"updated":"2016-11-08 16:21:00.000000000","message":"Now mention all the steps operator should use for zero downtime upgrades. Basically what you have in your \"Upgrade section below\"\nCall out that assumption is multiple API nodes running behind load balancer.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"58085ec3989ae7a2881b6300e976ba394bed53b9","unresolved":false,"context_lines":[{"line_number":150,"context_line":""},{"line_number":151,"context_line":"* Multinode environment with 2 nodes:"},{"line_number":152,"context_line":"        * controller  - All services (Mitaka)."},{"line_number":153,"context_line":"        * compute-api - Only n-cpu and n-api services (Mitaka)."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"* Setup Highly available load balancer (HAProxy) on top of the n-api services."},{"line_number":156,"context_line":"  See Instructions to setup HAProxy below."}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba5da102_7e8762d7","line":153,"updated":"2016-11-05 22:09:25.000000000","message":"Specifying this kind of setup is not appropriate for this documentation, IMHO. Not only does it make the documentation look like Nova is tested only on 2-node environments, but this type of setup (with n-cpu and n-api on the same node) simply doesn\u0027t occur in production.\n\nThe rest of the documentation in this upgrade section of the devref only mentions \"controller nodes\" and \"compute nodes\" (or \"nova computes\"). Let\u0027s stick to that.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"ff4dea601e708cd4932c16785ba5493a357d6738","unresolved":false,"context_lines":[{"line_number":150,"context_line":""},{"line_number":151,"context_line":"* Multinode environment with 2 nodes:"},{"line_number":152,"context_line":"        * controller  - All services (Mitaka)."},{"line_number":153,"context_line":"        * compute-api - Only n-cpu and n-api services (Mitaka)."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"* Setup Highly available load balancer (HAProxy) on top of the n-api services."},{"line_number":156,"context_line":"  See Instructions to setup HAProxy below."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a629dbe_53e2a0d6","line":153,"in_reply_to":"ba5da102_7e8762d7","updated":"2016-11-07 19:57:59.000000000","message":"This is the proposed test environment for the gate.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"58085ec3989ae7a2881b6300e976ba394bed53b9","unresolved":false,"context_lines":[{"line_number":153,"context_line":"        * compute-api - Only n-cpu and n-api services (Mitaka)."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"* Setup Highly available load balancer (HAProxy) on top of the n-api services."},{"line_number":156,"context_line":"  See Instructions to setup HAProxy below."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"* Add a new region to keystone with nova api endpoints pointing to haproxy"},{"line_number":159,"context_line":"  virtual IP."}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba5da102_9e6c3638","line":156,"updated":"2016-11-05 22:09:25.000000000","message":"This is way too specific of a deployment setup to put in the developer reference, IMHO. This type of opinionated deployment advice belongs in either a deployment tool\u0027s documentation or in the OpenStack Administrator\u0027s Manual as an example page.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"ff4dea601e708cd4932c16785ba5493a357d6738","unresolved":false,"context_lines":[{"line_number":153,"context_line":"        * compute-api - Only n-cpu and n-api services (Mitaka)."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"* Setup Highly available load balancer (HAProxy) on top of the n-api services."},{"line_number":156,"context_line":"  See Instructions to setup HAProxy below."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"* Add a new region to keystone with nova api endpoints pointing to haproxy"},{"line_number":159,"context_line":"  virtual IP."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a629dbe_b3ca1c4e","line":156,"in_reply_to":"ba5da102_9e6c3638","updated":"2016-11-07 19:57:59.000000000","message":"So I thought that, but this is really useful for reproducing issues when draining and transferring the API.\n\nWe have two choices for switching from old API to new API, we change the service catalog and hack it, or we update the config of a loadbalancer.\n\nThats because devstack only installs one service at once.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"58085ec3989ae7a2881b6300e976ba394bed53b9","unresolved":false,"context_lines":[{"line_number":156,"context_line":"  See Instructions to setup HAProxy below."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"* Add a new region to keystone with nova api endpoints pointing to haproxy"},{"line_number":159,"context_line":"  virtual IP."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"* Change osapi_compute_listen option in nova.conf to IP address on each node"},{"line_number":162,"context_line":"  to avoid issues with minimum version during the upgrade process."}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba5da102_be69fa27","line":159,"updated":"2016-11-05 22:09:25.000000000","message":"Err, no. New regions in Keystone are definitely not intended to be used in upgrade scenarios.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"ff4dea601e708cd4932c16785ba5493a357d6738","unresolved":false,"context_lines":[{"line_number":156,"context_line":"  See Instructions to setup HAProxy below."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"* Add a new region to keystone with nova api endpoints pointing to haproxy"},{"line_number":159,"context_line":"  virtual IP."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"* Change osapi_compute_listen option in nova.conf to IP address on each node"},{"line_number":162,"context_line":"  to avoid issues with minimum version during the upgrade process."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9a629dbe_d3cb1071","line":159,"in_reply_to":"ba5da102_be69fa27","updated":"2016-11-07 19:57:59.000000000","message":"Agreed, what we would want here is the service catalog updating to point to the HA proxy.","commit_id":"a1eeb7286dc471f736d231c817a6319e63b07b7a"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"3db0fddf37ca8328b33fc761fa8aa1dd15568b62","unresolved":false,"context_lines":[{"line_number":138,"context_line":"-----------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"The rolling-upgrade process allows for minimal downtime of the control plane"},{"line_number":142,"context_line":"during the upgrade process, since  this process  allows restarting all"},{"line_number":143,"context_line":"API services together which results in minimal downtime of API."},{"line_number":144,"context_line":"Zero down time upgrade allows services to completely eliminate"},{"line_number":145,"context_line":"API downtime of the control plane during upgrade."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Here we introduce the process needed for a no API downtime upgrade of nova."},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a629dbe_fe948c37","line":145,"range":{"start_line":141,"start_character":0,"end_line":145,"end_character":49},"updated":"2016-11-11 20:08:59.000000000","message":"I will reword it to something like\n\"In zero down time upgrade process no disruption to nova API service is incurred with comparison to rolling upgrade process where minimal downtime of control services is accepted.\nIn zero downtime upgrade process API services are upgraded in the last.\"","commit_id":"e24b25f004010bd45ada43f8b5051af4f462e030"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"3db0fddf37ca8328b33fc761fa8aa1dd15568b62","unresolved":false,"context_lines":[{"line_number":144,"context_line":"Zero down time upgrade allows services to completely eliminate"},{"line_number":145,"context_line":"API downtime of the control plane during upgrade."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Here we introduce the process needed for a no API downtime upgrade of nova."},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Note: This process is experimental and not completely tested. This is under"},{"line_number":150,"context_line":"the assumption that multiple API nodes are running behind load balancer."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a629dbe_3ef524ec","line":147,"range":{"start_line":147,"start_character":0,"end_line":147,"end_character":75},"updated":"2016-11-11 20:08:59.000000000","message":"This is redundant so delete this line.","commit_id":"e24b25f004010bd45ada43f8b5051af4f462e030"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"3db0fddf37ca8328b33fc761fa8aa1dd15568b62","unresolved":false,"context_lines":[{"line_number":149,"context_line":"Note: This process is experimental and not completely tested. This is under"},{"line_number":150,"context_line":"the assumption that multiple API nodes are running behind load balancer."},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"Zero downtime upgrade requires upgrading nova-api services one at a time."},{"line_number":153,"context_line":"This results in both old (N) and new (N+1) nova-api services running"},{"line_number":154,"context_line":"concurrently. So that there is always an API service available to process"},{"line_number":155,"context_line":"requests while we upgrade the other eliminating the API downtime."}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a629dbe_7ecf9c55","line":152,"range":{"start_line":152,"start_character":59,"end_line":152,"end_character":72},"updated":"2016-11-11 20:08:59.000000000","message":"in batches, making sure their are always some API services running during the process.","commit_id":"e24b25f004010bd45ada43f8b5051af4f462e030"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"3db0fddf37ca8328b33fc761fa8aa1dd15568b62","unresolved":false,"context_lines":[{"line_number":151,"context_line":""},{"line_number":152,"context_line":"Zero downtime upgrade requires upgrading nova-api services one at a time."},{"line_number":153,"context_line":"This results in both old (N) and new (N+1) nova-api services running"},{"line_number":154,"context_line":"concurrently. So that there is always an API service available to process"},{"line_number":155,"context_line":"requests while we upgrade the other eliminating the API downtime."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a629dbe_3eb284e5","line":154,"updated":"2016-11-11 20:08:59.000000000","message":"at some point of process.","commit_id":"e24b25f004010bd45ada43f8b5051af4f462e030"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"3db0fddf37ca8328b33fc761fa8aa1dd15568b62","unresolved":false,"context_lines":[{"line_number":151,"context_line":""},{"line_number":152,"context_line":"Zero downtime upgrade requires upgrading nova-api services one at a time."},{"line_number":153,"context_line":"This results in both old (N) and new (N+1) nova-api services running"},{"line_number":154,"context_line":"concurrently. So that there is always an API service available to process"},{"line_number":155,"context_line":"requests while we upgrade the other eliminating the API downtime."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"#. Before maintenance window:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"9a629dbe_7ef6bc8f","line":155,"range":{"start_line":154,"start_character":14,"end_line":155,"end_character":64},"updated":"2016-11-11 20:08:59.000000000","message":"This process might degrade API response time, but will eliminate the API downtime.","commit_id":"e24b25f004010bd45ada43f8b5051af4f462e030"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"53f6e7522eb26f71bc55d97203d3acc174f6dbb9","unresolved":false,"context_lines":[{"line_number":142,"context_line":"with comparison to rolling upgrade process where minimal downtime of control"},{"line_number":143,"context_line":"services is accepted."},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"In zero downtime upgrade process API services are upgraded in the last."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Note: This process is experimental and not completely tested. This is under"},{"line_number":148,"context_line":"the assumption that multiple API nodes are running behind load balancer."}],"source_content_type":"text/x-rst","patch_set":6,"id":"7a77a97e_d625e45c","line":145,"updated":"2016-11-14 18:11:46.000000000","message":"This sentence reads funny, perhaps \"API services are upgraded last.\"","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"f773baa1048b1485d2c6eb459d2fda980f99fd0d","unresolved":false,"context_lines":[{"line_number":142,"context_line":"with comparison to rolling upgrade process where minimal downtime of control"},{"line_number":143,"context_line":"services is accepted."},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"In zero downtime upgrade process API services are upgraded in the last."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Note: This process is experimental and not completely tested. This is under"},{"line_number":148,"context_line":"the assumption that multiple API nodes are running behind load balancer."}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a74a57a_aa944937","line":145,"in_reply_to":"7a77a97e_d625e45c","updated":"2016-11-23 21:25:27.000000000","message":"Done","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"53f6e7522eb26f71bc55d97203d3acc174f6dbb9","unresolved":false,"context_lines":[{"line_number":156,"context_line":""},{"line_number":157,"context_line":"#. Before maintenance window:"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"   * Change osapi_compute_listen option in nova.conf to IP address on each"},{"line_number":160,"context_line":"     node to avoid issues with minimum version during the upgrade process."},{"line_number":161,"context_line":"     Currently there is a bug filed on this issue:"},{"line_number":162,"context_line":"     ``https://bugs.launchpad.net/nova/+bug/1631430``"}],"source_content_type":"text/x-rst","patch_set":6,"id":"7a77a97e_f6af4092","line":159,"updated":"2016-11-14 18:11:46.000000000","message":"\"to a unique reachable IP address for each node\"","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"f773baa1048b1485d2c6eb459d2fda980f99fd0d","unresolved":false,"context_lines":[{"line_number":156,"context_line":""},{"line_number":157,"context_line":"#. Before maintenance window:"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"   * Change osapi_compute_listen option in nova.conf to IP address on each"},{"line_number":160,"context_line":"     node to avoid issues with minimum version during the upgrade process."},{"line_number":161,"context_line":"     Currently there is a bug filed on this issue:"},{"line_number":162,"context_line":"     ``https://bugs.launchpad.net/nova/+bug/1631430``"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a74a57a_eabba170","line":159,"in_reply_to":"7a77a97e_f6af4092","updated":"2016-11-23 21:25:27.000000000","message":"Done","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"53f6e7522eb26f71bc55d97203d3acc174f6dbb9","unresolved":false,"context_lines":[{"line_number":171,"context_line":"   * Start all other services in the controller node with"},{"line_number":172,"context_line":"     ``[upgrade_levels]compute \u003d auto`` in nova.conf. It is safest to start with"},{"line_number":173,"context_line":"     nova-conductor."},{"line_number":174,"context_line":"   * In small batches gracefully shutdown nova-compute, then start the compute"},{"line_number":175,"context_line":"     service with new version of the code. Note this is done in batches so only"},{"line_number":176,"context_line":"     a few compute nodes will have any delayed API actions."},{"line_number":177,"context_line":"   * Drain connections and gracefully shutdown n-api service. This allows n-api"}],"source_content_type":"text/x-rst","patch_set":6,"id":"7a77a97e_1654bc52","line":174,"updated":"2016-11-14 18:11:46.000000000","message":"In the rolling upgrade document, nova-compute is done last. Why would the order change with a live upgrade method, so that nova-compute is done before nova-api?","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"e6b33dbe96eba59579c2e7c08b4c876d5bc2c5b6","unresolved":false,"context_lines":[{"line_number":171,"context_line":"   * Start all other services in the controller node with"},{"line_number":172,"context_line":"     ``[upgrade_levels]compute \u003d auto`` in nova.conf. It is safest to start with"},{"line_number":173,"context_line":"     nova-conductor."},{"line_number":174,"context_line":"   * In small batches gracefully shutdown nova-compute, then start the compute"},{"line_number":175,"context_line":"     service with new version of the code. Note this is done in batches so only"},{"line_number":176,"context_line":"     a few compute nodes will have any delayed API actions."},{"line_number":177,"context_line":"   * Drain connections and gracefully shutdown n-api service. This allows n-api"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1a6eadb0_c492119f","line":174,"in_reply_to":"5a74a57a_7b4da571","updated":"2016-12-14 18:11:16.000000000","message":"There is a reason to reverse: the idea is we don\u0027t want to expose new API features until all the compute nodes are upgraded to deliver them.\n\nTo have a new API, you need an upgraded conductor node in all cells. It seems better if we upgrade each cell one by one, as a unit, then finally complete the API upgrade to expose the new features. The alternative is upgrade all cell conductors, then API, then all cell computes, so you visit every cell twice, which seems a bit odd. We are not quite there yet with that story.\n\nIt does mean no online data migrations are possible until all the API nodes are upgraded. Thats useful for the rollback story, but thats for another document.","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"f773baa1048b1485d2c6eb459d2fda980f99fd0d","unresolved":false,"context_lines":[{"line_number":171,"context_line":"   * Start all other services in the controller node with"},{"line_number":172,"context_line":"     ``[upgrade_levels]compute \u003d auto`` in nova.conf. It is safest to start with"},{"line_number":173,"context_line":"     nova-conductor."},{"line_number":174,"context_line":"   * In small batches gracefully shutdown nova-compute, then start the compute"},{"line_number":175,"context_line":"     service with new version of the code. Note this is done in batches so only"},{"line_number":176,"context_line":"     a few compute nodes will have any delayed API actions."},{"line_number":177,"context_line":"   * Drain connections and gracefully shutdown n-api service. This allows n-api"}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a74a57a_7b4da571","line":174,"in_reply_to":"7a77a97e_1654bc52","updated":"2016-11-23 21:25:27.000000000","message":"This is the process we followed for testing this upgrade process. I think doing n-api last is helpful in context of cells environment. To answer your question exactly we need to test this approach more precisely and come up with scenario\u0027s on where it breaks if we don\u0027t upgrade n-api last.\n\nIn this upgrade process we mainly tested if we can run with two api nodes. While one being upgraded the other api node is handling the requests.","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"53f6e7522eb26f71bc55d97203d3acc174f6dbb9","unresolved":false,"context_lines":[{"line_number":178,"context_line":"     to complete the pending requests. Restart n-api with new version of the"},{"line_number":179,"context_line":"     code. Note this is done only on one n-api service at a time so that"},{"line_number":180,"context_line":"     other api node can serve the requests while this is being upgraded."},{"line_number":181,"context_line":"   * Once n-api is upgraded, enable the service on load balancer. At this point"},{"line_number":182,"context_line":"     both old and new api services run concurrently."},{"line_number":183,"context_line":"   * Repeat the same process for n-api services on other nodes."},{"line_number":184,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"7a77a97e_36c7f846","line":181,"updated":"2016-11-14 18:11:46.000000000","message":"This mentions enabling the service on the load balancer, but no previous step mentions disabling it on the load balancer. I assume that the \"Drain connections\" step is supposed to target a load balancer, but that part isn\u0027t made clear.","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"f773baa1048b1485d2c6eb459d2fda980f99fd0d","unresolved":false,"context_lines":[{"line_number":178,"context_line":"     to complete the pending requests. Restart n-api with new version of the"},{"line_number":179,"context_line":"     code. Note this is done only on one n-api service at a time so that"},{"line_number":180,"context_line":"     other api node can serve the requests while this is being upgraded."},{"line_number":181,"context_line":"   * Once n-api is upgraded, enable the service on load balancer. At this point"},{"line_number":182,"context_line":"     both old and new api services run concurrently."},{"line_number":183,"context_line":"   * Repeat the same process for n-api services on other nodes."},{"line_number":184,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5a74a57a_104010a5","line":181,"in_reply_to":"7a77a97e_36c7f846","updated":"2016-11-23 21:25:27.000000000","message":"Done","commit_id":"362c1e27d44859df4e98ea5ead6eed76d09ff1bd"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"743eb952ac6b86fc9167668915c6b8e52d57c1ec","unresolved":false,"context_lines":[{"line_number":159,"context_line":"   * Change osapi_compute_listen option in nova.conf to a unique reachable"},{"line_number":160,"context_line":"     IP address for each node to avoid issues with minimum version during the"},{"line_number":161,"context_line":"     upgrade process. Currently there is a bug filed on this issue:"},{"line_number":162,"context_line":"     ``https://bugs.launchpad.net/nova/+bug/1631430``"},{"line_number":163,"context_line":"   * Start the process with the controller node. Follow the same steps as in"},{"line_number":164,"context_line":"     Rolling Upgrade process to install new code and sync the db for schema"},{"line_number":165,"context_line":"     changes."}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a6eadb0_5aec4c8f","line":162,"range":{"start_line":162,"start_character":5,"end_line":162,"end_character":53},"updated":"2016-12-13 21:47:31.000000000","message":"nit: Better to have this as a hyper link for \"Bug 1631430\"","commit_id":"e444b9c18b3e44d09ef67a6b40da530b2746a0a8"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"743eb952ac6b86fc9167668915c6b8e52d57c1ec","unresolved":false,"context_lines":[{"line_number":173,"context_line":"     nova-conductor."},{"line_number":174,"context_line":"   * In small batches gracefully shutdown nova-compute, then start the compute"},{"line_number":175,"context_line":"     service with new version of the code. Note this is done in batches so only"},{"line_number":176,"context_line":"     a few compute nodes will have any delayed API actions."},{"line_number":177,"context_line":"   * Drain connections and gracefully shutdown n-api service. This step allows"},{"line_number":178,"context_line":"     n-api to complete the pending requests and disables it on load balancer."},{"line_number":179,"context_line":"     Restart n-api with new version of the code. Note this is done only on one"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a6eadb0_3a45d06f","line":176,"range":{"start_line":176,"start_character":35,"end_line":176,"end_character":58},"updated":"2016-12-13 21:47:31.000000000","message":"delayed response?","commit_id":"e444b9c18b3e44d09ef67a6b40da530b2746a0a8"},{"author":{"_account_id":19590,"name":"Sarafraj Singh","email":"Sarafraj.Singh@intel.com","username":"sarafrajsingh"},"change_message_id":"743eb952ac6b86fc9167668915c6b8e52d57c1ec","unresolved":false,"context_lines":[{"line_number":174,"context_line":"   * In small batches gracefully shutdown nova-compute, then start the compute"},{"line_number":175,"context_line":"     service with new version of the code. Note this is done in batches so only"},{"line_number":176,"context_line":"     a few compute nodes will have any delayed API actions."},{"line_number":177,"context_line":"   * Drain connections and gracefully shutdown n-api service. This step allows"},{"line_number":178,"context_line":"     n-api to complete the pending requests and disables it on load balancer."},{"line_number":179,"context_line":"     Restart n-api with new version of the code. Note this is done only on one"},{"line_number":180,"context_line":"     n-api service at a time so that other api node can serve the requests"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a6eadb0_3d0a2a22","line":177,"range":{"start_line":177,"start_character":47,"end_line":177,"end_character":60},"updated":"2016-12-13 21:47:31.000000000","message":"one n-api service at a time","commit_id":"e444b9c18b3e44d09ef67a6b40da530b2746a0a8"},{"author":{"_account_id":4,"name":"Dolph Mathews","email":"dolph.mathews@gmail.com","username":"dolph"},"change_message_id":"9710630ea16d7a64281c23170020ce18ef29a8a0","unresolved":false,"context_lines":[{"line_number":134,"context_line":"     upgrade is performed."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Zero Downtime Upgrade Process"},{"line_number":138,"context_line":"-----------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_68d495bc","line":137,"updated":"2017-01-05 21:51:17.000000000","message":"This section obviously shares some redundancies with the section above (rolling upgrades). Is there any reason why it shouldn\u0027t just be documented inline with the rolling upgrade process? As a deployer, I see little reason why I\u0027d want to follow minimal downtime, rolling upgrade guide, when I can have zero downtime, rolling upgrades instead, for minimal extra complexity.","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"676bfc19ade1817c6f4bf1550e5e5dca04657202","unresolved":false,"context_lines":[{"line_number":134,"context_line":"     upgrade is performed."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Zero Downtime Upgrade Process"},{"line_number":138,"context_line":"-----------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_d88fa8ca","line":137,"in_reply_to":"ba5201f7_68d495bc","updated":"2017-01-09 19:58:07.000000000","message":"I agree with you Dolph. But right now this process is not fully tested, we added the steps to get initial feedback on the process. This is added in a separate section as it would be easier to review for the developers. Once the process get stabilized then as you suggested, we will mix these two sections which would be easier for the operators.","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":4,"name":"Dolph Mathews","email":"dolph.mathews@gmail.com","username":"dolph"},"change_message_id":"9710630ea16d7a64281c23170020ce18ef29a8a0","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"In zero downtime upgrade process API services are upgraded last."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Note: This process is experimental and not completely tested. This is under"},{"line_number":148,"context_line":"the assumption that multiple API nodes are running behind load balancer."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The basic idea of the zero downtime upgrade process is to have the connections"}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_887e1917","line":147,"updated":"2017-01-05 21:51:17.000000000","message":".\n\n  .. note::\n\n    This process...","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"676bfc19ade1817c6f4bf1550e5e5dca04657202","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"In zero downtime upgrade process API services are upgraded last."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Note: This process is experimental and not completely tested. This is under"},{"line_number":148,"context_line":"the assumption that multiple API nodes are running behind load balancer."},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"The basic idea of the zero downtime upgrade process is to have the connections"}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_98ac4037","line":147,"in_reply_to":"ba5201f7_887e1917","updated":"2017-01-09 19:58:07.000000000","message":"Done","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":4,"name":"Dolph Mathews","email":"dolph.mathews@gmail.com","username":"dolph"},"change_message_id":"9710630ea16d7a64281c23170020ce18ef29a8a0","unresolved":false,"context_lines":[{"line_number":161,"context_line":"   * Change osapi_compute_listen option in nova.conf to a unique reachable"},{"line_number":162,"context_line":"     IP address for each node to avoid issues with minimum version during the"},{"line_number":163,"context_line":"     upgrade process. Currently there is a bug filed on this issue:"},{"line_number":164,"context_line":"     ``https://bugs.launchpad.net/nova/+bug/1631430``"},{"line_number":165,"context_line":"   * Start the process with the controller node. Follow the same steps as in"},{"line_number":166,"context_line":"     Rolling Upgrade process to install new code and sync the db for schema"},{"line_number":167,"context_line":"     changes."}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_a853dd69","line":164,"updated":"2017-01-05 21:51:17.000000000","message":"I\u0027d suggest making this an embedded URL, rather than an arbitrary literal:\n\n  http://docutils.sourceforge.net/docs/user/rst/quickref.html#external-hyperlink-targets","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"676bfc19ade1817c6f4bf1550e5e5dca04657202","unresolved":false,"context_lines":[{"line_number":161,"context_line":"   * Change osapi_compute_listen option in nova.conf to a unique reachable"},{"line_number":162,"context_line":"     IP address for each node to avoid issues with minimum version during the"},{"line_number":163,"context_line":"     upgrade process. Currently there is a bug filed on this issue:"},{"line_number":164,"context_line":"     ``https://bugs.launchpad.net/nova/+bug/1631430``"},{"line_number":165,"context_line":"   * Start the process with the controller node. Follow the same steps as in"},{"line_number":166,"context_line":"     Rolling Upgrade process to install new code and sync the db for schema"},{"line_number":167,"context_line":"     changes."}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_d8a6c852","line":164,"in_reply_to":"ba5201f7_a853dd69","updated":"2017-01-09 19:58:07.000000000","message":"Done","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":4,"name":"Dolph Mathews","email":"dolph.mathews@gmail.com","username":"dolph"},"change_message_id":"9710630ea16d7a64281c23170020ce18ef29a8a0","unresolved":false,"context_lines":[{"line_number":192,"context_line":"     once all the services are upgraded, so all the services clear any cached"},{"line_number":193,"context_line":"     service version data."},{"line_number":194,"context_line":"   * Complete all online data migrations using:"},{"line_number":195,"context_line":"     ``nova-manage db online_data_migrations --limit\u003cnumber\u003e``"},{"line_number":196,"context_line":""},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"Current Database Upgrade Types"}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_489d1109","line":195,"updated":"2017-01-05 21:51:17.000000000","message":"Missing a space after --limit","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":20217,"name":"Anusha Unnam","email":"anusha.unnam25@gmail.com","username":"aunnam"},"change_message_id":"676bfc19ade1817c6f4bf1550e5e5dca04657202","unresolved":false,"context_lines":[{"line_number":192,"context_line":"     once all the services are upgraded, so all the services clear any cached"},{"line_number":193,"context_line":"     service version data."},{"line_number":194,"context_line":"   * Complete all online data migrations using:"},{"line_number":195,"context_line":"     ``nova-manage db online_data_migrations --limit\u003cnumber\u003e``"},{"line_number":196,"context_line":""},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"Current Database Upgrade Types"}],"source_content_type":"text/x-rst","patch_set":8,"id":"ba5201f7_386c3408","line":195,"in_reply_to":"ba5201f7_489d1109","updated":"2017-01-09 19:58:07.000000000","message":"Done","commit_id":"1f71576e605f5bf443a8aacde61ad28a952801a5"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"cd303dcfe2fc80f3dc426c21295a6311453844ab","unresolved":false,"context_lines":[{"line_number":137,"context_line":"Zero Downtime Upgrade Process"},{"line_number":138,"context_line":"-----------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"In zero down time upgrade process no disruption to nova API service is incurred"},{"line_number":142,"context_line":"with comparison to rolling upgrade process where minimal downtime of control"},{"line_number":143,"context_line":"services is accepted."}],"source_content_type":"text/x-rst","patch_set":9,"id":"3a461143_212a7f90","line":140,"updated":"2017-01-31 16:31:05.000000000","message":"Nit: extra blank line here.","commit_id":"ffe1b18fc6f4c804d047c51ccc0edbdea6ce44d4"},{"author":{"_account_id":4,"name":"Dolph Mathews","email":"dolph.mathews@gmail.com","username":"dolph"},"change_message_id":"68b6faa77a7fe4e11d5b93a6c976d0240f876cd4","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"In zero downtime upgrade process API services are upgraded last."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":".. note::"},{"line_number":148,"context_line":"  This process is experimental and not completely tested. This is under"},{"line_number":149,"context_line":"  the assumption that multiple API nodes are running behind load balancer."},{"line_number":150,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a430d35_6e81ddfa","line":147,"updated":"2017-02-08 16:44:36.000000000","message":"RST syntax -- this needs a trailing blank line to be rendered as a note.","commit_id":"ffe1b18fc6f4c804d047c51ccc0edbdea6ce44d4"}]}