)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"change_message_id":"fc5c8ecc9523492d5cc800dd31137c580f6dcadc","unresolved":false,"context_lines":[{"line_number":5,"context_line":"CommitDate: 2020-03-17 10:11:27 +0100"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"How to upgrade from xenial to bionic on rocky"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"This is my recipe for upgrading from xenial to bionic on rocky by"},{"line_number":10,"context_line":"reinstalling hosts one at a time. Possibly useful for others."},{"line_number":11,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"df33271e_3032f6a8","line":8,"updated":"2020-03-20 19:12:17.000000000","message":"Would be good to note for stable policy purposes that this is being proposed directly to only stable/rocky and the reason why.","commit_id":"557bf42d60815eb6ca93fbbef24b5e11c083015c"}],"doc/source/admin/upgrades/distribution-upgrades.rst":[{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"65ca0fbcf999156085223e42566640971176fbd1","unresolved":false,"context_lines":[{"line_number":22,"context_line":"Checklist"},{"line_number":23,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"#. Identifying your \"primary\" haproxy/galera/rabbitmq infrastructure_host"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"   This will be the last box you\u0027ll want to reinstall."},{"line_number":28,"context_line":"   Log into your infrastructure nodes"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_91d5761e","line":25,"range":{"start_line":25,"start_character":45,"end_line":25,"end_character":53},"updated":"2020-03-10 13:01:22.000000000","message":"I think there\u0027s no \"primary\" rabbitmq host, since all of them are used at the same time and do not use haproxy","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"4927e93971c18ea6de7b310a7ae736f10d77383a","unresolved":false,"context_lines":[{"line_number":22,"context_line":"Checklist"},{"line_number":23,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"#. Identifying your \"primary\" haproxy/galera/rabbitmq infrastructure_host"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"   This will be the last box you\u0027ll want to reinstall."},{"line_number":28,"context_line":"   Log into your infrastructure nodes"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_b4d8a052","line":25,"range":{"start_line":25,"start_character":45,"end_line":25,"end_character":53},"in_reply_to":"1fa4df85_91d5761e","updated":"2020-03-17 09:13:18.000000000","message":"There\u0027s no \"primary\" in that sense. There is however one of them that gets preferred for initial setup by the playbooks. And in all my experiments on testing this upgrade, I ended up with a rabbitmq in a cluster of it\u0027s own instead of joining the other two. Even when running with -e rabbitmq_upgrade\u003dtrue.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"65ca0fbcf999156085223e42566640971176fbd1","unresolved":false,"context_lines":[{"line_number":41,"context_line":"   and can visit https://admin:password@external_lb_vip_address:1936/ and read"},{"line_number":42,"context_line":"   \"Statistics Report for pid # on infrastructure_host"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"#. Disable your non-primary repo_all-back"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"   If you\u0027ve enabled haproxy_stats as described above, you can visit"},{"line_number":47,"context_line":"   https://admin:password@external_lb_vip_address:1936/ and select them and"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_d1c76e63","line":44,"range":{"start_line":44,"start_character":3,"end_line":44,"end_character":41},"updated":"2020-03-10 13:01:22.000000000","message":"I\u0027d say add instruction how to set host to maint through socket.\nhttps://cbonte.github.io/haproxy-dconv/1.7/snapshot/management.html#9.3-disable%20server","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"4927e93971c18ea6de7b310a7ae736f10d77383a","unresolved":false,"context_lines":[{"line_number":41,"context_line":"   and can visit https://admin:password@external_lb_vip_address:1936/ and read"},{"line_number":42,"context_line":"   \"Statistics Report for pid # on infrastructure_host"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"#. Disable your non-primary repo_all-back"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"   If you\u0027ve enabled haproxy_stats as described above, you can visit"},{"line_number":47,"context_line":"   https://admin:password@external_lb_vip_address:1936/ and select them and"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_f7ca3644","line":44,"range":{"start_line":44,"start_character":3,"end_line":44,"end_character":41},"in_reply_to":"1fa4df85_d1c76e63","updated":"2020-03-17 09:13:18.000000000","message":"Done.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"65ca0fbcf999156085223e42566640971176fbd1","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"   .. code-block:: yaml"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"      repo_build_global_links_path: \"{{ repo_build_base_path }}/links/{{ repo_build_os_distro_version }}\""},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"#. ceph luminous for bionic"},{"line_number":60,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_1197464a","line":57,"range":{"start_line":57,"start_character":6,"end_line":57,"end_character":34},"updated":"2020-03-10 13:01:22.000000000","message":"wouldn\u0027t it disable package build for xenial in case if some hosts will continue to work on it?","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"4927e93971c18ea6de7b310a7ae736f10d77383a","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"   .. code-block:: yaml"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"      repo_build_global_links_path: \"{{ repo_build_base_path }}/links/{{ repo_build_os_distro_version }}\""},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"#. ceph luminous for bionic"},{"line_number":60,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_57e84e7a","line":57,"range":{"start_line":57,"start_character":6,"end_line":57,"end_character":34},"in_reply_to":"1fa4df85_1197464a","updated":"2020-03-17 09:13:18.000000000","message":"The links directory is only used by the repo_build role to create wheels in the container they\u0027re in. But there is a sync process in repo_package_sync, that ends up copying these ubuntu-18 links to your \"primary\" ubuntu-16 repo_container. Later your ubuntu-16 repo_container uses these links (now pointing to ubuntu-18 builds) in it\u0027s repo_build process, and you end up deploying a keystone complaining about some crypto lib.\n\nI think;\n1) We shouldn\u0027t sync that dir. See https://review.opendev.org/#/c/712784/\n2) We can prevent this nasty mix-up by separating links according to distro-arch, as is already done under pools/ and os-releases/version/","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"65ca0fbcf999156085223e42566640971176fbd1","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"   .. code-block:: yaml"},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"      apt_pinned_packages: [{ package: \u0027*\u0027, origin: \"{{ (ansible_distribution_release \u003d\u003d \u0027bionic\u0027) | ternary(\u0027XX.archive.ubuntu.com\u0027, \u0027ceph.com\u0027) }}\" }]"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"   replace XX with your country code"},{"line_number":71,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_d1752e33","line":68,"range":{"start_line":68,"start_character":110,"end_line":68,"end_character":131},"updated":"2020-03-10 13:01:22.000000000","message":"Why in https://review.opendev.org/#/c/708420/ is just \"Ubuntu\"?","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"4927e93971c18ea6de7b310a7ae736f10d77383a","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"   .. code-block:: yaml"},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"      apt_pinned_packages: [{ package: \u0027*\u0027, origin: \"{{ (ansible_distribution_release \u003d\u003d \u0027bionic\u0027) | ternary(\u0027XX.archive.ubuntu.com\u0027, \u0027ceph.com\u0027) }}\" }]"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"   replace XX with your country code"},{"line_number":71,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_b4942038","line":68,"range":{"start_line":68,"start_character":110,"end_line":68,"end_character":131},"in_reply_to":"1fa4df85_d1752e33","updated":"2020-03-17 09:13:18.000000000","message":"Because I discovered and simplified it after I wrote this, good eye though ;) Will update.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"65ca0fbcf999156085223e42566640971176fbd1","unresolved":false,"context_lines":[{"line_number":87,"context_line":""},{"line_number":88,"context_line":"      (* because we\u0027re deleting all container facts for the host aswell.)"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"   #. If rabbitmq container was running on this host"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"      we forget it by running these commands on another rabbitmq host."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"      .. code:: console"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"         rabbitmqctl cluster_status"},{"line_number":97,"context_line":"         rabbitmqctl forget_cluster_node rabbit@removed_host_rabbitmq_container"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"#. If it\u0027s not a \"primary\", install everything on the new node"},{"line_number":100,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_11a506a2","line":97,"range":{"start_line":90,"start_character":0,"end_line":97,"end_character":79},"updated":"2020-03-10 13:01:22.000000000","message":"Not sure it\u0027s really required","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"4927e93971c18ea6de7b310a7ae736f10d77383a","unresolved":false,"context_lines":[{"line_number":87,"context_line":""},{"line_number":88,"context_line":"      (* because we\u0027re deleting all container facts for the host aswell.)"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"   #. If rabbitmq container was running on this host"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"      we forget it by running these commands on another rabbitmq host."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"      .. code:: console"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"         rabbitmqctl cluster_status"},{"line_number":97,"context_line":"         rabbitmqctl forget_cluster_node rabbit@removed_host_rabbitmq_container"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"#. If it\u0027s not a \"primary\", install everything on the new node"},{"line_number":100,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_91ce8981","line":97,"range":{"start_line":90,"start_character":0,"end_line":97,"end_character":79},"in_reply_to":"1fa4df85_11a506a2","updated":"2020-03-17 09:13:18.000000000","message":"If you don\u0027t clear a node from the cluster, you\u0027ll get something like this;\n\nfatal: [grown-cobra_rabbit_mq_container-76a90bd4]: FAILED! \u003d\u003e {\"attempts\": 5, \"changed\": true, \"cmd\": [\"rabbitmqctl\", \"join_cluster\", \"rabbit@guided-fawn-rabbit-mq-container-dd97ecb8\"], \"msg\": \"non-zero return code\", \"rc\": 70, \"stderr\": \"Error:\\n{:inconsistent_cluster, \u0027Node \\\\\u0027rabbit@guided-fawn-rabbit-mq-container-dd97ecb8\\\\\u0027 thinks it\\\\\u0027s clustered with node \\\\\u0027rabbit@grown-cobra-rabbit-mq-container-76a90bd4\\\\\u0027, but \\\\\u0027rabbit@grown-cobra-rabbit-mq-container-76a90bd4\\\\\u0027 disagrees\u0027}\", \"stdout\": \"Clustering node rabbit@grown-cobra-rabbit-mq-container-76a90bd4 with rabbit@guided-fawn-rabbit-mq-container-dd97ecb8\"}\n\nYou\u0027re introducing a new node with the same name (that has no state). So the cluster needs to forget about the old one to accept a new one.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"bcf40b52957b6bca9e5d7f106468df32c8cf4f38","unresolved":false,"context_lines":[{"line_number":87,"context_line":""},{"line_number":88,"context_line":"      (* because we\u0027re deleting all container facts for the host aswell.)"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"   #. If rabbitmq container was running on this host"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"      we forget it by running these commands on another rabbitmq host."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"      .. code:: console"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"         rabbitmqctl cluster_status"},{"line_number":97,"context_line":"         rabbitmqctl forget_cluster_node rabbit@removed_host_rabbitmq_container"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"#. If it\u0027s not a \"primary\", install everything on the new node"},{"line_number":100,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_15ea60ab","line":97,"range":{"start_line":90,"start_character":0,"end_line":97,"end_character":79},"in_reply_to":"1fa4df85_91ce8981","updated":"2020-03-17 18:21:37.000000000","message":"Ah, yes, recalled that part, you\u0027re right here","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"65ca0fbcf999156085223e42566640971176fbd1","unresolved":false,"context_lines":[{"line_number":96,"context_line":"         rabbitmqctl cluster_status"},{"line_number":97,"context_line":"         rabbitmqctl forget_cluster_node rabbit@removed_host_rabbitmq_container"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"#. If it\u0027s not a \"primary\", install everything on the new node"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   .. code:: console"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"      openstack-ansible setup-everything.yml"},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"#. If it IS a \"primary\", do these steps"},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"   .. code:: console"},{"line_number":108,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_91081692","line":105,"range":{"start_line":99,"start_character":0,"end_line":105,"end_character":39},"updated":"2020-03-10 13:01:22.000000000","message":"Not sure this segregation is relevant here. Like you\u0027ll need to check galera and rabbit for every host. Otherwise you\u0027ll have galera having single host, which is smth I\u0027d not really appreciate.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"bcf40b52957b6bca9e5d7f106468df32c8cf4f38","unresolved":false,"context_lines":[{"line_number":96,"context_line":"         rabbitmqctl cluster_status"},{"line_number":97,"context_line":"         rabbitmqctl forget_cluster_node rabbit@removed_host_rabbitmq_container"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"#. If it\u0027s not a \"primary\", install everything on the new node"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   .. code:: console"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"      openstack-ansible setup-everything.yml"},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"#. If it IS a \"primary\", do these steps"},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"   .. code:: console"},{"line_number":108,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_95c0902c","line":105,"range":{"start_line":99,"start_character":0,"end_line":105,"end_character":39},"in_reply_to":"1fa4df85_11a059a5","updated":"2020-03-17 18:21:37.000000000","message":"But once you have only one single galera, which is eventually master, you get all read queries on it, and, in case of it failure (like because of OOM as of increased load) you will have to do cluster re-init or other nasty things. So I wouldn\u0027t leave single galera node...","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"4927e93971c18ea6de7b310a7ae736f10d77383a","unresolved":false,"context_lines":[{"line_number":96,"context_line":"         rabbitmqctl cluster_status"},{"line_number":97,"context_line":"         rabbitmqctl forget_cluster_node rabbit@removed_host_rabbitmq_container"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"#. If it\u0027s not a \"primary\", install everything on the new node"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   .. code:: console"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"      openstack-ansible setup-everything.yml"},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"#. If it IS a \"primary\", do these steps"},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"   .. code:: console"},{"line_number":108,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_11a059a5","line":105,"range":{"start_line":99,"start_character":0,"end_line":105,"end_character":39},"in_reply_to":"1fa4df85_91081692","updated":"2020-03-17 09:13:18.000000000","message":"I can\u0027t be sure about everyones complicated setups, but in the easy case of 3 infra nodes having all the containers, the \"primary\" for all services/containers ends up being the same host[0]. What makes these containers unique is stuff like the repo_server on it gets a lsync script, the galera_container on it is the one that\u0027s queried to check the cluster state. (Which is probably why it ends up in a cluster by itself after installation, and why rabbitmq ends up in a cluster by itself aswell.)","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"f3ee9ace91f71ad1ac814aa8ae44e915fe383bc1","unresolved":false,"context_lines":[{"line_number":96,"context_line":"         rabbitmqctl cluster_status"},{"line_number":97,"context_line":"         rabbitmqctl forget_cluster_node rabbit@removed_host_rabbitmq_container"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"#. If it\u0027s not a \"primary\", install everything on the new node"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"   .. code:: console"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"      openstack-ansible setup-everything.yml"},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"#. If it IS a \"primary\", do these steps"},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"   .. code:: console"},{"line_number":108,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_bbd7b64f","line":105,"range":{"start_line":99,"start_character":0,"end_line":105,"end_character":39},"in_reply_to":"1fa4df85_95c0902c","updated":"2020-03-18 13:39:36.000000000","message":"Yeah, this is where it gets funny. I\u0027ll try to make it clearer in the guide.\n\nBefore you finally take out the \"primary\", all other nodes should have been reinstalled with bionic, and happily joined back into the cluster. So you should have a working cluster of 3 galeras.\n\nWhen you take out the primary, your keepalived/haproxy could/should switch to another box, your galera node count goes down to 2. On this new haproxy, your \"primary\" galera is down, and all traffic gets redirected to one of the others on the list. However as soon as the galera service gets installed on the \"primary\", even if it\u0027s not in sync with anyone, haproxy will see that it\u0027s operational and forward requests there. This is why you want to set it into MAINT. Then run the galera-install, check that your primary is in a cluster of itself, restart the mariadb.service, see that it joins the cluster with the other two, and gets all databasees populated.\n\nNow you\u0027re back to 3 galeras, you can take it out of maint in haproxy.\n\nThis is also why you don\u0027t want to run with \"upgrade_galera\" or \"upgrade_rabbitmq\", because you\u0027re not upgrading any packages. And you don\u0027t want it to start uninstalling any packages on the working servers, which is what will happen if you set those flags.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"65ca0fbcf999156085223e42566640971176fbd1","unresolved":false,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":"      openstack-ansible rabbitmq-install.yml"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"   rabbitmq-primary will also be in a weird cluster of it\u0027s own state. You fix"},{"line_number":139,"context_line":"   this by doing these commands on it."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"   .. code:: console"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"      rabbitmqctl stop_app"},{"line_number":144,"context_line":"      rabbitmqctl join_cluster rabbit@some_operational_rabbitmq_container"},{"line_number":145,"context_line":"      rabbitmqctl start_app"},{"line_number":146,"context_line":"      rabbitmqctl cluster_status"},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"   Everything should now be in a working state and we can finish it off with"},{"line_number":149,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_b1bd12b8","line":146,"range":{"start_line":138,"start_character":0,"end_line":146,"end_character":32},"updated":"2020-03-10 13:01:22.000000000","message":"I\u0027d say just to run `openstack-ansible rabbitmq-install.yml -e rabbitmq_upgrade\u003dtrue` - this should make lost host to re-join cluster","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"f3ee9ace91f71ad1ac814aa8ae44e915fe383bc1","unresolved":false,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":"      openstack-ansible rabbitmq-install.yml"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"   rabbitmq-primary will also be in a weird cluster of it\u0027s own state. You fix"},{"line_number":139,"context_line":"   this by doing these commands on it."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"   .. code:: console"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"      rabbitmqctl stop_app"},{"line_number":144,"context_line":"      rabbitmqctl join_cluster rabbit@some_operational_rabbitmq_container"},{"line_number":145,"context_line":"      rabbitmqctl start_app"},{"line_number":146,"context_line":"      rabbitmqctl cluster_status"},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"   Everything should now be in a working state and we can finish it off with"},{"line_number":149,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_3df3b675","line":146,"range":{"start_line":138,"start_character":0,"end_line":146,"end_character":32},"in_reply_to":"1fa4df85_9572d048","updated":"2020-03-18 13:39:36.000000000","message":"I just finished testing that, two rabbits in cluster unaware of the \"primary\", ran openstack-ansible rabbitmq-install.yml -e \u0027rabbitmq_upgrade\u003dtrue\u0027 and it\u0027s a nope. Doesn\u0027t work. It is really bad infact. cause now the two rabbits are dead, and your primary is in a cluster with itself, and has no queues, so now everything is bad until you figure out how you recover rabbit, most likely by nuking the others or everything and trying again, then waiting for setup-openstack.yml to complete and populate the queues.\n\nI\u0027m gonna venture to say that running \"anything\"_upgrade\u003dyes on a failed/incomplete cluster is gonna be a bad experience.\n\nUpgrade shuts down the working nodes, while attempting to do something on the primary.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":28619,"name":"Dmitriy Rabotyagov","email":"noonedeadpunk@gmail.com","username":"noonedeadpunk"},"change_message_id":"bcf40b52957b6bca9e5d7f106468df32c8cf4f38","unresolved":false,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":"      openstack-ansible rabbitmq-install.yml"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"   rabbitmq-primary will also be in a weird cluster of it\u0027s own state. You fix"},{"line_number":139,"context_line":"   this by doing these commands on it."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"   .. code:: console"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"      rabbitmqctl stop_app"},{"line_number":144,"context_line":"      rabbitmqctl join_cluster rabbit@some_operational_rabbitmq_container"},{"line_number":145,"context_line":"      rabbitmqctl start_app"},{"line_number":146,"context_line":"      rabbitmqctl cluster_status"},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"   Everything should now be in a working state and we can finish it off with"},{"line_number":149,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_9572d048","line":146,"range":{"start_line":138,"start_character":0,"end_line":146,"end_character":32},"in_reply_to":"1fa4df85_97c42242","updated":"2020-03-17 18:21:37.000000000","message":"Yeah, you are probably right that it doesn\u0027t if you don\u0027t forget this node.\nBut I\u0027m pretty sure it should work in case you drop server from cluster in advance.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":25600,"name":"Ebbex","display_name":"ebbex","email":"openstack@slipsprogrammor.no","username":"ebbex"},"change_message_id":"4927e93971c18ea6de7b310a7ae736f10d77383a","unresolved":false,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":"      openstack-ansible rabbitmq-install.yml"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"   rabbitmq-primary will also be in a weird cluster of it\u0027s own state. You fix"},{"line_number":139,"context_line":"   this by doing these commands on it."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"   .. code:: console"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"      rabbitmqctl stop_app"},{"line_number":144,"context_line":"      rabbitmqctl join_cluster rabbit@some_operational_rabbitmq_container"},{"line_number":145,"context_line":"      rabbitmqctl start_app"},{"line_number":146,"context_line":"      rabbitmqctl cluster_status"},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"   Everything should now be in a working state and we can finish it off with"},{"line_number":149,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1fa4df85_97c42242","line":146,"range":{"start_line":138,"start_character":0,"end_line":146,"end_character":32},"in_reply_to":"1fa4df85_b1bd12b8","updated":"2020-03-17 09:13:18.000000000","message":"It doesn\u0027t though. Atleast not if you skip clearing the node out from the cluster before doing this. Now you\u0027re left with two dead rabbitmq nodes that refuse to start because they think they\u0027re in a cluster with your \"primary\", but the primary disagrees.","commit_id":"bc7b2a0f609c6a9edad183b544befda7fa713fc2"},{"author":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"change_message_id":"fc5c8ecc9523492d5cc800dd31137c580f6dcadc","unresolved":false,"context_lines":[{"line_number":29,"context_line":""},{"line_number":30,"context_line":"   This will be the LAST box you\u0027ll want to reinstall."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"   . haproxy/keepalived"},{"line_number":33,"context_line":"   Finding your haproxy/keepalived primary is as easy as"},{"line_number":34,"context_line":"   .. code:: console"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"      ssh {{ external_lb_vip_address }}"},{"line_number":37,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"df33271e_505fbae4","line":34,"range":{"start_line":32,"start_character":0,"end_line":34,"end_character":20},"updated":"2020-03-20 19:12:17.000000000","message":"Some extra blank lines will be needed between these so the generated HTML formats properly.","commit_id":"557bf42d60815eb6ca93fbbef24b5e11c083015c"},{"author":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"change_message_id":"fc5c8ecc9523492d5cc800dd31137c580f6dcadc","unresolved":false,"context_lines":[{"line_number":45,"context_line":"   and can visit https://admin:password@external_lb_vip_address:1936/ and read"},{"line_number":46,"context_line":"   \"Statistics Report for pid # on infrastructure_host"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"   . repo_container"},{"line_number":49,"context_line":"   Check all your repo_containers and look for /etc/lsyncd/lsyncd.conf.lua"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"#. Disable your non-primary repo_all-back"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df33271e_f069ae8b","line":48,"range":{"start_line":48,"start_character":3,"end_line":48,"end_character":19},"updated":"2020-03-20 19:12:17.000000000","message":"Are these sourcing something. Not clear what these are for.","commit_id":"557bf42d60815eb6ca93fbbef24b5e11c083015c"},{"author":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"change_message_id":"fc5c8ecc9523492d5cc800dd31137c580f6dcadc","unresolved":false,"context_lines":[{"line_number":92,"context_line":""},{"line_number":93,"context_line":"      apt_pinned_packages: [{ package: \u0027*\u0027, release: \"{{ (ansible_distribution_release \u003d\u003d \u0027bionic\u0027) | ternary(\u0027Ubuntu\u0027, \u0027ceph.com\u0027) }}\" }]"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"   This fix was merged in the latest stable/rocky on 2020-03-11."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Actually deploying"},{"line_number":98,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df33271e_d0f74ad4","line":95,"updated":"2020-03-20 19:12:17.000000000","message":"Might be nice to include a link to the fix?","commit_id":"557bf42d60815eb6ca93fbbef24b5e11c083015c"}]}