)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"6f578dee6b4b5c3a7c141bc4ad9f2686bfe5475a","unresolved":true,"context_lines":[{"line_number":27,"context_line":"\"inventory_hostname\" trick makes it run only once for the rolling"},{"line_number":28,"context_line":"update of the controllers."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"[1] see https://docs.ansible.com/ansible/2.3/playbooks_delegation.html#id12"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Change-Id: I4356d5f25c3ddac305276743ecc54b6b1b5d96cd"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"51ce0c85_afd2b093","line":30,"range":{"start_line":30,"start_character":4,"end_line":30,"end_character":75},"updated":"2021-03-10 18:23:51.000000000","message":"Should update to latest documentation even if 2.6 is used in queens.\n\nIn latest documentation ansible_play_hosts_all is used, I believe the difference is that failed/unreachable are not included there, according to https://docs.ansible.com/ansible/latest/reference_appendices/special_variables.html (infering here).\n\nIt shouldn\u0027t make a big difference, if anybody has a preference...","commit_id":"271868d430049107e1fc03f96f7ce17fca91e83d"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"99c50aa870517becbe47b53eeb8b9a70c2bf7b7d","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[Queens] Handle major upgrade of rabbitmq during update."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When upgrading rabbitmq to a new major version, the cluster cannot be"},{"line_number":10,"context_line":"reform correctly as the old and the new version cannot talk to each"},{"line_number":11,"context_line":"other."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"The algorithm used here is to solve the problem is:"},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":14,"id":"e33b4543_f8fd5eee","line":11,"range":{"start_line":9,"start_character":48,"end_line":11,"end_character":6},"updated":"2021-04-06 08:22:54.000000000","message":"is that a problem?\ndo we really need ban/unban nodes?\ncan\u0027t we just upgrade it as it goes, allowing clients using rabbit nodes even if unclustered?","commit_id":"10dbc36bc61bc7cd17a11c46bf58e9be46468bee"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"dc0afbf35196339e5e3e23e7dcf1a20818c72239","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[Queens] Handle major upgrade of rabbitmq during update."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When upgrading rabbitmq to a new major version, the cluster cannot be"},{"line_number":10,"context_line":"reform correctly as the old and the new version cannot talk to each"},{"line_number":11,"context_line":"other."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"The algorithm used here is to solve the problem is:"},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":14,"id":"6646bd29_b35181e4","line":11,"range":{"start_line":9,"start_character":48,"end_line":11,"end_character":6},"in_reply_to":"26e180ae_f2be600d","updated":"2021-04-08 09:01:16.000000000","message":"So I\u0027ve tested this patch without the ban dance[1] and it fails when the first node updated is the rabbitmq bootstrap node. It refuses to start with:\n\n:inconsistent_cluster, \u0027Rabbit version mismatch: local node is 3.7.28, remote node 3.6.15\u0027} \n\nas it\u0027s the boostrap rabbitmq node, some puppet code (rabbitmq-ready) wait for it to start, which isn\u0027t going to happen. The update then fail during common deploy step at step 2.\n\n[1] https://review.opendev.org/c/openstack/tripleo-heat-templates/+/784904","commit_id":"10dbc36bc61bc7cd17a11c46bf58e9be46468bee"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"92fae03683be3d8149fbaf6c6abbbe371cb4f7fd","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[Queens] Handle major upgrade of rabbitmq during update."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When upgrading rabbitmq to a new major version, the cluster cannot be"},{"line_number":10,"context_line":"reform correctly as the old and the new version cannot talk to each"},{"line_number":11,"context_line":"other."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"The algorithm used here is to solve the problem is:"},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":14,"id":"55f1fdf0_5752bdd1","line":11,"range":{"start_line":9,"start_character":48,"end_line":11,"end_character":6},"in_reply_to":"6646bd29_b35181e4","updated":"2021-04-08 11:27:35.000000000","message":"Ack","commit_id":"10dbc36bc61bc7cd17a11c46bf58e9be46468bee"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"fd48a678df79fb1d466b1bcfe2ec20bbfd3eb538","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"[Queens] Handle major upgrade of rabbitmq during update."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When upgrading rabbitmq to a new major version, the cluster cannot be"},{"line_number":10,"context_line":"reform correctly as the old and the new version cannot talk to each"},{"line_number":11,"context_line":"other."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"The algorithm used here is to solve the problem is:"},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":14,"id":"26e180ae_f2be600d","line":11,"range":{"start_line":9,"start_character":48,"end_line":11,"end_character":6},"in_reply_to":"e33b4543_f8fd5eee","updated":"2021-04-06 09:44:41.000000000","message":"So rabbitmq 3.7 cannot talks to rabbitmq 3.6 at all, so as soon as we update one one the cluster cannot reform and there will be error. Initially we had the utf-8 error that prevented the update from working at all, so we never tested the update with the utf-8 fix and no ban/unban dance. The point being that maybe the cluster eventually recover but this has to be tested. We need two tests one when we start by the rabbitmq_boostrap node and another one when we start we a non rabbitmq_bootstrap node.\n\nThe ban/unban dance prevent any error during the update and have been tested from z3 to z14 successfully in those two mentioned configuration (12 jobs all in all), but I agree it\u0027s quite a dance.","commit_id":"10dbc36bc61bc7cd17a11c46bf58e9be46468bee"}],"docker/services/pacemaker/rabbitmq.yaml":[{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"19c1e19ffb8a767252955165ca68b09426d356ca","unresolved":true,"context_lines":[{"line_number":297,"context_line":"                  with_items: \"{{ rabbitmq_containers_to_destroy.results | map(attribute\u003d\u0027stdout_lines\u0027) | list | flatten }}\""},{"line_number":298,"context_line":"                - name: Remove previous Rabbitmq images"},{"line_number":299,"context_line":"                  shell: \"docker rmi -f {{ item }}\""},{"line_number":300,"context_line":"                  with_items: \"{{ rabbitmq_image_id.stdout_lines }}\""},{"line_number":301,"context_line":"              when:"},{"line_number":302,"context_line":"                - rabbitmq_image_id.stdout !\u003d \u0027\u0027"},{"line_number":303,"context_line":"            - name: Pull latest Rabbitmq images"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"c2ec74cb_65861f70","line":300,"updated":"2021-03-04 17:37:19.000000000","message":"Seems we can have several output here, which one to choose ... ?","commit_id":"7c8e4d7fbc2b05060e9667e3c9f844dac892b395"},{"author":{"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},"change_message_id":"45c6c4a47a888d61d2932168e3d9bfa9dace8aeb","unresolved":true,"context_lines":[{"line_number":309,"context_line":"        - name: Handle major upgrade of rabbitmq"},{"line_number":310,"context_line":"          when: step|int \u003d\u003d 5"},{"line_number":311,"context_line":"          run_once: true"},{"line_number":312,"context_line":"          block:"},{"line_number":313,"context_line":"            - name: Discover the running version of rabbitmq"},{"line_number":314,"context_line":"              shell: |"},{"line_number":315,"context_line":"                docker_name\u003d$(crm_mon --as-xml | \\"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"53790f7c_a89a1f76","line":312,"range":{"start_line":312,"start_character":10,"end_line":312,"end_character":1},"updated":"2021-03-04 17:06:23.000000000","message":"Looks like the need is to get the always changing name of the rabbitmq container spawned by pacemaker (of the form /rabbitmq-bundle-docker-[0-9]/), to fetch the rabbitmq version.\n\nCan you instead run it from a temporary container that uses the same container image? i.e.\ndocker run --rm {{ docker_image_latest }} \"\u003cthe rpm command\u003e\"\n\nThat way you don\u0027t depend on a running container or a complex xpath.\n(That still require to be run before the re-tagging if docker_image_latest above)","commit_id":"7c8e4d7fbc2b05060e9667e3c9f844dac892b395"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"19c1e19ffb8a767252955165ca68b09426d356ca","unresolved":true,"context_lines":[{"line_number":309,"context_line":"        - name: Handle major upgrade of rabbitmq"},{"line_number":310,"context_line":"          when: step|int \u003d\u003d 5"},{"line_number":311,"context_line":"          run_once: true"},{"line_number":312,"context_line":"          block:"},{"line_number":313,"context_line":"            - name: Discover the running version of rabbitmq"},{"line_number":314,"context_line":"              shell: |"},{"line_number":315,"context_line":"                docker_name\u003d$(crm_mon --as-xml | \\"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"7d2c23ab_964a4cbc","line":312,"range":{"start_line":312,"start_character":10,"end_line":312,"end_character":1},"in_reply_to":"53790f7c_a89a1f76","updated":"2021-03-04 17:37:19.000000000","message":"Hum was about to implement that but when looking at l300, I see that it can return several images ... it must happen under certain condition and then I cannot choose which one is the right one.\n\nNeed something else.","commit_id":"7c8e4d7fbc2b05060e9667e3c9f844dac892b395"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"4551ff29f3027373f64efa7f8af3a5c829d68b5a","unresolved":true,"context_lines":[{"line_number":276,"context_line":"      metadata_settings:"},{"line_number":277,"context_line":"        get_attr: [RabbitmqBase, role_data, metadata_settings]"},{"line_number":278,"context_line":"      update_tasks:"},{"line_number":279,"context_line":"        - name: Verify if a major upgrade of rabbitmq is going to happen"},{"line_number":280,"context_line":"          when: step|int \u003d\u003d 1"},{"line_number":281,"context_line":"          run_once: true"},{"line_number":282,"context_line":"          block:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"594b781b_0194bf8a","line":279,"range":{"start_line":279,"start_character":9,"end_line":279,"end_character":72},"updated":"2021-03-05 10:50:08.000000000","message":"Currently only checking the existing version and assuming we\u0027re going to jump.  Should we check the next version as well ? It will requires some more action after we pulled in the images.","commit_id":"856b31c67475051d879140e89845d8bb884befd9"},{"author":{"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},"change_message_id":"935a8ea67d2dd06535c52c896aae89a7f15c09b2","unresolved":true,"context_lines":[{"line_number":323,"context_line":"          run_once: true"},{"line_number":324,"context_line":"          block:"},{"line_number":325,"context_line":"            - name: Disable rabbitmq to prepare for major version bump"},{"line_number":326,"context_line":"              shell: \"pcs resource disable --wait rabbitmq-bundle\""},{"line_number":327,"context_line":"              when: rabbitmq_need_upgrade|bool"},{"line_number":328,"context_line":"      post_update_tasks:"},{"line_number":329,"context_line":"        - name: Reactivate rabbitmq if a major upgrade of rabbitmq happened"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"35feeec0_fc56df2a","line":326,"updated":"2021-03-04 21:30:10.000000000","message":"What is the expected behaviour? 3 restarts of the entire rabbitmq cluster? Or just rabbitmq restarting locally on the node that is being updated?","commit_id":"856b31c67475051d879140e89845d8bb884befd9"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"ea5393c73b3da01d3b86a0a52f5ab21a01063979","unresolved":true,"context_lines":[{"line_number":323,"context_line":"          run_once: true"},{"line_number":324,"context_line":"          block:"},{"line_number":325,"context_line":"            - name: Disable rabbitmq to prepare for major version bump"},{"line_number":326,"context_line":"              shell: \"pcs resource disable --wait rabbitmq-bundle\""},{"line_number":327,"context_line":"              when: rabbitmq_need_upgrade|bool"},{"line_number":328,"context_line":"      post_update_tasks:"},{"line_number":329,"context_line":"        - name: Reactivate rabbitmq if a major upgrade of rabbitmq happened"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"8692d469_a355a25c","line":326,"in_reply_to":"35feeec0_fc56df2a","updated":"2021-03-16 10:20:12.000000000","message":"Initially it was a stop before the update of the rabbitmq cluster followed by an update of the first node, followed by a restart of the rabbitmq cluster.\n\nThe hope was that it would start the updated rabbitmq node.\n\nThen we would update the other node (without the disable/enable rabbitmq) and the updated rabbitmq would naturally re-join the cluster...\n\nWell, it doesn\u0027t work \\o/\n\nin the end the updated node goes into timeout, then when we try to update the last controller (still running) it fails for \"some reason\" (couldn\u0027t find the root cause there)\n\nThe updated nodes don\u0027t restart:\n\nFailed Resource Actions:\n* rabbitmq_start_0 on rabbitmq-bundle-1 \u0027unknown error\u0027 (1): call\u003d13, status\u003dTimed Out, exitreason\u003d\u0027\u0027,                                                     \n    last-rc-change\u003d\u0027Mon Mar 15 18:51:50 2021\u0027, queued\u003d0ms, exec\u003d200564ms\n* rabbitmq_start_0 on rabbitmq-bundle-2 \u0027unknown error\u0027 (1): call\u003d14, status\u003dTimed Out, exitreason\u003d\u0027\u0027,                                                     \n    last-rc-change\u003d\u0027Mon Mar 15 18:21:09 2021\u0027, queued\u003d0ms, exec\u003d200511ms\n\nNeed help around the root cause analysis about the failure so that we can test a new way to do that update.","commit_id":"856b31c67475051d879140e89845d8bb884befd9"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"ee076bb3b384960201c11693df3d76c1b22fa634","unresolved":true,"context_lines":[{"line_number":288,"context_line":"            - name: Register if an major upgrade of rabbitmq is going to take place"},{"line_number":289,"context_line":"              set_fact:"},{"line_number":290,"context_line":"                rabbitmq_need_upgrade: \"{{ rabbitmq_subversion.stdout|int \u003d\u003d 6 }}\""},{"line_number":291,"context_line":"        - name: Save variables for major upgrade of rabbitmq orchestration."},{"line_number":292,"context_line":"          when:"},{"line_number":293,"context_line":"            - step|int \u003d\u003d 0"},{"line_number":294,"context_line":"            - rabbitmq_need_upgrade|bool"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"080db855_0fa89eec","line":291,"range":{"start_line":291,"start_character":31,"end_line":291,"end_character":75},"updated":"2021-03-19 18:52:29.000000000","message":"reword","commit_id":"05c61f01f2100828969957c5a92109e003962ea2"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"ee076bb3b384960201c11693df3d76c1b22fa634","unresolved":true,"context_lines":[{"line_number":298,"context_line":"                hiera -c /etc/puppet/hiera.yaml rabbitmq_short_node_names | sed -e s/nil/[]/"},{"line_number":299,"context_line":"              register: rabbitmq_nodes"},{"line_number":300,"context_line":"              when: rabbitmq_nodes_list is undefined"},{"line_number":301,"context_line":"            - name: Capture if there any ban on the rabbitmq cluster."},{"line_number":302,"context_line":"              shell: |"},{"line_number":303,"context_line":"                pcs constraint ref rabbitmq-bundle | awk \u0027/cli-ban/{print $0}\u0027"},{"line_number":304,"context_line":"              register: rabbitmq_ban"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"3c4d3a9f_549724bf","line":301,"range":{"start_line":301,"start_character":30,"end_line":301,"end_character":40},"updated":"2021-03-19 18:52:29.000000000","message":"there is","commit_id":"05c61f01f2100828969957c5a92109e003962ea2"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"ee076bb3b384960201c11693df3d76c1b22fa634","unresolved":true,"context_lines":[{"line_number":302,"context_line":"              shell: |"},{"line_number":303,"context_line":"                pcs constraint ref rabbitmq-bundle | awk \u0027/cli-ban/{print $0}\u0027"},{"line_number":304,"context_line":"              register: rabbitmq_ban"},{"line_number":305,"context_line":"            - name: Save rabbitmq ban status on the cluster and nodes information."},{"line_number":306,"context_line":"              set_fact:"},{"line_number":307,"context_line":"                rabbitmq_ban_exists: \"{{ (rabbitmq_ban.stdout_lines|length) \u003e 0 }}\""},{"line_number":308,"context_line":"                rabbitmq_nodes_list: \"{{ rabbitmq_nodes.stdout | default(\u0027[]\u0027) | from_json }}\""}],"source_content_type":"text/x-yaml","patch_set":7,"id":"35b48c28_42af5470","line":305,"range":{"start_line":305,"start_character":45,"end_line":305,"end_character":47},"updated":"2021-03-19 18:52:29.000000000","message":"of","commit_id":"05c61f01f2100828969957c5a92109e003962ea2"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"71f3358bef62a19d35d61cae6309429201e54d86","unresolved":true,"context_lines":[{"line_number":306,"context_line":"            - name: Save rabbitmq ban status on the cluster and nodes information."},{"line_number":307,"context_line":"              set_fact:"},{"line_number":308,"context_line":"                rabbitmq_ban_exists: \"{{ (rabbitmq_ban.stdout_lines|length) \u003e 0 }}\""},{"line_number":309,"context_line":"                rabbitmq_nodes_list: \"{{ rabbitmq_nodes.stdout | default(\u0027[]\u0027) | from_json }}\""},{"line_number":310,"context_line":"        - name: Rabbit fetch and retag container image for pacemaker"},{"line_number":311,"context_line":"          when: step|int \u003d\u003d 2"},{"line_number":312,"context_line":"          block: \u0026rabbitmq_fetch_retag_container_tasks"}],"source_content_type":"text/x-yaml","patch_set":16,"id":"fd06ff37_948a2fa2","line":309,"range":{"start_line":309,"start_character":41,"end_line":309,"end_character":55},"updated":"2021-04-08 11:35:39.000000000","message":"when: rabbitmq_nodes is defined and rabbitmq_nodes.stdout is defined?","commit_id":"33ea5d33380c576696e5dd50be8bd3b4c3fe57dd"},{"author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"change_message_id":"c0e02d9a15f1355730c948b325b65528958ddd72","unresolved":true,"context_lines":[{"line_number":306,"context_line":"            - name: Save rabbitmq ban status on the cluster and nodes information."},{"line_number":307,"context_line":"              set_fact:"},{"line_number":308,"context_line":"                rabbitmq_ban_exists: \"{{ (rabbitmq_ban.stdout_lines|length) \u003e 0 }}\""},{"line_number":309,"context_line":"                rabbitmq_nodes_list: \"{{ rabbitmq_nodes.stdout | default(\u0027[]\u0027) | from_json }}\""},{"line_number":310,"context_line":"        - name: Rabbit fetch and retag container image for pacemaker"},{"line_number":311,"context_line":"          when: step|int \u003d\u003d 2"},{"line_number":312,"context_line":"          block: \u0026rabbitmq_fetch_retag_container_tasks"}],"source_content_type":"text/x-yaml","patch_set":16,"id":"daa25871_3ed8a8ec","line":309,"range":{"start_line":309,"start_character":41,"end_line":309,"end_character":55},"in_reply_to":"fd06ff37_948a2fa2","updated":"2021-04-08 12:58:37.000000000","message":"hum... I see your point, there is a conditional in l301 so maybe this could be undefined.\n\nFor some reason I didn\u0027t had any issue in the real jobs, as rabbitmq_nodes is always defined in the playbook run.\n\nFor consistency either I should cond the rabbitmq_nodes_list fact to being undefined in the same block (get the shell value and save the fact)","commit_id":"33ea5d33380c576696e5dd50be8bd3b4c3fe57dd"}],"puppet/services/rabbitmq.yaml":[{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"0ba4a6d60c01fdc682ffe6d4aa0585de7d25ca98","unresolved":true,"context_lines":[{"line_number":111,"context_line":"            rabbitmq::tcp_keepalive: true"},{"line_number":112,"context_line":"            rabbitmq_environment:"},{"line_number":113,"context_line":"              LANG: \u0027en_US.UTF-8\u0027"},{"line_number":114,"context_line":"              LC_ALL: \u0027en_US.UTF-8\u0027"},{"line_number":115,"context_line":"              NODE_PORT: \u0027\u0027"},{"line_number":116,"context_line":"              NODE_IP_ADDRESS: \u0027\u0027"},{"line_number":117,"context_line":"              RABBITMQ_NODENAME: \"rabbit@%{::hostname}\""}],"source_content_type":"text/x-yaml","patch_set":14,"id":"69c5af75_d100cca3","line":114,"updated":"2021-04-06 08:18:32.000000000","message":"could we split the UTF-8 part (which is just +2) off the other changes?","commit_id":"10dbc36bc61bc7cd17a11c46bf58e9be46468bee"}]}