)]}'
{"id":"openstack%2Fpuppet-pacemaker~742093","triplet_id":"openstack%2Fpuppet-pacemaker~master~I1b5c2ed35f83b6572db9e919a4af4cd46f0e98e8","project":"openstack/puppet-pacemaker","branch":"master","topic":"bug/1888398","hashtags":[],"change_id":"I1b5c2ed35f83b6572db9e919a4af4cd46f0e98e8","subject":"Retry calling backup_cib()","status":"MERGED","created":"2020-07-21 08:16:01.000000000","updated":"2020-07-30 14:03:58.000000000","submitted":"2020-07-30 14:03:58.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"742093-1596117838442-b464fa81","meta_rev_id":"b9f9417353c33f4d096f198bea3ca8f08fd34096","_number":742093,"virtual_id_number":742093,"owner":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:zuul:gate","value":2,"date":"2020-07-30 14:03:58.000000000","post_submit":true,"permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":11166,"name":"Lukas Bezdicka","email":"social@v3.sk","username":"xbezdick"},{"value":0,"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},{"value":0,"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},{"value":0,"_account_id":26343,"name":"Jose Luis Franco","email":"jfrancoa@redhat.com","username":"jfrancoa"},{"value":0,"_account_id":11090,"name":"Sergii Golovatiuk","email":"sgolovat@redhat.com","username":"holser"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"all":[{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":11166,"name":"Lukas Bezdicka","email":"social@v3.sk","username":"xbezdick"},{"value":2,"date":"2020-07-30 13:41:41.000000000","_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},{"value":2,"date":"2020-07-23 08:21:01.000000000","_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},{"value":0,"_account_id":26343,"name":"Jose Luis Franco","email":"jfrancoa@redhat.com","username":"jfrancoa"},{"value":0,"_account_id":11090,"name":"Sergii Golovatiuk","email":"sgolovat@redhat.com","username":"holser"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"all":[{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"date":"2020-07-27 14:58:12.000000000","_account_id":11166,"name":"Lukas Bezdicka","email":"social@v3.sk","username":"xbezdick"},{"value":1,"date":"2020-07-30 13:41:41.000000000","_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},{"value":0,"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},{"value":0,"date":"2020-07-27 14:58:14.000000000","_account_id":26343,"name":"Jose Luis Franco","email":"jfrancoa@redhat.com","username":"jfrancoa"},{"value":0,"date":"2020-07-27 14:58:09.000000000","_account_id":11090,"name":"Sergii Golovatiuk","email":"sgolovat@redhat.com","username":"holser"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},{"_account_id":11090,"name":"Sergii Golovatiuk","email":"sgolovat@redhat.com","username":"holser"},{"_account_id":11166,"name":"Lukas Bezdicka","email":"social@v3.sk","username":"xbezdick"},{"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":26343,"name":"Jose Luis Franco","email":"jfrancoa@redhat.com","username":"jfrancoa"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-07-23 08:21:01.000000000","updated_by":{"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},"reviewer":{"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},"state":"REVIEWER"},{"updated":"2020-07-27 14:58:09.000000000","updated_by":{"_account_id":11090,"name":"Sergii Golovatiuk","email":"sgolovat@redhat.com","username":"holser"},"reviewer":{"_account_id":11090,"name":"Sergii Golovatiuk","email":"sgolovat@redhat.com","username":"holser"},"state":"REVIEWER"},{"updated":"2020-07-27 14:58:12.000000000","updated_by":{"_account_id":11166,"name":"Lukas Bezdicka","email":"social@v3.sk","username":"xbezdick"},"reviewer":{"_account_id":11166,"name":"Lukas Bezdicka","email":"social@v3.sk","username":"xbezdick"},"state":"REVIEWER"},{"updated":"2020-07-27 14:58:14.000000000","updated_by":{"_account_id":26343,"name":"Jose Luis Franco","email":"jfrancoa@redhat.com","username":"jfrancoa"},"reviewer":{"_account_id":26343,"name":"Jose Luis Franco","email":"jfrancoa@redhat.com","username":"jfrancoa"},"state":"REVIEWER"},{"updated":"2020-07-30 13:41:41.000000000","updated_by":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"reviewer":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"state":"REVIEWER"},{"updated":"2020-07-30 14:03:58.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"01f4d072d870d9f65d28c2a63a96cb7a1fd4ae4f","author":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"date":"2020-07-21 08:16:01.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"c0983e91fd45fddbcfbb7322c2040b5f3d0d7df8","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-21 08:37:09.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\n\n- puppet-openstack-lint-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/dbd4fc1c20d54d47a0466898d69e104a : SUCCESS in 7m 09s\n- puppet-openstack-syntax-5-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/fe6bad818fd940bf95466290174c5c8c : SUCCESS in 4m 08s\n- puppet-openstack-syntax-6-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/8c86f4accfe64a358e57dc6a1dcc0d48 : SUCCESS in 6m 31s\n- puppet-openstack-unit-5.5-centos-8 https://zuul.opendev.org/t/openstack/build/78533a07fcf046ee9a23beaef659fae7 : SUCCESS in 12m 17s\n- puppet-openstack-unit-6.14-centos-8 https://zuul.opendev.org/t/openstack/build/0a80458688be49bb9719090fdcac2b82 : SUCCESS in 11m 01s\n- puppet-openstack-unit-6.14-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/3d3faae98a944469a1c20f73007131db : SUCCESS in 10m 54s\n- puppet-openstack-unit-latest-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/e32303f5ec874252826467ea4619d782 : SUCCESS in 11m 24s (non-voting)","accounts_in_message":[],"_revision_number":1},{"id":"0502647f09d9669d6143d3aff9b94098031c47e3","author":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"date":"2020-07-22 06:37:38.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"2a426182d29b68ec8112274a069ddf73e30c34b0","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-22 07:00:42.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\n\n- puppet-openstack-lint-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/2884890cf8054baab72153f208a2813d : SUCCESS in 6m 06s\n- puppet-openstack-syntax-5-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/e41b9617e15c485eaf0342b7ab750f39 : SUCCESS in 7m 00s\n- puppet-openstack-syntax-6-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/4f0ffcc855bc45cba14b17d7f3f5523a : SUCCESS in 7m 24s\n- puppet-openstack-unit-5.5-centos-8 https://zuul.opendev.org/t/openstack/build/57b42b5ed5404b75b2a8b1ba5879d919 : SUCCESS in 13m 40s\n- puppet-openstack-unit-6.14-centos-8 https://zuul.opendev.org/t/openstack/build/c1dce2a49d8d4319ab91b7f18aeb6e67 : SUCCESS in 17m 02s\n- puppet-openstack-unit-6.14-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/815afa9d006d4f8b834c5e692f28bf09 : SUCCESS in 13m 22s\n- puppet-openstack-unit-latest-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/89ea7a0018c14914aa28846144bb0157 : SUCCESS in 11m 00s (non-voting)","accounts_in_message":[],"_revision_number":2},{"id":"03090d81eba75a97c7a74f9743d6f1f11cc38fca","author":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"date":"2020-07-22 08:14:42.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"4264ce010c1788e5aca79e8d85f7c64d8e9017ee","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-22 08:41:33.000000000","message":"Patch Set 3: Verified+1\n\nBuild succeeded (check pipeline).\n\n- puppet-openstack-lint-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/6d134316ce854f8f93c8070399f24eea : SUCCESS in 5m 39s\n- puppet-openstack-syntax-5-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/6fe6c11f576140e1ba94d3db4699592b : SUCCESS in 4m 40s\n- puppet-openstack-syntax-6-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/88b5f410b8254643bebe97764d6d173e : SUCCESS in 4m 58s\n- puppet-openstack-unit-5.5-centos-8 https://zuul.opendev.org/t/openstack/build/5a349f4d367345b4b002ce46d7ed039f : SUCCESS in 13m 23s\n- puppet-openstack-unit-6.14-centos-8 https://zuul.opendev.org/t/openstack/build/cd3eb139ef684f749533103e8d9355bf : SUCCESS in 13m 10s\n- puppet-openstack-unit-6.14-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/27a3fa0d14b944a7a210e54e68c2d97d : SUCCESS in 11m 16s\n- puppet-openstack-unit-latest-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/f1067e39881645c8a888e429485a260c : SUCCESS in 11m 18s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"98b649e9fa2441adbffb780e24b1a3ccaf5a67a8","author":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"date":"2020-07-23 06:17:43.000000000","message":"Uploaded patch set 4: Commit message was updated.","accounts_in_message":[],"_revision_number":4},{"id":"3dacdbd3e9690b0a1ffdd1b0ca52628c147c2107","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-23 07:20:12.000000000","message":"Patch Set 4: Verified+1\n\nBuild succeeded (check pipeline).\n\n- puppet-openstack-lint-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/31cb67bd0f1f434eb1b6021ecbb5aea3 : SUCCESS in 6m 57s\n- puppet-openstack-syntax-5-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/2c420bc7faba492483bb77a239e780cc : SUCCESS in 7m 40s\n- puppet-openstack-syntax-6-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/d83a5ba0e9e04ea887c4fba684b61d46 : SUCCESS in 4m 48s\n- puppet-openstack-unit-5.5-centos-8 https://zuul.opendev.org/t/openstack/build/216d129856e6452a83cdcbeb8004633c : SUCCESS in 11m 11s\n- puppet-openstack-unit-6.14-centos-8 https://zuul.opendev.org/t/openstack/build/9daea20cc57c451aa45b4c9660294249 : SUCCESS in 14m 18s\n- puppet-openstack-unit-6.14-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/e52507b387f24350b5ca873cc2070adc : SUCCESS in 13m 09s\n- puppet-openstack-unit-latest-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/5432b1ed500a436abb9c3a6685d6d4bd : SUCCESS in 10m 20s (non-voting)","accounts_in_message":[],"_revision_number":4},{"id":"f189f6fca9d9bd513b6f007f2356fe359096c1ab","author":{"_account_id":20778,"name":"Damien Ciabrini","email":"dciabrin@redhat.com","username":"dciabrin"},"date":"2020-07-23 08:21:01.000000000","message":"Patch Set 4: Code-Review+2","accounts_in_message":[],"_revision_number":4},{"id":"e2575cf6f7319ad5f266a29079bc124361f39ef5","author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"date":"2020-07-27 14:46:36.000000000","message":"Topic changed from backup-retry to bug/1888398","accounts_in_message":[],"_revision_number":4},{"id":"acf2282768bab2af11ea7e27daaa70b113b3423b","author":{"_account_id":8297,"name":"Sofer Athlan-Guyot","email":"sathlang@redhat.com","username":"chem"},"date":"2020-07-30 13:41:41.000000000","message":"Patch Set 4: Code-Review+2 Workflow+1\n\namazing.","accounts_in_message":[],"_revision_number":4},{"id":"98d066c595359e7822d9491459854d8ddfa6b087","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-30 13:41:54.000000000","message":"Patch Set 4: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":4},{"id":"254542b8d3784f1e8eeca05d062f831f35ef491b","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-30 14:03:58.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":4},{"id":"b9f9417353c33f4d096f198bea3ca8f08fd34096","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-07-30 14:03:58.000000000","message":"Patch Set 4: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- puppet-openstack-lint-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/d1277b9179444b7bac76eb4823b0a8db : SUCCESS in 5m 18s\n- puppet-openstack-syntax-5-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/94f7d1e4858f46f39f924c3d3c2e8f7c : SUCCESS in 5m 37s\n- puppet-openstack-syntax-6-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/3aa23865cf5d4f22a801c9fff5c37c72 : SUCCESS in 3m 51s\n- puppet-openstack-unit-5.5-centos-8 https://zuul.opendev.org/t/openstack/build/3c8dddf2d6aa4ee49c54eb114d274ae2 : SUCCESS in 12m 27s\n- puppet-openstack-unit-6.14-centos-8 https://zuul.opendev.org/t/openstack/build/a1f029a28d504eabbf9fc738147374d6 : SUCCESS in 13m 23s\n- puppet-openstack-unit-6.14-ubuntu-bionic https://zuul.opendev.org/t/openstack/build/49911a77df4a4b97b942e85a22bf37c4 : SUCCESS in 9m 06s","accounts_in_message":[],"_revision_number":4}],"current_revision_number":4,"current_revision":"b7af3b650923df80e067e5545108d9cfe4c89f83","revisions":{"09566e88206a358a0d7c1d170d0944fbdf47990f":{"kind":"REWORK","_number":1,"created":"2020-07-21 08:16:01.000000000","uploader":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"ref":"refs/changes/93/742093/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/puppet-pacemaker","ref":"refs/changes/93/742093/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/1"}}},"commit":{"parents":[{"commit":"1bb2fe70b1e09dfb04ada5286d86d8d08de48f53","subject":"Fix typo in CRMDIFF_BIN","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/1bb2fe70b1e09dfb04ada5286d86d8d08de48f53"}]}],"author":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-21 08:07:23.000000000","tz":120},"committer":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-21 08:07:23.000000000","tz":120},"subject":"WIP Retry backup_cib()","message":"WIP Retry backup_cib()\n\nCurrently if backup_cib() fails we just give up instantly.\nIn most cases this is actually okay since there are no chances\nthat the command succeeds in later retries. There are a number of cases\nthough were retrying should be done. IHA FFU is one of them because the\nfollowing might happen:\nJul 21 04:59:45 compute-1 puppet-user[61009]: Debug: Executing: \u0027/sbin/ip6tables-save\u0027\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: warning: Cannot proxy IPC connection from uid 0 gid 0 to cib_rw because not connected to cluster\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: error: Error in connection setup (/dev/shm/qb-42459-61225-14-DzDuCR/qb): Remote I/O error (121)\nJul 21 04:59:46 compute-1 puppet-user[61009]: Error: /Stage[main]/Tripleo::Profile::Pacemaker::Compute_instanceha/Pacemaker::Property[compute-instanceha-role-node-property]/Pcmk_property[property-compute-1-compute-instanceha-role]: Could not evaluate: backup_cib: Running: pcs cluster cib /var/lib/pacemaker/cib/puppet-cib-backup20200721-61008-hz2sdw failed with code: 1 -\u003e Error: unable to get cib\n\nAnd only later we get:\nJul 21 05:01:22 compute-1 pacemaker-remoted[42459]: notice: Remote client connection accepted\n\nThat is due to the fact that we cannot necessarily control at what point\nthe core cluster retries the connection, so we should strive for\nretrying a few times no matter what\n\nChange-Id: I1b5c2ed35f83b6572db9e919a4af4cd46f0e98e8\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/09566e88206a358a0d7c1d170d0944fbdf47990f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/09566e88206a358a0d7c1d170d0944fbdf47990f"}]},"branch":"refs/heads/master"},"34ae106f9af8254de8411c54bca07252f6e9ad71":{"kind":"REWORK","_number":2,"created":"2020-07-22 06:37:38.000000000","uploader":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"ref":"refs/changes/93/742093/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/puppet-pacemaker","ref":"refs/changes/93/742093/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/2"}}},"commit":{"parents":[{"commit":"1bb2fe70b1e09dfb04ada5286d86d8d08de48f53","subject":"Fix typo in CRMDIFF_BIN","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/1bb2fe70b1e09dfb04ada5286d86d8d08de48f53"}]}],"author":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-21 08:07:23.000000000","tz":120},"committer":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-22 06:37:31.000000000","tz":120},"subject":"WIP Retry backup_cib()","message":"WIP Retry backup_cib()\n\nCurrently if backup_cib() fails we just give up instantly.\nIn most cases this is actually okay since there are no chances\nthat the command succeeds in later retries. There are a number of cases\nthough were retrying should be done. IHA FFU is one of them because the\nfollowing might happen:\nJul 21 04:59:45 compute-1 puppet-user[61009]: Debug: Executing: \u0027/sbin/ip6tables-save\u0027\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: warning: Cannot proxy IPC connection from uid 0 gid 0 to cib_rw because not connected to cluster\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: error: Error in connection setup (/dev/shm/qb-42459-61225-14-DzDuCR/qb): Remote I/O error (121)\nJul 21 04:59:46 compute-1 puppet-user[61009]: Error: /Stage[main]/Tripleo::Profile::Pacemaker::Compute_instanceha/Pacemaker::Property[compute-instanceha-role-node-property]/Pcmk_property[property-compute-1-compute-instanceha-role]: Could not evaluate: backup_cib: Running: pcs cluster cib /var/lib/pacemaker/cib/puppet-cib-backup20200721-61008-hz2sdw failed with code: 1 -\u003e Error: unable to get cib\n\nAnd only later we get:\nJul 21 05:01:22 compute-1 pacemaker-remoted[42459]: notice: Remote client connection accepted\n\nThat is due to the fact that we cannot necessarily control at what point\nthe core cluster retries the connection, so we should strive for\nretrying a few times no matter what\n\nChange-Id: I1b5c2ed35f83b6572db9e919a4af4cd46f0e98e8\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/34ae106f9af8254de8411c54bca07252f6e9ad71"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/34ae106f9af8254de8411c54bca07252f6e9ad71"}]},"branch":"refs/heads/master"},"1f3a44511e3eb97157b7d549dd0b0feec0caa5f8":{"kind":"REWORK","_number":3,"created":"2020-07-22 08:14:42.000000000","uploader":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"ref":"refs/changes/93/742093/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/puppet-pacemaker","ref":"refs/changes/93/742093/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/3"}}},"commit":{"parents":[{"commit":"1bb2fe70b1e09dfb04ada5286d86d8d08de48f53","subject":"Fix typo in CRMDIFF_BIN","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/1bb2fe70b1e09dfb04ada5286d86d8d08de48f53"}]}],"author":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-21 08:07:23.000000000","tz":120},"committer":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-22 08:14:32.000000000","tz":120},"subject":"WIP Retry backup_cib()","message":"WIP Retry backup_cib()\n\nCurrently if backup_cib() fails we just give up instantly.\nIn most cases this is actually okay since there are no chances\nthat the command succeeds in later retries. There are a number of cases\nthough were retrying should be done. IHA FFU is one of them because the\nfollowing might happen:\nJul 21 04:59:45 compute-1 puppet-user[61009]: Debug: Executing: \u0027/sbin/ip6tables-save\u0027\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: warning: Cannot proxy IPC connection from uid 0 gid 0 to cib_rw because not connected to cluster\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: error: Error in connection setup (/dev/shm/qb-42459-61225-14-DzDuCR/qb): Remote I/O error (121)\nJul 21 04:59:46 compute-1 puppet-user[61009]: Error: /Stage[main]/Tripleo::Profile::Pacemaker::Compute_instanceha/Pacemaker::Property[compute-instanceha-role-node-property]/Pcmk_property[property-compute-1-compute-instanceha-role]: Could not evaluate: backup_cib: Running: pcs cluster cib /var/lib/pacemaker/cib/puppet-cib-backup20200721-61008-hz2sdw failed with code: 1 -\u003e Error: unable to get cib\n\nAnd only later we get:\nJul 21 05:01:22 compute-1 pacemaker-remoted[42459]: notice: Remote client connection accepted\n\nThat is due to the fact that we cannot necessarily control at what point\nthe core cluster retries the connection, so we should strive for\nretrying a few times no matter what\n\nChange-Id: I1b5c2ed35f83b6572db9e919a4af4cd46f0e98e8\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/1f3a44511e3eb97157b7d549dd0b0feec0caa5f8"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/1f3a44511e3eb97157b7d549dd0b0feec0caa5f8"}]},"branch":"refs/heads/master"},"b7af3b650923df80e067e5545108d9cfe4c89f83":{"kind":"NO_CODE_CHANGE","_number":4,"created":"2020-07-23 06:17:43.000000000","uploader":{"_account_id":20172,"name":"Michele Baldessari","email":"michele@acksyn.org","username":"michele"},"ref":"refs/changes/93/742093/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/puppet-pacemaker","ref":"refs/changes/93/742093/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/puppet-pacemaker refs/changes/93/742093/4"}}},"commit":{"parents":[{"commit":"1bb2fe70b1e09dfb04ada5286d86d8d08de48f53","subject":"Fix typo in CRMDIFF_BIN","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/1bb2fe70b1e09dfb04ada5286d86d8d08de48f53"}]}],"author":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-21 08:07:23.000000000","tz":120},"committer":{"name":"Michele Baldessari","email":"michele@acksyn.org","date":"2020-07-23 06:14:19.000000000","tz":120},"subject":"Retry calling backup_cib()","message":"Retry calling backup_cib()\n\nCurrently if backup_cib() fails we just give up instantly.\nIn most cases this is actually okay since there are no chances\nthat the command succeeds in later retries. There are a number of cases\nthough were retrying should be done. IHA FFU is one of them because the\nfollowing might happen:\n1) On the compute node we will try to set up a property when the remote\n   is not yet connected\nJul 21 04:59:45 compute-1 puppet-user[61009]: Debug: Executing: \u0027/sbin/ip6tables-save\u0027\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: warning: Cannot proxy IPC connection from uid 0 gid 0 to cib_rw because not connected to cluster\nJul 21 04:59:46 compute-1 pacemaker-remoted[42459]: error: Error in connection setup (/dev/shm/qb-42459-61225-14-DzDuCR/qb): Remote I/O error (121)\nJul 21 04:59:46 compute-1 puppet-user[61009]: Error: /Stage[main]/Tripleo::Profile::Pacemaker::Compute_instanceha/Pacemaker::Property[compute-instanceha-role-node-property]/Pcmk_property[property-compute-1-compute-instanceha-role]: Could not evaluate: backup_cib: Running: pcs cluster cib /var/lib/pacemaker/cib/puppet-cib-backup20200721-61008-hz2sdw failed with code: 1 -\u003e Error: unable to get cib\n\n2) This happens because in IHA FFU after the control plane has been\n   upgraded we upgrade the compute nodes one by one, without running\n   any other commands on the control plane. So we need to keep retrying\n   setting the property even if backup_cib() fails because eventually\n   the core cluster *will* reconnect to the remote:\nJul 21 05:01:22 compute-1 pacemaker-remoted[42459]: notice: Remote client connection accepted\n\nThat is due to the fact that we cannot necessarily control at what point\nthe core cluster retries the connection, so we should strive for\nretrying a few times no matter what.\n\nTested this 4 times in a row successfully (before this patch we would\nfail quite often)\n\nRelated-Bug: #1888398\n\nChange-Id: I1b5c2ed35f83b6572db9e919a4af4cd46f0e98e8\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/b7af3b650923df80e067e5545108d9cfe4c89f83"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/puppet-pacemaker/commit/b7af3b650923df80e067e5545108d9cfe4c89f83"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}