)]}'
{"id":"openstack%2Fswift~783998","triplet_id":"openstack%2Fswift~master~Ifb3ad7c4794cab047aaf8cb95d915763755435ab","project":"openstack/swift","branch":"master","hashtags":[],"change_id":"Ifb3ad7c4794cab047aaf8cb95d915763755435ab","subject":"probe: Use whole cycles in test_replication_servers_working","status":"ABANDONED","created":"2021-03-31 04:54:42.000000000","updated":"2021-03-31 22:53:56.000000000","total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"fe29e616d97c319e8343bf573196115e00fc7e58","_number":783998,"virtual_id_number":783998,"owner":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"actions":{},"labels":{"Verified":{"recommended":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"tag":"autogenerated:zuul:check","value":1,"date":"2021-03-31 06:13:55.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":1,"default_value":0,"optional":true},"Code-Review":{"disliked":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"all":[{"value":-1,"date":"2021-03-31 15:40:01.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","value":-1,"default_value":0,"optional":true},"Workflow":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2021-03-31 06:13:55.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2021-03-31 15:40:01.000000000","updated_by":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"reviewer":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"state":"REVIEWER"}],"messages":[{"id":"2c8a83ba7edaa5c81831bce60fd3455167e6fa33","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2021-03-31 04:54:42.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"4f1c57ded079ad380f1d81eda24cf984ae990a9a","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2021-03-31 04:56:03.000000000","message":"Patch Set 1:\n\nI *think* this explains the failures on https://review.opendev.org/c/openstack/swift/+/783981 ? Weird, though -- I feel like that ought to be a pretty narrow window.","accounts_in_message":[],"_revision_number":1},{"id":"d7a5f9d518d1dce1774cb4713acf25f92b301ce1","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-03-31 06:13:55.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/3eed4840415140adb96d07a672deddff : SUCCESS in 8m 00s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/ea9f5ceaa6214b66ab378a00fe86044d : SUCCESS in 1h 01m 39s\n- swift-probetests-centos-8 https://zuul.opendev.org/t/openstack/build/f5ee09c390ca43d4948787c2c696a169 : SUCCESS in 40m 10s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/eb1115c34ef147a5956bfa64d4d2d5b8 : SUCCESS in 5m 36s","accounts_in_message":[],"_revision_number":1},{"id":"2b849057e46282f1edb61b18960a260ff5f62bae","author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"date":"2021-03-31 15:40:01.000000000","message":"Patch Set 1: Code-Review-1\n\nThe replicators are only stopped when everything is in sync, so it shouldn\u0027t matter when they are stopped within a cycle should it?\n\nI wonder if the root cause is actually the \u0027random\u0027 do_listdir in the replicator update: on the test_node, the replicator run at line 180 in the test *may* get_hashes with do_listdir, which would cause the hashes.pkl on the test_node to change, so then replicator on another node *will* replicate objects to the test_node.\n\nThis hack will reproduce the test failure at line 188:\n\ndiff --git a/swift/obj/replicator.py b/swift/obj/replicator.py\nindex 7e753d545..9dabc509b 100644\n--- a/swift/obj/replicator.py\n+++ b/swift/obj/replicator.py\n@@ -655,9 +655,7 @@ class ObjectReplicator(Daemon):\n             hashed, local_hash \u003d tpool.execute(\n                 df_mgr._get_hashes, job[\u0027device\u0027],\n                 job[\u0027partition\u0027], job[\u0027policy\u0027],\n-                do_listdir\u003d_do_listdir(\n-                    int(job[\u0027partition\u0027]),\n-                    self.replication_cycle))\n+                do_listdir\u003dTrue)\n             stats.suffix_hash +\u003d hashed\n             self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)\n             attempts_left \u003d len(job[\u0027nodes\u0027])","accounts_in_message":[],"_revision_number":1},{"id":"8ca51c988ad25ef1378adb11ceba4f55e0654ef1","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2021-03-31 22:53:50.000000000","message":"Patch Set 1:\n\nSo at first I was thinking something like\n\n diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py\n index f76121176..f9f2b415c 100644\n --- a/swift/obj/replicator.py\n +++ b/swift/obj/replicator.py\n @@ -473,6 +473,7 @@ class ObjectReplicator(Daemon):\n          if success or not job[\u0027delete\u0027]:\n              headers \u003d dict(self.default_headers)\n              headers[\u0027X-Backend-Storage-Policy-Index\u0027] \u003d int(job[\u0027policy\u0027])\n +            time.sleep(5)\n              with Timeout(self.http_timeout):\n                  conn \u003d http_connect(\n                      node[\u0027replication_ip\u0027], node[\u0027replication_port\u0027],\n\ncould expose the race I was thinking of -- but it didn\u0027t, since we took out the whole partition tree, there was no hashes.invalid until the replicate call. Even something like\n\n diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py\n index d738b66ce..e676902fe 100644\n --- a/swift/obj/diskfile.py\n +++ b/swift/obj/diskfile.py\n @@ -440,6 +440,7 @@ def invalidate_hash(suffix_dir):\n      if not isinstance(suffix, bytes):\n          suffix \u003d suffix.encode(\u0027utf-8\u0027)\n      with lock_path(partition_dir), open(invalidations_file, \u0027ab\u0027) as inv_fh:\n +        time.sleep(5)\n          inv_fh.write(suffix + b\"\\n\")\n \ndidn\u0027t do it, which surprised me a little. Eh -- https://review.opendev.org/c/openstack/swift/+/784108 makes *way* more sense as a root cause; I\u0027m content to abandon this.","accounts_in_message":[],"_revision_number":1},{"id":"fe29e616d97c319e8343bf573196115e00fc7e58","tag":"autogenerated:gerrit:abandon","author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"date":"2021-03-31 22:53:56.000000000","message":"Abandoned","accounts_in_message":[],"_revision_number":1}],"current_revision_number":1,"current_revision":"4738b4600555ae4e0fe4c7505e892cf191e23d7a","revisions":{"4738b4600555ae4e0fe4c7505e892cf191e23d7a":{"kind":"REWORK","_number":1,"created":"2021-03-31 04:54:42.000000000","uploader":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"ref":"refs/changes/98/783998/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/98/783998/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/98/783998/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/98/783998/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/98/783998/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/98/783998/1"}}},"commit":{"parents":[{"commit":"6dd3c7cd153bea7d579d480763e7c2e39f4b8caf","subject":"Merge \"Update docs to discourage policy names being numbers\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/6dd3c7cd153bea7d579d480763e7c2e39f4b8caf"}]}],"author":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2021-03-31 04:53:16.000000000","tz":-420},"committer":{"name":"Tim Burke","email":"tim.burke@gmail.com","date":"2021-03-31 04:53:24.000000000","tz":-420},"subject":"probe: Use whole cycles in test_replication_servers_working","message":"probe: Use whole cycles in test_replication_servers_working\n\nOtherwise, there\u0027s a chance we kill the replicator between when rsync\nhas run and when the REPLICATE request has updated hashes.invalid.\n\nChange-Id: Ifb3ad7c4794cab047aaf8cb95d915763755435ab\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4738b4600555ae4e0fe4c7505e892cf191e23d7a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4738b4600555ae4e0fe4c7505e892cf191e23d7a"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}