)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":12860,"name":"Dongcan Ye","email":"yedongcan@yeah.net","username":"yedongcan"},"change_message_id":"b5a167c2a23689e28bc390ad046cf00ecd4291d9","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"rbd_utils: increase _destroy_volume timeout"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes bug: 1856845"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":12,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_184e2a90","line":9,"range":{"start_line":9,"start_character":0,"end_line":9,"end_character":19},"updated":"2020-02-05 01:54:01.000000000","message":"Please remove this line under L20. And using format like:\nCloses-Bug: #1856845\n\nhttps://docs.openstack.org/infra/manual/developers.html","commit_id":"efde8c4edc2a1d2ed38e014ae8a6daa0cb422199"},{"author":{"_account_id":31575,"name":"Sasha Andonov","email":"sandonov@suse.com","username":"sandonov"},"change_message_id":"a69418b3b000e00c5672712c0ea5f2b0e07f6569","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"rbd_utils: increase _destroy_volume timeout"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Closes bug: 1856845"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":12,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"1fa4df85_10094edc","line":9,"range":{"start_line":9,"start_character":0,"end_line":9,"end_character":19},"in_reply_to":"3fa7e38b_184e2a90","updated":"2020-03-12 19:21:10.000000000","message":"Changed. Thanks!","commit_id":"efde8c4edc2a1d2ed38e014ae8a6daa0cb422199"},{"author":{"_account_id":12860,"name":"Dongcan Ye","email":"yedongcan@yeah.net","username":"yedongcan"},"change_message_id":"b5a167c2a23689e28bc390ad046cf00ecd4291d9","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":12,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":13,"context_line":"second intervals, totaling 10 seconds overall - which, on large"},{"line_number":14,"context_line":"deployments, might result in intermittent object removal failures on"},{"line_number":15,"context_line":"Ceph side."},{"line_number":16,"context_line":"This patch increases the interval between two consecutive Ceph object"},{"line_number":17,"context_line":"removal attempts to 5 seconds, which, combined with total number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"3fa7e38b_98377afe","line":14,"range":{"start_line":13,"start_character":55,"end_line":14,"end_character":11},"updated":"2020-02-05 01:54:01.000000000","message":"How do you think 60 secs satisfied for all large deployments?","commit_id":"efde8c4edc2a1d2ed38e014ae8a6daa0cb422199"},{"author":{"_account_id":31575,"name":"Sasha Andonov","email":"sandonov@suse.com","username":"sandonov"},"change_message_id":"a69418b3b000e00c5672712c0ea5f2b0e07f6569","unresolved":false,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":12,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":13,"context_line":"second intervals, totaling 10 seconds overall - which, on large"},{"line_number":14,"context_line":"deployments, might result in intermittent object removal failures on"},{"line_number":15,"context_line":"Ceph side."},{"line_number":16,"context_line":"This patch increases the interval between two consecutive Ceph object"},{"line_number":17,"context_line":"removal attempts to 5 seconds, which, combined with total number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"1fa4df85_f00352bd","line":14,"range":{"start_line":13,"start_character":55,"end_line":14,"end_character":11},"in_reply_to":"3fa7e38b_98377afe","updated":"2020-03-12 19:21:10.000000000","message":"Sorry, that was wrong wording. When image removal fails with rbd.ImageBusy, watcher on ceph side is still present, therefore the failure. Since it times out after thirty seconds [1] - sixty second loop seems generous enough. \n\n[1] https://github.com/ceph/ceph/blob/master/src/include/rados/librados.h#L2465","commit_id":"efde8c4edc2a1d2ed38e014ae8a6daa0cb422199"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"1cc2d979aa0fd3bac77aab5658bfcdbeea6342ac","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":10,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":11,"context_line":"second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":12,"context_line":"second ceph watcher timeout, might result in intermittent object"},{"line_number":13,"context_line":"removal failures on Ceph side."},{"line_number":14,"context_line":"This patch increases the interval between two consecutive Ceph object"},{"line_number":15,"context_line":"removal attempts to 5 seconds, which, combined with total number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"1fa4df85_b03f1a86","line":12,"range":{"start_line":11,"start_character":64,"end_line":12,"end_character":27},"updated":"2020-03-12 18:41:14.000000000","message":"Is this saying that ceph watcher (part of ceph) has its own timeout that defaults to 30 seconds, and that is currently incompatible with our timeout on the nova side and is thus causing problems?","commit_id":"a47ba740b8afecfd66e9f54b2c8e095d35201cce"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"bdb0b1ba87688621e3345e8cee71b990033d9b07","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":10,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":11,"context_line":"second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":12,"context_line":"second ceph watcher timeout, might result in intermittent object"},{"line_number":13,"context_line":"removal failures on Ceph side."},{"line_number":14,"context_line":"This patch increases the interval between two consecutive Ceph object"},{"line_number":15,"context_line":"removal attempts to 5 seconds, which, combined with total number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"1fa4df85_fb0333bd","line":12,"range":{"start_line":11,"start_character":64,"end_line":12,"end_character":27},"in_reply_to":"1fa4df85_309d6a66","updated":"2020-03-12 19:42:59.000000000","message":"so i was going to coment on this in the code but ill do it here.\n\nthe fact we need to update this because of a configuratble option in ceph make me think we should have a config option in nova as well. maybe in the workarounds section","commit_id":"a47ba740b8afecfd66e9f54b2c8e095d35201cce"},{"author":{"_account_id":31575,"name":"Sasha Andonov","email":"sandonov@suse.com","username":"sandonov"},"change_message_id":"a69418b3b000e00c5672712c0ea5f2b0e07f6569","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":10,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":11,"context_line":"second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":12,"context_line":"second ceph watcher timeout, might result in intermittent object"},{"line_number":13,"context_line":"removal failures on Ceph side."},{"line_number":14,"context_line":"This patch increases the interval between two consecutive Ceph object"},{"line_number":15,"context_line":"removal attempts to 5 seconds, which, combined with total number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"1fa4df85_309d6a66","line":12,"range":{"start_line":11,"start_character":64,"end_line":12,"end_character":27},"in_reply_to":"1fa4df85_b03f1a86","updated":"2020-03-12 19:21:10.000000000","message":"Unfortunately I am not a ceph expert. However, according to [2], the timeout is configurable, although in that case ceph admin who changes the timeout bears the responsibility if watcher doesn\u0027t timeout in time. According to [1], default is 30 seconds. I can also confirm the patch fixed the issue on a production system, and is being stable as of today.\n\n[1] https://github.com/ceph/ceph/blob/master/src/include/rados/librados.h#L2465\n[2] https://github.com/ceph/ceph/blob/master/src/osd/PrimaryLogPG.cc#L6677","commit_id":"a47ba740b8afecfd66e9f54b2c8e095d35201cce"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e436f04655bbb671d7a0b50eb46ba18f4e30524e","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":10,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":11,"context_line":"second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":12,"context_line":"second ceph watcher timeout, might result in intermittent object"},{"line_number":13,"context_line":"removal failures on Ceph side."},{"line_number":14,"context_line":"This patch increases the interval between two consecutive Ceph object"},{"line_number":15,"context_line":"removal attempts to 5 seconds, which, combined with total number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"1fa4df85_a1202eb2","line":12,"range":{"start_line":11,"start_character":64,"end_line":12,"end_character":27},"in_reply_to":"1fa4df85_b6e8727c","updated":"2020-03-12 23:08:07.000000000","message":"i only suggested wrokaround as i was not sure if it was a ligitamet tunabel we would want to expsoe.\n\nbut it felt like it should be a config option to me.\nwhen i was thinking about if it should be under compute or libvirt i was not sure if other virt divers supported cpeh so i decided to suggest workaround since i did not really know where to put it.\n\nno reason other then that.","commit_id":"a47ba740b8afecfd66e9f54b2c8e095d35201cce"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"805b8c323344c1f30a9e0eabf0d677142c0d4fde","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":10,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":11,"context_line":"second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":12,"context_line":"second ceph watcher timeout, might result in intermittent object"},{"line_number":13,"context_line":"removal failures on Ceph side."},{"line_number":14,"context_line":"This patch increases the interval between two consecutive Ceph object"},{"line_number":15,"context_line":"removal attempts to 5 seconds, which, combined with total number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"1fa4df85_b6e8727c","line":12,"range":{"start_line":11,"start_character":64,"end_line":12,"end_character":27},"in_reply_to":"1fa4df85_fb0333bd","updated":"2020-03-12 21:16:48.000000000","message":"Why a workaround? It seems like workarounds has become abused for anything and everything.\n\nIf it\u0027s a legitimate tunable, then don\u0027t put it into workarounds. Workarounds are supposed fo be for things like \"some distro libvirt libraries haven\u0027t been updated to account for bug X so add a timer or flag or something to allow WORKINGAROUND that thing for the time being.\"\n\nAgree that if it\u0027s tunable on the ceph side, it needs to be tunable on the nova side, unless there is some way we can discover what the ceph tunable is and calculate 2*$ceph value or something.","commit_id":"a47ba740b8afecfd66e9f54b2c8e095d35201cce"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":7,"context_line":"rbd_utils: increase _destroy_volume timeout"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"If RBD backend is used for Nova ephemeral storage, Nova tries to remove"},{"line_number":10,"context_line":"ephemeral storage object from Ceph in a retry loop: 10 attempts at 1"},{"line_number":11,"context_line":"second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":12,"context_line":"second ceph watcher timeout, might result in intermittent object"},{"line_number":13,"context_line":"removal failures on Ceph side."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"ff570b3c_b0005003","line":10,"range":{"start_line":10,"start_character":18,"end_line":10,"end_character":24},"updated":"2020-05-15 13:36:11.000000000","message":"nit - I appreciate these are objects in the backend of ceph but within Nova we refer to rbd volumes. Just to keep future reviewers sane do you think you could s/object/volume/g in this change?","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"}],"nova/conf/libvirt.py":[{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"920d7ca27a974fc363d0d9cf5d24efc896215a3c","unresolved":false,"context_lines":[{"line_number":1097,"context_line":"The RADOS client timeout in seconds when initially connecting to the cluster."},{"line_number":1098,"context_line":"\"\"\"),"},{"line_number":1099,"context_line":"     cfg.IntOpt(\u0027rbd_destroy_volume_retry_interval\u0027,"},{"line_number":1100,"context_line":"               default\u003d5,"},{"line_number":1101,"context_line":"               help\u003d\"\"\""},{"line_number":1102,"context_line":"Number of seconds to wait between each consecutive retry to destroy a"},{"line_number":1103,"context_line":"RBD volume."},{"line_number":1104,"context_line":"\"\"\"),"}],"source_content_type":"text/x-python","patch_set":3,"id":"df33271e_558470d4","line":1101,"range":{"start_line":1100,"start_character":15,"end_line":1101,"end_character":23},"updated":"2020-03-26 20:05:16.000000000","message":"Indention here is off by 1.","commit_id":"dd3141c3369f1054b301a012ae4e7249fe6d9771"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"920d7ca27a974fc363d0d9cf5d24efc896215a3c","unresolved":false,"context_lines":[{"line_number":1103,"context_line":"RBD volume."},{"line_number":1104,"context_line":"\"\"\"),"},{"line_number":1105,"context_line":"     cfg.IntOpt(\u0027rbd_destroy_volume_retries\u0027,"},{"line_number":1106,"context_line":"               default\u003d12,"},{"line_number":1107,"context_line":"               help\u003d\"\"\""},{"line_number":1108,"context_line":"Number of retries to destroy a RBD volume."},{"line_number":1109,"context_line":"\"\"\"),"},{"line_number":1110,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":3,"id":"df33271e_956d5834","line":1107,"range":{"start_line":1106,"start_character":15,"end_line":1107,"end_character":23},"updated":"2020-03-26 20:05:16.000000000","message":"Indention here is off by 1.","commit_id":"dd3141c3369f1054b301a012ae4e7249fe6d9771"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"920d7ca27a974fc363d0d9cf5d24efc896215a3c","unresolved":false,"context_lines":[{"line_number":1106,"context_line":"               default\u003d12,"},{"line_number":1107,"context_line":"               help\u003d\"\"\""},{"line_number":1108,"context_line":"Number of retries to destroy a RBD volume."},{"line_number":1109,"context_line":"\"\"\"),"},{"line_number":1110,"context_line":"]"},{"line_number":1111,"context_line":""},{"line_number":1112,"context_line":"libvirt_volume_nfs_opts \u003d ["}],"source_content_type":"text/x-python","patch_set":3,"id":"df33271e_d5224000","line":1109,"updated":"2020-03-26 20:05:16.000000000","message":"It might be a good idea to put minimums on these, like 0? Otherwise they could be set as negative integers.","commit_id":"dd3141c3369f1054b301a012ae4e7249fe6d9771"},{"author":{"_account_id":31575,"name":"Sasha Andonov","email":"sandonov@suse.com","username":"sandonov"},"change_message_id":"087f6675168eb29620e8ef015d17cd309b0883bb","unresolved":false,"context_lines":[{"line_number":1106,"context_line":"               default\u003d12,"},{"line_number":1107,"context_line":"               help\u003d\"\"\""},{"line_number":1108,"context_line":"Number of retries to destroy a RBD volume."},{"line_number":1109,"context_line":"\"\"\"),"},{"line_number":1110,"context_line":"]"},{"line_number":1111,"context_line":""},{"line_number":1112,"context_line":"libvirt_volume_nfs_opts \u003d ["}],"source_content_type":"text/x-python","patch_set":3,"id":"df33271e_4ebfd7fa","line":1109,"in_reply_to":"df33271e_d5224000","updated":"2020-04-07 11:49:27.000000000","message":"Agreed, thanks","commit_id":"dd3141c3369f1054b301a012ae4e7249fe6d9771"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":1096,"context_line":"               help\u003d\"\"\""},{"line_number":1097,"context_line":"The RADOS client timeout in seconds when initially connecting to the cluster."},{"line_number":1098,"context_line":"\"\"\"),"},{"line_number":1099,"context_line":"    cfg.IntOpt(\u0027rbd_destroy_volume_retry_interval\u0027,"},{"line_number":1100,"context_line":"               default\u003d5,"},{"line_number":1101,"context_line":"               min\u003d0,"},{"line_number":1102,"context_line":"               help\u003d\"\"\""},{"line_number":1103,"context_line":"Number of seconds to wait between each consecutive retry to destroy a"},{"line_number":1104,"context_line":"RBD volume."},{"line_number":1105,"context_line":"\"\"\"),"},{"line_number":1106,"context_line":"    cfg.IntOpt(\u0027rbd_destroy_volume_retries\u0027,"},{"line_number":1107,"context_line":"               default\u003d12,"},{"line_number":1108,"context_line":"               min\u003d0,"},{"line_number":1109,"context_line":"               help\u003d\"\"\""},{"line_number":1110,"context_line":"Number of retries to destroy a RBD volume."},{"line_number":1111,"context_line":"\"\"\"),"},{"line_number":1112,"context_line":"]"},{"line_number":1113,"context_line":""},{"line_number":1114,"context_line":"libvirt_volume_nfs_opts \u003d ["}],"source_content_type":"text/x-python","patch_set":6,"id":"ff570b3c_ed3831ef","line":1111,"range":{"start_line":1099,"start_character":0,"end_line":1111,"end_character":5},"updated":"2020-05-15 13:36:11.000000000","message":"Can you add that this only relates to the rbd imagebackend?\n\n  Related options:\n\n  * [libvirt]/images_type \u003d \u0027rbd\u0027","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"}],"nova/tests/unit/virt/libvirt/storage/test_rbd.py":[{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"920d7ca27a974fc363d0d9cf5d24efc896215a3c","unresolved":false,"context_lines":[{"line_number":524,"context_line":"        # Six requests failed, seventh successful"},{"line_number":525,"context_line":"        rbd.remove.assert_has_calls([remove_call] * 6)"},{"line_number":526,"context_line":"        client.__exit__.assert_called_once_with(None, None, None)"},{"line_number":527,"context_line":""},{"line_number":528,"context_line":"    @mock.patch.object(rbd_utils, \u0027RADOSClient\u0027)"},{"line_number":529,"context_line":"    def test_remove_image(self, mock_client):"},{"line_number":530,"context_line":"        name \u003d \u002712345_disk.config.rescue\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"df33271e_75d6d4c5","line":527,"updated":"2020-03-26 20:05:16.000000000","message":"Is this test going to take 30 seconds to run? If so, can we condense it to be faster by using self.flags() to set the tunables lower at the beginning?\n\nOur gate runs take a lot of time and lots of 30 seconds added together makes the time significant.\n\nAnother thing I was thinking is you could do a different test instead of this one where it just verifies that after setting self.flags() for the tunables that the expected values are passed to a mocked FixedIntervalLoopingCall start and _cleanup_vol. That would be obviously the fastest test but perhaps you were going for more of a functional test here rather than a unit test?","commit_id":"dd3141c3369f1054b301a012ae4e7249fe6d9771"},{"author":{"_account_id":31575,"name":"Sasha Andonov","email":"sandonov@suse.com","username":"sandonov"},"change_message_id":"087f6675168eb29620e8ef015d17cd309b0883bb","unresolved":false,"context_lines":[{"line_number":524,"context_line":"        # Six requests failed, seventh successful"},{"line_number":525,"context_line":"        rbd.remove.assert_has_calls([remove_call] * 6)"},{"line_number":526,"context_line":"        client.__exit__.assert_called_once_with(None, None, None)"},{"line_number":527,"context_line":""},{"line_number":528,"context_line":"    @mock.patch.object(rbd_utils, \u0027RADOSClient\u0027)"},{"line_number":529,"context_line":"    def test_remove_image(self, mock_client):"},{"line_number":530,"context_line":"        name \u003d \u002712345_disk.config.rescue\u0027"}],"source_content_type":"text/x-python","patch_set":3,"id":"df33271e_6e629baf","line":527,"in_reply_to":"df33271e_75d6d4c5","updated":"2020-04-07 11:49:27.000000000","message":"Yes, it took 30 seconds to finish, but you\u0027re right - no need for such a long test. Mocking FixedIntervalLoopingCall is definitely a better approach. I made the changes in patch set 4. Thanks a lot for the review.","commit_id":"dd3141c3369f1054b301a012ae4e7249fe6d9771"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"f7e31bd2ca4b0022b3bf1f56a174d1172f95665d","unresolved":false,"context_lines":[{"line_number":528,"context_line":"        # Make sure that we entered and exited the RADOSClient"},{"line_number":529,"context_line":"        client.__enter__.assert_called_once_with()"},{"line_number":530,"context_line":"        client.__exit__.assert_called_once_with(None, None, None)"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"    @mock.patch.object(rbd_utils, \u0027RADOSClient\u0027)"},{"line_number":533,"context_line":"    def test_remove_image(self, mock_client):"},{"line_number":534,"context_line":"        name \u003d \u002712345_disk.config.rescue\u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff570b3c_80c48c41","line":531,"updated":"2020-05-20 00:36:52.000000000","message":"Nice, this test is running fast now [1]:\n\n test_destroy_volume_with_retries [0.040598s] ... ok\n\nhttps://zuul.opendev.org/t/openstack/build/5a5dfd518c984dacb048f687d519d542/log/job-output.txt#16901","commit_id":"6458c3dba53b9a9fb903bdb6e5e08af14ad015d6"}],"nova/virt/libvirt/storage/rbd_utils.py":[{"author":{"_account_id":12860,"name":"Dongcan Ye","email":"yedongcan@yeah.net","username":"yedongcan"},"change_message_id":"b5a167c2a23689e28bc390ad046cf00ecd4291d9","unresolved":false,"context_lines":[{"line_number":351,"context_line":"            if retryctx[\u0027retries\u0027] \u003c\u003d 0:"},{"line_number":352,"context_line":"                raise loopingcall.LoopingCallDone()"},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"        # NOTE(danms): We let it go for ten seconds"},{"line_number":355,"context_line":"        retryctx \u003d {\u0027retries\u0027: 12}"},{"line_number":356,"context_line":"        timer \u003d loopingcall.FixedIntervalLoopingCall("},{"line_number":357,"context_line":"            _cleanup_vol, client.ioctx, volume, retryctx)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3fa7e38b_381e8675","line":354,"range":{"start_line":354,"start_character":40,"end_line":354,"end_character":43},"updated":"2020-02-05 01:54:01.000000000","message":"If you change the retry and interval, this should also need update.","commit_id":"efde8c4edc2a1d2ed38e014ae8a6daa0cb422199"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"bdb0b1ba87688621e3345e8cee71b990033d9b07","unresolved":false,"context_lines":[{"line_number":351,"context_line":"            if retryctx[\u0027retries\u0027] \u003c\u003d 0:"},{"line_number":352,"context_line":"                raise loopingcall.LoopingCallDone()"},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"        # NOTE(danms): We let it go for sixty seconds"},{"line_number":355,"context_line":"        retryctx \u003d {\u0027retries\u0027: 12}"},{"line_number":356,"context_line":"        timer \u003d loopingcall.FixedIntervalLoopingCall("},{"line_number":357,"context_line":"            _cleanup_vol, client.ioctx, volume, retryctx)"},{"line_number":358,"context_line":"        timed_out \u003d timer.start(interval\u003d5).wait()"}],"source_content_type":"text/x-python","patch_set":2,"id":"1fa4df85_3b6c6b9b","line":355,"range":{"start_line":354,"start_character":9,"end_line":355,"end_character":34},"updated":"2020-03-12 19:42:59.000000000","message":"you proably shoudl update the node to be your name instead.","commit_id":"a47ba740b8afecfd66e9f54b2c8e095d35201cce"}],"releasenotes/notes/rbd-increase-timeout-c4e5a34cf5da7fdc.yaml":[{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"bb5dc9910c23648f96f96bfc6736ca994f5c4143","unresolved":false,"context_lines":[{"line_number":9,"context_line":"fixes:"},{"line_number":10,"context_line":"  - |"},{"line_number":11,"context_line":"    Nova tries to remove an object from Ceph in a retry loop of 10 attempts at"},{"line_number":12,"context_line":"    1 second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":13,"context_line":"    second ceph watcher timeout, might result in intermittent object removal"},{"line_number":14,"context_line":"    failures on Ceph side. Setting default values for params "},{"line_number":15,"context_line":"    rbd_destroy_volume_retries to 12 and rbd_destroy_volume_retry_interval to"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"ff570b3c_511cc869","line":12,"range":{"start_line":12,"start_character":70,"end_line":12,"end_character":76},"updated":"2020-05-11 10:17:00.000000000","message":"30","commit_id":"1caf9a6bada6ed48aa4057e5f6421b9b039b6d40"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"bb5dc9910c23648f96f96bfc6736ca994f5c4143","unresolved":false,"context_lines":[{"line_number":11,"context_line":"    Nova tries to remove an object from Ceph in a retry loop of 10 attempts at"},{"line_number":12,"context_line":"    1 second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":13,"context_line":"    second ceph watcher timeout, might result in intermittent object removal"},{"line_number":14,"context_line":"    failures on Ceph side. Setting default values for params "},{"line_number":15,"context_line":"    rbd_destroy_volume_retries to 12 and rbd_destroy_volume_retry_interval to"},{"line_number":16,"context_line":"    5, now gives Ceph reasonable amount of time to complete the operation"},{"line_number":17,"context_line":"    successfully."}],"source_content_type":"text/x-yaml","patch_set":5,"id":"ff570b3c_b10d6439","line":14,"range":{"start_line":14,"start_character":60,"end_line":14,"end_character":61},"updated":"2020-05-11 10:17:00.000000000","message":"whoops","commit_id":"1caf9a6bada6ed48aa4057e5f6421b9b039b6d40"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"bb5dc9910c23648f96f96bfc6736ca994f5c4143","unresolved":false,"context_lines":[{"line_number":12,"context_line":"    1 second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":13,"context_line":"    second ceph watcher timeout, might result in intermittent object removal"},{"line_number":14,"context_line":"    failures on Ceph side. Setting default values for params "},{"line_number":15,"context_line":"    rbd_destroy_volume_retries to 12 and rbd_destroy_volume_retry_interval to"},{"line_number":16,"context_line":"    5, now gives Ceph reasonable amount of time to complete the operation"},{"line_number":17,"context_line":"    successfully."},{"line_number":18,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":5,"id":"ff570b3c_11125059","line":15,"range":{"start_line":15,"start_character":4,"end_line":15,"end_character":30},"updated":"2020-05-11 10:17:00.000000000","message":"``rbd_destroy_volume_retries``","commit_id":"1caf9a6bada6ed48aa4057e5f6421b9b039b6d40"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"bb5dc9910c23648f96f96bfc6736ca994f5c4143","unresolved":false,"context_lines":[{"line_number":12,"context_line":"    1 second intervals, totaling 10 seconds overall - which, due to a thirty"},{"line_number":13,"context_line":"    second ceph watcher timeout, might result in intermittent object removal"},{"line_number":14,"context_line":"    failures on Ceph side. Setting default values for params "},{"line_number":15,"context_line":"    rbd_destroy_volume_retries to 12 and rbd_destroy_volume_retry_interval to"},{"line_number":16,"context_line":"    5, now gives Ceph reasonable amount of time to complete the operation"},{"line_number":17,"context_line":"    successfully."},{"line_number":18,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":5,"id":"ff570b3c_f10ebc30","line":15,"range":{"start_line":15,"start_character":41,"end_line":15,"end_character":74},"updated":"2020-05-11 10:17:00.000000000","message":"``rbd_destroy_volume_retry_interval``","commit_id":"1caf9a6bada6ed48aa4057e5f6421b9b039b6d40"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":1,"context_line":"---"},{"line_number":2,"context_line":"features:"},{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    Added params ``rbd_destroy_volume_retries``, defaulting to 12, and"},{"line_number":5,"context_line":"    ``rbd_destroy_volume_retry_interval``, defaulting to 5, which Nova will"},{"line_number":6,"context_line":"    use when trying to remove an object from Ceph in a retry loop that"},{"line_number":7,"context_line":"    combines these parameters together. Thus, maximum elapsing time is by"}],"source_content_type":"text/x-yaml","patch_set":6,"id":"ff570b3c_6d2c21ac","line":4,"range":{"start_line":4,"start_character":19,"end_line":4,"end_character":45},"updated":"2020-05-15 13:36:11.000000000","message":"[libvirt]/rbd_destroy_volume_retries","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":2,"context_line":"features:"},{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    Added params ``rbd_destroy_volume_retries``, defaulting to 12, and"},{"line_number":5,"context_line":"    ``rbd_destroy_volume_retry_interval``, defaulting to 5, which Nova will"},{"line_number":6,"context_line":"    use when trying to remove an object from Ceph in a retry loop that"},{"line_number":7,"context_line":"    combines these parameters together. Thus, maximum elapsing time is by"},{"line_number":8,"context_line":"    default 60 seconds."}],"source_content_type":"text/x-yaml","patch_set":6,"id":"ff570b3c_ad1bd97e","line":5,"range":{"start_line":5,"start_character":6,"end_line":5,"end_character":39},"updated":"2020-05-15 13:36:11.000000000","message":"[libvirt]/rbd_destroy_volume_retry_interval","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":3,"context_line":"  - |"},{"line_number":4,"context_line":"    Added params ``rbd_destroy_volume_retries``, defaulting to 12, and"},{"line_number":5,"context_line":"    ``rbd_destroy_volume_retry_interval``, defaulting to 5, which Nova will"},{"line_number":6,"context_line":"    use when trying to remove an object from Ceph in a retry loop that"},{"line_number":7,"context_line":"    combines these parameters together. Thus, maximum elapsing time is by"},{"line_number":8,"context_line":"    default 60 seconds."},{"line_number":9,"context_line":"fixes:"}],"source_content_type":"text/x-yaml","patch_set":6,"id":"ff570b3c_b04e906c","line":6,"range":{"start_line":6,"start_character":33,"end_line":6,"end_character":39},"updated":"2020-05-15 13:36:11.000000000","message":"volume","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":8,"context_line":"    default 60 seconds."},{"line_number":9,"context_line":"fixes:"},{"line_number":10,"context_line":"  - |"},{"line_number":11,"context_line":"    Nova tries to remove an object from Ceph in a retry loop of 10 attempts at"},{"line_number":12,"context_line":"    1 second intervals, totaling 10 seconds overall - which, due to 30 second"},{"line_number":13,"context_line":"    ceph watcher timeout, might result in intermittent object removal failures"},{"line_number":14,"context_line":"    on Ceph side. Setting default values for ``rbd_destroy_volume_retries`` to"}],"source_content_type":"text/x-yaml","patch_set":6,"id":"ff570b3c_f0548800","line":11,"range":{"start_line":11,"start_character":28,"end_line":11,"end_character":34},"updated":"2020-05-15 13:36:11.000000000","message":"volume","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":11,"context_line":"    Nova tries to remove an object from Ceph in a retry loop of 10 attempts at"},{"line_number":12,"context_line":"    1 second intervals, totaling 10 seconds overall - which, due to 30 second"},{"line_number":13,"context_line":"    ceph watcher timeout, might result in intermittent object removal failures"},{"line_number":14,"context_line":"    on Ceph side. Setting default values for ``rbd_destroy_volume_retries`` to"},{"line_number":15,"context_line":"    12 and ``rbd_destroy_volume_retry_interval`` to 5, now gives Ceph"},{"line_number":16,"context_line":"    reasonable amount of time to complete the operation successfully."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":6,"id":"ff570b3c_d024e4b1","line":14,"range":{"start_line":14,"start_character":47,"end_line":14,"end_character":73},"updated":"2020-05-15 13:36:11.000000000","message":"as above","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"},{"author":{"_account_id":10135,"name":"Lee Yarwood","display_name":"Lee Yarwood","email":"lyarwood@redhat.com","username":"lyarwood"},"change_message_id":"96f1fd023ae3de5ea54842b66e0c34bb6f961d64","unresolved":false,"context_lines":[{"line_number":12,"context_line":"    1 second intervals, totaling 10 seconds overall - which, due to 30 second"},{"line_number":13,"context_line":"    ceph watcher timeout, might result in intermittent object removal failures"},{"line_number":14,"context_line":"    on Ceph side. Setting default values for ``rbd_destroy_volume_retries`` to"},{"line_number":15,"context_line":"    12 and ``rbd_destroy_volume_retry_interval`` to 5, now gives Ceph"},{"line_number":16,"context_line":"    reasonable amount of time to complete the operation successfully."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"    .. _`bug 1856845`: https://bugs.launchpad.net/nova/+bug/1856845"}],"source_content_type":"text/x-yaml","patch_set":6,"id":"ff570b3c_303d60c4","line":15,"range":{"start_line":15,"start_character":13,"end_line":15,"end_character":46},"updated":"2020-05-15 13:36:11.000000000","message":"as above","commit_id":"04a3af0170504060bffca43bc627a3d27dad7e7d"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"23366ea26e33a98048dfb446693af3cf1152d6e9","unresolved":false,"context_lines":[{"line_number":16,"context_line":"    ``[libvirt]/rbd_destroy_volume_retry_interval`` to 5, now gives Ceph"},{"line_number":17,"context_line":"    reasonable amount of time to complete the operation successfully."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"    .. _`bug 1856845`: https://bugs.launchpad.net/nova/+bug/1856845"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"ff570b3c_3e3de6dc","line":19,"updated":"2020-05-20 00:06:36.000000000","message":"You\u0027ve not referred to the bug in any of the text, so your intention to link it has not worked and it doesn\u0027t show up in the release notes:\n\nhttps://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c0e/705764/7/check/build-openstack-releasenotes/c0e94e2/docs/unreleased.html\n\nYou can either refer to it somewhere in the fixes text by for example: \"object removal failures on the Ceph side (`bug 1856845`_)\"\n\nExample:\n\nhttps://github.com/openstack/nova/blob/master/releasenotes/notes/bug-1414895-8f7d8da6499f8e94.yaml","commit_id":"11359c0b5b8e58e50a58f9b417123e8c28d12c72"}]}