)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"change_message_id":"b7bddd2050b01293154e3a80cddf1af808bcacd7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"7a357e20_ca232633","updated":"2022-05-26 09:01:38.000000000","message":"Please, could you approve this ? ","commit_id":"074e79384c2488036b6616469b36bdf11c2afeef"},{"author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"change_message_id":"86370e625091f336c8d15b8096a808017a99e7e4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"2bfe72e5_d0d548b8","updated":"2022-05-26 12:59:34.000000000","message":"Thank you guys, just for info :\n\nFrom time to time cinder-volume goes crazy and start to spam this below : \nWe have a theory it\u0027s something with default timeouts, so we added \u0026timeout\u003d120 which should workarodund issue, but it\u0027s short time to analyze it.\n\nI didn\u0027t changed default value in k-a, but I think it\u0027s good to have possibility to override connection string to redis.\n\nSo, once again, thanks.\n\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis Traceback (most recent call last):\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/tooz/drivers/redis.py\", line 58, in wrapper\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     return func(*args, **kwargs)\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/tooz/drivers/redis.py\", line 132, in heartbeat\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     self._lock.reacquire()\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/redis/lock.py\", line 265, in reacquire\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     return self.do_reacquire()\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/redis/lock.py\", line 272, in do_reacquire\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     raise LockNotOwnedError(\"Cannot reacquire a lock that\u0027s\"\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis redis.exceptions.LockNotOwnedError: Cannot reacquire a lock that\u0027s no longer owned\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis The above exception was the direct cause of the following exception:\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis Traceback (most recent call last):\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/tooz/drivers/redis.py\", line 544, in heartbeat\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     lock.heartbeat()\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/tooz/drivers/redis.py\", line 74, in wrapper\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     cause\u003de)\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/tooz/utils.py\", line 224, in raise_with_cause\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     excutils.raise_with_cause(exc_cls, message, *args, **kwargs)\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"/usr/lib/python3/dist-packages/oslo_utils/excutils.py\", line 143, in raise_with_cause\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis     six.raise_from(exc_cls(message, *args, **kwargs), kwargs.get(\u0027cause\u0027))\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis   File \"\u003cstring\u003e\", line 3, in raise_from\n2022-04-27 16:19:12.173 829 ERROR tooz.drivers.redis tooz.ToozError: Cannot reacquire a lock that\u0027s no longer owned","commit_id":"71ea62543afe17002656d7f18d4651e54089ca51"},{"author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"change_message_id":"ba6727d171b0dca839605d366b8ce3464b06ac6d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"3d7f2024_73d8cbc6","in_reply_to":"2bfe72e5_d0d548b8","updated":"2022-05-26 13:02:17.000000000","message":"Some test to replicate :\n\nfrom tooz.coordination import get_coordinator\n\ncoordinator \u003d get_coordinator(\"redis://172.17.0.3:26379?sentinel\u003dmymaster\u0026db\u003d0\u0026socket_timeout\u003d60\u0026retry_on_timeout\u003dyes\", \"MyUnique\")\ncoordinator.start(start_heart\u003dTrue)\nprint(\"before\")\nlock \u003d coordinator.get_lock(\"myLock\")\nwith lock(True):\n    second_lock \u003d coordinator.get_lock(\"myLock\")\n    with second_lock(True):\n        print(\"after\")\n\n\n\nsudo ip link set down  veth53584a8 ; sleep 60 ; sudo ip link set up  veth53584a8\n\n\n\n\n\n\n\nTraceback (most recent call last):\n  File \"/home/michalarbet/PycharmProjects/toozTest/venv/lib/python3.9/site-packages/tooz/drivers/redis.py\", line 544, in heartbeat\n    lock.heartbeat()\n  File \"/home/michalarbet/PycharmProjects/toozTest/venv/lib/python3.9/site-packages/tooz/drivers/redis.py\", line 72, in wrapper\n    utils.raise_with_cause(tooz.ToozError,\n  File \"/home/michalarbet/PycharmProjects/toozTest/venv/lib/python3.9/site-packages/tooz/utils.py\", line 224, in raise_with_cause\n    excutils.raise_with_cause(exc_cls, message, *args, **kwargs)\n  File \"/home/michalarbet/PycharmProjects/toozTest/venv/lib/python3.9/site-packages/oslo_utils/excutils.py\", line 142, in raise_with_cause\n    raise exc_cls(message, *args, **kwargs) from kwargs.get(\u0027cause\u0027)\ntooz.ToozError: Cannot reacquire a lock that\u0027s no longer owned","commit_id":"71ea62543afe17002656d7f18d4651e54089ca51"}],"ansible/group_vars/all.yml":[{"author":{"_account_id":22629,"name":"Michal Nasiadka","email":"mnasiadka@gmail.com","username":"mnasiadka"},"change_message_id":"a48b3596f357e6f7abc54b5c0c6613f2a301e84c","unresolved":true,"context_lines":[{"line_number":783,"context_line":"# Redis options"},{"line_number":784,"context_line":"####################"},{"line_number":785,"context_line":"redis_connection_string: \"redis://{% for host in groups[\u0027redis\u0027] %}{% if host \u003d\u003d groups[\u0027redis\u0027][0] %}admin:{{ redis_master_password }}@{{ \u0027api\u0027 | kolla_address(host) | put_address_in_context(\u0027url\u0027) }}:{{ redis_sentinel_port }}?sentinel\u003dkolla{% else %}\u0026sentinel_fallback\u003d{{ \u0027api\u0027 | kolla_address(host) | put_address_in_context(\u0027url\u0027) }}:{{ redis_sentinel_port }}{% endif %}{% endfor %}{{ redis_cmdline_extras }}\""},{"line_number":786,"context_line":"redis_cmdline_extras: \"\u0026db\u003d0\u0026socket_timeout\u003d60\u0026retry_on_timeout\u003dyes\""},{"line_number":787,"context_line":""},{"line_number":788,"context_line":"####################"},{"line_number":789,"context_line":"# Osprofiler options"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"72deccf5_bd01d141","line":786,"range":{"start_line":786,"start_character":6,"end_line":786,"end_character":13},"updated":"2022-04-26 14:49:52.000000000","message":"cmdline? not used anywhere else than redis_connection_string - out of the blue seems a bit incorrect\nAnd you\u0027re changing the default - some better explanation and probably a reno needed.","commit_id":"074e79384c2488036b6616469b36bdf11c2afeef"},{"author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"change_message_id":"f455498f5fd34dd206853a33a027a557d78a95e8","unresolved":true,"context_lines":[{"line_number":783,"context_line":"# Redis options"},{"line_number":784,"context_line":"####################"},{"line_number":785,"context_line":"redis_connection_string: \"redis://{% for host in groups[\u0027redis\u0027] %}{% if host \u003d\u003d groups[\u0027redis\u0027][0] %}admin:{{ redis_master_password }}@{{ \u0027api\u0027 | kolla_address(host) | put_address_in_context(\u0027url\u0027) }}:{{ redis_sentinel_port }}?sentinel\u003dkolla{% else %}\u0026sentinel_fallback\u003d{{ \u0027api\u0027 | kolla_address(host) | put_address_in_context(\u0027url\u0027) }}:{{ redis_sentinel_port }}{% endif %}{% endfor %}{{ redis_cmdline_extras }}\""},{"line_number":786,"context_line":"redis_cmdline_extras: \"\u0026db\u003d0\u0026socket_timeout\u003d60\u0026retry_on_timeout\u003dyes\""},{"line_number":787,"context_line":""},{"line_number":788,"context_line":"####################"},{"line_number":789,"context_line":"# Osprofiler options"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"d661c191_bf4afc78","line":786,"range":{"start_line":786,"start_character":6,"end_line":786,"end_character":13},"in_reply_to":"72deccf5_bd01d141","updated":"2022-04-26 15:00:27.000000000","message":"Well, ok, maybe the name of value is really bad, maybe better name is redis_connection_string_extras.\n\nWell, we found that (probably) there is need to add \"\u0026timeout\u003d120\" to have redis more stable, from time to time we have some problems with redis.\n\nAs we are still testing if socket_timeout\u003d60 and timeout\u003d120 is right combination I didn\u0027t want to change it and hardcode into code as it is now, so I moved hardcoded value from connection string variable and moved to special variable proposed in this patch.\n\nNO Default changed ! \n\nThis is change which is just allowing user to specify his connection string based on extra variable.","commit_id":"074e79384c2488036b6616469b36bdf11c2afeef"},{"author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"change_message_id":"51f76b9cb6559c6079afa3a4df293960c2116ca5","unresolved":false,"context_lines":[{"line_number":783,"context_line":"# Redis options"},{"line_number":784,"context_line":"####################"},{"line_number":785,"context_line":"redis_connection_string: \"redis://{% for host in groups[\u0027redis\u0027] %}{% if host \u003d\u003d groups[\u0027redis\u0027][0] %}admin:{{ redis_master_password }}@{{ \u0027api\u0027 | kolla_address(host) | put_address_in_context(\u0027url\u0027) }}:{{ redis_sentinel_port }}?sentinel\u003dkolla{% else %}\u0026sentinel_fallback\u003d{{ \u0027api\u0027 | kolla_address(host) | put_address_in_context(\u0027url\u0027) }}:{{ redis_sentinel_port }}{% endif %}{% endfor %}{{ redis_cmdline_extras }}\""},{"line_number":786,"context_line":"redis_cmdline_extras: \"\u0026db\u003d0\u0026socket_timeout\u003d60\u0026retry_on_timeout\u003dyes\""},{"line_number":787,"context_line":""},{"line_number":788,"context_line":"####################"},{"line_number":789,"context_line":"# Osprofiler options"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"54a3887f_d5a13387","line":786,"range":{"start_line":786,"start_character":6,"end_line":786,"end_character":13},"in_reply_to":"d661c191_bf4afc78","updated":"2022-05-26 09:00:48.000000000","message":"Done","commit_id":"074e79384c2488036b6616469b36bdf11c2afeef"}]}