)]}'
{"id":"openstack%2Ftooz~672402","triplet_id":"openstack%2Ftooz~master~I0dd40f578ff0fc7bd1cd64c7889de2148ae6b06c","project":"openstack/tooz","branch":"master","topic":"bug/1837693","hashtags":[],"change_id":"I0dd40f578ff0fc7bd1cd64c7889de2148ae6b06c","subject":"Fix wrong log level during heartbeat","status":"MERGED","created":"2019-07-24 05:27:15.000000000","updated":"2019-07-27 01:59:16.000000000","submitted":"2019-07-26 14:47:54.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"672402-1564152475060-31ef7618","meta_rev_id":"0ac6d89084623d265bcf41220584a0fefa0cb6b8","_number":672402,"virtual_id_number":672402,"owner":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"date":"2019-07-25 01:52:04.000000000","_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},{"value":2,"date":"2019-07-26 14:47:54.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"all":[{"value":2,"date":"2019-07-24 14:09:48.000000000","_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":2,"date":"2019-07-26 12:59:39.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"all":[{"value":0,"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":1,"date":"2019-07-26 12:59:39.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Backport-Candidate":{"all":[{"value":0,"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do Not Backport","-1":"Not A Backport Candidate"," 0":"Backport Review Needed","+1":"Proposed Backport","+2":"Should Backport"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2019-07-24 14:09:48.000000000","updated_by":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"reviewer":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"state":"REVIEWER"},{"updated":"2019-07-26 12:59:39.000000000","updated_by":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"reviewer":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"state":"REVIEWER"},{"updated":"2019-07-26 14:47:54.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"56bbfc2f68396ac6ca2309cf31faf898a0d9dbca","author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"date":"2019-07-24 05:27:15.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"79d41e0ab4f81109fbc02d4ecd03870ee2ea27cb","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-24 07:08:05.000000000","message":"Patch Set 1: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/02/672402/1/check/tempest-full/7388f67/ : SUCCESS in 1h 38m 41s\n- tempest-full-py3 http://logs.openstack.org/02/672402/1/check/tempest-full-py3/0c06967/ : FAILURE in 1h 11m 30s\n- openstack-tox-cover http://logs.openstack.org/02/672402/1/check/openstack-tox-cover/7ba33ed/cover/ : SUCCESS in 4m 17s\n- openstack-tox-pep8 http://logs.openstack.org/02/672402/1/check/openstack-tox-pep8/83a0574/ : SUCCESS in 3m 29s\n- openstack-tox-py27 http://logs.openstack.org/02/672402/1/check/openstack-tox-py27/9b80499/ : SUCCESS in 4m 06s\n- openstack-tox-py36 http://logs.openstack.org/02/672402/1/check/openstack-tox-py36/cc66b67/ : SUCCESS in 3m 53s\n- openstack-tox-py37 http://logs.openstack.org/02/672402/1/check/openstack-tox-py37/9d62e50/ : SUCCESS in 4m 07s\n- openstack-tox-docs http://logs.openstack.org/02/672402/1/check/openstack-tox-docs/719b8d1/html/ : SUCCESS in 3m 39s\n- tooz-tox-py27-etcd http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-etcd/e411819/ : SUCCESS in 4m 06s\n- tooz-tox-py36-etcd http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-etcd/ad09cee/ : SUCCESS in 4m 26s\n- tooz-tox-py27-etcd3 http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-etcd3/170bc06/ : SUCCESS in 4m 21s\n- tooz-tox-py36-etcd3 http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-etcd3/57a18a2/ : SUCCESS in 4m 08s\n- tooz-tox-py27-etcd3gw http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-etcd3gw/08d9b94/ : SUCCESS in 5m 13s\n- tooz-tox-py36-etcd3gw http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-etcd3gw/d08a6ce/ : SUCCESS in 3m 58s\n- tooz-tox-py27-zookeeper http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-zookeeper/9d9e3cf/ : SUCCESS in 4m 27s\n- tooz-tox-py36-zookeeper http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-zookeeper/c8b3a93/ : SUCCESS in 4m 28s\n- tooz-tox-py27-redis http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-redis/874c098/ : SUCCESS in 5m 07s\n- tooz-tox-py36-redis http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-redis/c9260c1/ : SUCCESS in 5m 13s\n- tooz-tox-py27-sentinel http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-sentinel/5e1c446/ : SUCCESS in 5m 00s\n- tooz-tox-py36-sentinel http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-sentinel/225f42e/ : SUCCESS in 4m 59s\n- tooz-tox-py27-memcached http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-memcached/0d2942d/ : SUCCESS in 5m 08s\n- tooz-tox-py36-memcached http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-memcached/12c8ca4/ : SUCCESS in 4m 52s\n- tooz-tox-py27-postgresql http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-postgresql/cac5ae9/ : SUCCESS in 4m 43s\n- tooz-tox-py36-postgresql http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-postgresql/246a413/ : SUCCESS in 5m 01s\n- tooz-tox-py27-mysql http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-mysql/16038a4/ : SUCCESS in 4m 50s\n- tooz-tox-py36-mysql http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-mysql/f967717/ : SUCCESS in 4m 19s\n- tooz-tox-py27-consul http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-consul/6fcf806/ : SUCCESS in 4m 59s\n- tooz-tox-py36-consul http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-consul/36a020c/ : SUCCESS in 4m 43s","accounts_in_message":[],"_revision_number":1},{"id":"17e284fd99db5702c07e8d5269bc4e2ec05b03f9","author":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"date":"2019-07-24 14:09:48.000000000","message":"Patch Set 1: Code-Review+2\n\nrecheck\n\nThanks for the fix. Evidently we don\u0027t have unit test coverage of this. :-(","accounts_in_message":[],"_revision_number":1},{"id":"ad9b7f0800262664e0f6017492fa49b4b79db16e","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-24 15:57:48.000000000","message":"Patch Set 1: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full http://logs.openstack.org/02/672402/1/check/tempest-full/2641f23/ : SUCCESS in 1h 34m 05s\n- tempest-full-py3 http://logs.openstack.org/02/672402/1/check/tempest-full-py3/801c4e8/ : SUCCESS in 1h 42m 12s\n- openstack-tox-cover http://logs.openstack.org/02/672402/1/check/openstack-tox-cover/c05beac/cover/ : SUCCESS in 4m 41s\n- openstack-tox-pep8 http://logs.openstack.org/02/672402/1/check/openstack-tox-pep8/de598be/ : SUCCESS in 5m 37s\n- openstack-tox-py27 http://logs.openstack.org/02/672402/1/check/openstack-tox-py27/d11642f/ : SUCCESS in 4m 52s\n- openstack-tox-py36 http://logs.openstack.org/02/672402/1/check/openstack-tox-py36/6c0a32c/ : SUCCESS in 5m 39s\n- openstack-tox-py37 http://logs.openstack.org/02/672402/1/check/openstack-tox-py37/f31026c/ : SUCCESS in 5m 28s\n- openstack-tox-docs http://logs.openstack.org/02/672402/1/check/openstack-tox-docs/5f5ae8a/html/ : SUCCESS in 5m 04s\n- tooz-tox-py27-etcd http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-etcd/6a1a9e9/ : SUCCESS in 5m 10s\n- tooz-tox-py36-etcd http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-etcd/0e554e8/ : SUCCESS in 7m 22s\n- tooz-tox-py27-etcd3 http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-etcd3/fbdbafc/ : SUCCESS in 5m 00s\n- tooz-tox-py36-etcd3 http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-etcd3/914b6a3/ : SUCCESS in 5m 28s\n- tooz-tox-py27-etcd3gw http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-etcd3gw/5d3ea00/ : SUCCESS in 4m 54s\n- tooz-tox-py36-etcd3gw http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-etcd3gw/f5867ce/ : SUCCESS in 4m 02s\n- tooz-tox-py27-zookeeper http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-zookeeper/9e2ef40/ : SUCCESS in 4m 55s\n- tooz-tox-py36-zookeeper http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-zookeeper/ff62fbb/ : SUCCESS in 3m 42s\n- tooz-tox-py27-redis http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-redis/3671878/ : SUCCESS in 4m 04s\n- tooz-tox-py36-redis http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-redis/5805994/ : SUCCESS in 5m 28s\n- tooz-tox-py27-sentinel http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-sentinel/001d097/ : SUCCESS in 5m 55s\n- tooz-tox-py36-sentinel http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-sentinel/1bd566e/ : SUCCESS in 5m 39s\n- tooz-tox-py27-memcached http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-memcached/69b863b/ : SUCCESS in 5m 07s\n- tooz-tox-py36-memcached http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-memcached/337179a/ : SUCCESS in 5m 34s\n- tooz-tox-py27-postgresql http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-postgresql/ba81f38/ : SUCCESS in 4m 06s\n- tooz-tox-py36-postgresql http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-postgresql/4640146/ : SUCCESS in 4m 27s\n- tooz-tox-py27-mysql http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-mysql/34f5247/ : SUCCESS in 4m 59s\n- tooz-tox-py36-mysql http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-mysql/a769509/ : SUCCESS in 6m 13s\n- tooz-tox-py27-consul http://logs.openstack.org/02/672402/1/check/tooz-tox-py27-consul/6b00ed6/ : SUCCESS in 4m 50s\n- tooz-tox-py36-consul http://logs.openstack.org/02/672402/1/check/tooz-tox-py36-consul/8be9253/ : SUCCESS in 5m 56s","accounts_in_message":[],"_revision_number":1},{"id":"9bfdd3ff69dd23de938799209307f01a405c911c","author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"date":"2019-07-25 01:52:04.000000000","message":"Patch Set 1:\n\nHi Ben, I may need some help on using tooz, when I am playing tooz with small script for verification, it works fine, but when I embed it in the ironic inspector, there are various issues that seems to be related with the heartbeat and lease update.\n\nFor example, start coordinator with start_heart\u003dTrue never succeed, and observed obvious delay during startup:\n\n2019-07-25 09:25:04.774 9613 WARNING tooz.coordination [-] Retrying tooz.drivers.etcd3.Etcd3Driver.heartbeat in 1.0 seconds as it raised .\n\nIf I don use the start_heart\u003dTrue, while using a greenlet to do the heartbeat, it\u0027s also not working, mostly saying \"requested lease not found\", or connection timed out.\n\nIt seems tooz or underlying libraries it used may not work with the service base on oslo.service, or maybe there is something I don\u0027t know. Is there any caveat that needs attention? Any pointers would help, thanks!","accounts_in_message":[],"_revision_number":1},{"id":"d54e7462626e193ec37619c2870872511a39bb61","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-26 12:59:39.000000000","message":"Patch Set 1: Code-Review+2 Workflow+1\n\n\u003e Hi Ben, I may need some help on using tooz, when I am playing tooz\n \u003e with small script for verification, it works fine, but when I embed\n \u003e it in the ironic inspector, there are various issues that seems to\n \u003e be related with the heartbeat and lease update.\n \u003e \n \u003e For example, start coordinator with start_heart\u003dTrue never succeed,\n \u003e and observed obvious delay during startup:\n \u003e \n \u003e 2019-07-25 09:25:04.774 9613 WARNING tooz.coordination [-] Retrying\n \u003e tooz.drivers.etcd3.Etcd3Driver.heartbeat in 1.0 seconds as it\n \u003e raised .\n \u003e \n \u003e If I don use the start_heart\u003dTrue, while using a greenlet to do the\n \u003e heartbeat, it\u0027s also not working, mostly saying \"requested lease\n \u003e not found\", or connection timed out.\n \u003e \n \u003e It seems tooz or underlying libraries it used may not work with the\n \u003e service base on oslo.service, or maybe there is something I don\u0027t\n \u003e know. Is there any caveat that needs attention? Any pointers would\n \u003e help, thanks!\n\nHey\n\nthanks for your fix!\n\nInteresting thanks for the heads-up, can you share more info about the environment (apache, mod_wsgi, openstack version, etc...)?\n\nWe also facing some issues with greenlet and nova-api under apache so maybe it can be a similar issue between greenlet and apache...:\n- https://review.opendev.org/663074\n- https://review.opendev.org/668862\n- https://review.opendev.org/671321\n\nFor further info you can read my blog story about this:\n- https://herve.beraud.io/openstack/nova/oslo.messaging/rabbitmq/driver/2019/07/15/how-nova-consume-oslo-messaging.html\n- https://herve.beraud.io/openstack/tripleo/gerrit/patch/heat-template/2019/07/23/how-to-patch-openstack-tripleo.html\n- https://herve.beraud.io/openstack/tripleo/gerrit/patch/heat-template/2019/07/24/how-to-check-openstack-apache-mpm-engine-switching-fix-amqp-heartbeat-eventlet.html\n\nI\u0027m not sure if it\u0027s related to something similar than oslo.messaging but some piece of your puzzle looks like similar...","accounts_in_message":[],"_revision_number":1},{"id":"5da238deecfb0fd296b6182bb1f6749568ead02f","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-26 12:59:50.000000000","message":"Patch Set 1: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":1},{"id":"de01b48e4c8b90044ccebeba06f9fa17919f417e","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-26 14:47:54.000000000","message":"Patch Set 1: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- tempest-full http://logs.openstack.org/02/672402/1/gate/tempest-full/7a22d48/ : SUCCESS in 1h 43m 33s\n- tempest-full-py3 http://logs.openstack.org/02/672402/1/gate/tempest-full-py3/89af751/ : SUCCESS in 1h 24m 54s\n- openstack-tox-pep8 http://logs.openstack.org/02/672402/1/gate/openstack-tox-pep8/a2c0297/ : SUCCESS in 5m 38s\n- openstack-tox-py27 http://logs.openstack.org/02/672402/1/gate/openstack-tox-py27/5986b06/ : SUCCESS in 7m 00s\n- openstack-tox-py36 http://logs.openstack.org/02/672402/1/gate/openstack-tox-py36/a86166e/ : SUCCESS in 5m 01s\n- openstack-tox-py37 http://logs.openstack.org/02/672402/1/gate/openstack-tox-py37/f8759e7/ : SUCCESS in 7m 27s\n- openstack-tox-docs http://logs.openstack.org/02/672402/1/gate/openstack-tox-docs/93d29c6/html/ : SUCCESS in 7m 29s\n- tooz-tox-py27-etcd http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-etcd/f11671f/ : SUCCESS in 5m 26s\n- tooz-tox-py36-etcd http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-etcd/babd9e3/ : SUCCESS in 6m 02s\n- tooz-tox-py27-etcd3 http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-etcd3/c041df5/ : SUCCESS in 7m 25s\n- tooz-tox-py36-etcd3 http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-etcd3/674f290/ : SUCCESS in 7m 21s\n- tooz-tox-py27-etcd3gw http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-etcd3gw/1d70e99/ : SUCCESS in 5m 40s\n- tooz-tox-py36-etcd3gw http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-etcd3gw/d996575/ : SUCCESS in 6m 03s\n- tooz-tox-py27-zookeeper http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-zookeeper/75c818d/ : SUCCESS in 5m 30s\n- tooz-tox-py36-zookeeper http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-zookeeper/be02ee6/ : SUCCESS in 6m 53s\n- tooz-tox-py27-redis http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-redis/897b6cb/ : SUCCESS in 4m 55s\n- tooz-tox-py36-redis http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-redis/25f448a/ : SUCCESS in 4m 16s\n- tooz-tox-py27-sentinel http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-sentinel/eb3cc9a/ : SUCCESS in 5m 07s\n- tooz-tox-py36-sentinel http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-sentinel/3395dec/ : SUCCESS in 6m 56s\n- tooz-tox-py27-memcached http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-memcached/d26add8/ : SUCCESS in 5m 55s\n- tooz-tox-py36-memcached http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-memcached/96b7c3e/ : SUCCESS in 4m 53s\n- tooz-tox-py27-postgresql http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-postgresql/860380b/ : SUCCESS in 6m 37s\n- tooz-tox-py36-postgresql http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-postgresql/81c9a83/ : SUCCESS in 5m 26s\n- tooz-tox-py27-mysql http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-mysql/d89db02/ : SUCCESS in 7m 18s\n- tooz-tox-py36-mysql http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-mysql/8c61f62/ : SUCCESS in 5m 40s\n- tooz-tox-py27-consul http://logs.openstack.org/02/672402/1/gate/tooz-tox-py27-consul/0e586b0/ : SUCCESS in 5m 27s\n- tooz-tox-py36-consul http://logs.openstack.org/02/672402/1/gate/tooz-tox-py36-consul/08bed20/ : SUCCESS in 5m 23s","accounts_in_message":[],"_revision_number":1},{"id":"c7ee4879d828523ab617098cc30d9025f6c8c4f6","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-26 14:47:55.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":1},{"id":"88db15e6e37a76745f9aa77015afca5b16bf89b7","author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"date":"2019-07-27 01:59:16.000000000","message":"Patch Set 1:\n\nThank you Hervé for the kindly reply :)\n\n \u003e Interesting thanks for the heads-up, can you share more info about\n \u003e the environment (apache, mod_wsgi, openstack version, etc...)?\n\nI am debugging ironic-inspector with source of master, apache is not used.\n\n \u003e We also facing some issues with greenlet and nova-api under apache\n \u003e so maybe it can be a similar issue between greenlet and apache...:\n \u003e - https://review.opendev.org/663074\n \u003e - https://review.opendev.org/668862\n \u003e - https://review.opendev.org/671321\n \u003e \n \u003e For further info you can read my blog story about this:\n \u003e - https://herve.beraud.io/openstack/nova/oslo.messaging/rabbitmq/driver/2019/07/15/how-nova-consume-oslo-messaging.html\n \u003e - https://herve.beraud.io/openstack/tripleo/gerrit/patch/heat-template/2019/07/23/how-to-patch-openstack-tripleo.html\n \u003e - https://herve.beraud.io/openstack/tripleo/gerrit/patch/heat-template/2019/07/24/how-to-check-openstack-apache-mpm-engine-switching-fix-amqp-heartbeat-eventlet.html\n \u003e \n \u003e I\u0027m not sure if it\u0027s related to something similar than\n \u003e oslo.messaging but some piece of your puzzle looks like similar...\n\nThanks for the reference, at first time I thought it may be related with concurrency, but after I successfully adopted memcached up and running with tooz, I guess the issue  maybe lie in underlying libraries. etcd3 has an unresolved issue [1] about the lease, but I am not sure whether it\u0027s the same issue I encountered, will keep digging.\n\n[1] https://github.com/kragniz/python-etcd3/issues/560","accounts_in_message":[],"_revision_number":1}],"current_revision_number":1,"current_revision":"a4f1288ee4234865a4db8b9c191ba275e2deef3d","revisions":{"a4f1288ee4234865a4db8b9c191ba275e2deef3d":{"kind":"REWORK","_number":1,"created":"2019-07-24 05:27:15.000000000","uploader":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"ref":"refs/changes/02/672402/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/tooz","ref":"refs/changes/02/672402/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/tooz refs/changes/02/672402/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/tooz refs/changes/02/672402/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/tooz refs/changes/02/672402/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/tooz refs/changes/02/672402/1"}}},"commit":{"parents":[{"commit":"491dbee61961db86e8e3b8e76050b24c7e1cb2dd","subject":"Merge \"Move grpcio from requirements.txt to extras\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/tooz/commit/491dbee61961db86e8e3b8e76050b24c7e1cb2dd"}]}],"author":{"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","date":"2019-07-24 05:14:14.000000000","tz":480},"committer":{"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","date":"2019-07-24 05:20:36.000000000","tz":480},"subject":"Fix wrong log level during heartbeat","message":"Fix wrong log level during heartbeat\n\nIt appears that tooz used log.warning which is a method\ninstead of an integer required by tenacity.\n\nChange-Id: I0dd40f578ff0fc7bd1cd64c7889de2148ae6b06c\nCloses-Bug: #1837693\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/tooz/commit/a4f1288ee4234865a4db8b9c191ba275e2deef3d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/tooz/commit/a4f1288ee4234865a4db8b9c191ba275e2deef3d"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}