)]}'
{"id":"openstack%2Foslo.messaging~663074","triplet_id":"openstack%2Foslo.messaging~master~If8846599efc48fe18ecfb99c04e2c38f9a45b9ed","project":"openstack/oslo.messaging","branch":"master","topic":"bug/1826281","hashtags":[],"change_id":"If8846599efc48fe18ecfb99c04e2c38f9a45b9ed","subject":"Allow users run the rabbitmq heartbeat inside a standard pthread.","status":"MERGED","created":"2019-06-04 15:54:44.000000000","updated":"2019-08-08 15:50:07.000000000","submitted":"2019-08-08 15:50:07.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":19,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"663074-1565279407887-cf54861c","meta_rev_id":"f8c3da130245b1107c4576d8421088ed87b5c9ae","_number":663074,"virtual_id_number":663074,"owner":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":0,"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":0,"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},{"value":0,"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":0,"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},{"value":0,"date":"2019-08-07 15:27:07.000000000","_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":2,"date":"2019-08-08 15:50:07.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":9373,"name":"Vlad Gusev","email":"vlad.esten@gmail.com","username":"s10"},{"value":0,"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"all":[{"value":2,"date":"2019-08-07 20:48:09.000000000","_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":2,"date":"2019-08-08 13:59:01.000000000","_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":1,"date":"2019-08-08 14:07:39.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},{"value":0,"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":1,"date":"2019-08-07 15:37:11.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":9373,"name":"Vlad Gusev","email":"vlad.esten@gmail.com","username":"s10"},{"value":0,"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"all":[{"value":0,"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":1,"date":"2019-08-08 13:59:01.000000000","_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":0,"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},{"value":0,"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":0,"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":9373,"name":"Vlad Gusev","email":"vlad.esten@gmail.com","username":"s10"},{"value":0,"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Backport-Candidate":{"all":[{"value":0,"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"value":0,"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"value":0,"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},{"value":0,"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":0,"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":9373,"name":"Vlad Gusev","email":"vlad.esten@gmail.com","username":"s10"},{"value":0,"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"}],"values":{"-2":"Do Not Backport","-1":"Not A Backport Candidate"," 0":"Backport Review Needed","+1":"Proposed Backport","+2":"Should Backport"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},{"_account_id":9373,"name":"Vlad Gusev","email":"vlad.esten@gmail.com","username":"s10"},{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},{"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2019-07-03 14:58:14.000000000","updated_by":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"reviewer":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"state":"REVIEWER"},{"updated":"2019-07-05 16:29:57.000000000","updated_by":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"reviewer":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"state":"REVIEWER"},{"updated":"2019-07-17 23:19:33.000000000","updated_by":{"_account_id":9373,"name":"Vlad Gusev","email":"vlad.esten@gmail.com","username":"s10"},"reviewer":{"_account_id":9373,"name":"Vlad Gusev","email":"vlad.esten@gmail.com","username":"s10"},"state":"REVIEWER"},{"updated":"2019-08-07 15:37:11.000000000","updated_by":{"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},"reviewer":{"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},"state":"REVIEWER"},{"updated":"2019-08-07 20:48:09.000000000","updated_by":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"reviewer":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"state":"REVIEWER"},{"updated":"2019-08-08 13:59:01.000000000","updated_by":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"reviewer":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"state":"REVIEWER"},{"updated":"2019-08-08 14:07:39.000000000","updated_by":{"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},"reviewer":{"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},"state":"REVIEWER"},{"updated":"2019-08-08 15:50:07.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"a612e65278e1a2a1d36796b39f2436ce8037da2b","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-04 15:54:44.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"aa9230d7b02b4e7b219f2e348510d77c80653559","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-04 15:56:41.000000000","message":"Uploaded patch set 2: Commit message was updated.","accounts_in_message":[],"_revision_number":2},{"id":"f0c1bd467d769633912b91fc5e49f5ba8f479619","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-04 17:00:39.000000000","message":"Uploaded patch set 3: Commit message was updated.","accounts_in_message":[],"_revision_number":3},{"id":"4c5a87c0cf854cfbaf1577c3d9e62818bee65ef8","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-04 19:22:01.000000000","message":"Patch Set 3: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/3/check/tempest-full/8e2c486/ : TIMED_OUT in 2h 07m 44s\n- tempest-full-py3 http://logs.openstack.org/74/663074/3/check/tempest-full-py3/8f1dac2/ : SUCCESS in 1h 25m 06s\n- openstack-tox-cover http://logs.openstack.org/74/663074/3/check/openstack-tox-cover/e227a11/ : TIMED_OUT in 51m 13s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/3/check/openstack-tox-lower-constraints/7a81dcc/ : TIMED_OUT in 41m 07s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/3/check/openstack-tox-pep8/ec8a18b/ : SUCCESS in 6m 47s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/3/check/openstack-tox-py27/9da0710/ : TIMED_OUT in 41m 00s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/3/check/openstack-tox-py36/f7ff69c/ : TIMED_OUT in 40m 59s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/3/check/openstack-tox-py37/0d4f58d/ : TIMED_OUT in 40m 53s\n- openstack-tox-docs http://logs.openstack.org/74/663074/3/check/openstack-tox-docs/9daaa36/html/ : SUCCESS in 2m 52s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py27-func-rabbit/78fd751/ : SUCCESS in 5m 11s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py27-func-amqp1/2ff7008/ : FAILURE in 11m 47s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py27-func-kafka/23166e7/ : FAILURE in 4m 15s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py36-func-rabbit/393e76e/ : SUCCESS in 5m 52s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py36-func-amqp1/8af347b/ : FAILURE in 11m 23s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py36-func-kafka/354ab93/ : FAILURE in 6m 16s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-rabbit/afb524a/ : SUCCESS in 1h 57m 47s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/523fbd2/ : SUCCESS in 1h 34m 28s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/f0e884f/ : SUCCESS in 2h 09m 55s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-kafka-hybrid/aab2cf5/ : SUCCESS in 1h 57m 29s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-kafka-centos-7/2b24137/ : SUCCESS in 2h 07m 17s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-grenade-dsvm/7379bf6/ : SUCCESS in 1h 00m 57s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-grenade-dsvm-multinode/1a9a01f/ : FAILURE in 1h 14m 28s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-telemetry-dsvm-integration-rabbit/7ad8bf5/ : FAILURE in 50m 03s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/3/check/oslo.messaging-telemetry-dsvm-integration-amqp1/97518b6/ : FAILURE in 50m 34s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/3/check/oslo.messaging-telemetry-dsvm-integration-kafka/e1af9b7/ : FAILURE in 50m 45s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"d007ff4f67c834039d74f9bb0bc527a62c96c9f6","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-05 07:50:44.000000000","message":"Patch Set 3:\n\nrecheck","accounts_in_message":[],"_revision_number":3},{"id":"2398f18179cea52a1771d5ab8b6184b3fdb03c4a","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-05 10:02:25.000000000","message":"Patch Set 3:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/3/check/tempest-full/446c368/ : SUCCESS in 1h 34m 49s\n- tempest-full-py3 http://logs.openstack.org/74/663074/3/check/tempest-full-py3/03df248/ : SUCCESS in 1h 44m 03s\n- openstack-tox-cover http://logs.openstack.org/74/663074/3/check/openstack-tox-cover/7b47105/ : TIMED_OUT in 50m 55s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/3/check/openstack-tox-lower-constraints/a5cea2c/ : TIMED_OUT in 41m 04s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/3/check/openstack-tox-pep8/014a1f6/ : SUCCESS in 4m 21s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/3/check/openstack-tox-py27/43ce07c/ : TIMED_OUT in 40m 59s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/3/check/openstack-tox-py36/e39ca54/ : TIMED_OUT in 40m 53s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/3/check/openstack-tox-py37/2810c06/ : TIMED_OUT in 40m 49s\n- openstack-tox-docs http://logs.openstack.org/74/663074/3/check/openstack-tox-docs/85cf0f7/html/ : SUCCESS in 2m 54s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py27-func-rabbit/2b7040a/ : TIMED_OUT in 40m 48s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py27-func-amqp1/5b3c2fa/ : FAILURE in 11m 47s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py27-func-kafka/311749a/ : FAILURE in 4m 22s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py36-func-rabbit/44f7998/ : SUCCESS in 4m 55s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py36-func-amqp1/6286985/ : FAILURE in 11m 45s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/3/check/oslo.messaging-tox-py36-func-kafka/2f3051b/ : FAILURE in 4m 04s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-rabbit/c7edaf7/ : SUCCESS in 1h 42m 22s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/1964f74/ : SUCCESS in 1h 47m 44s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/929148c/ : SUCCESS in 2h 06m 12s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-kafka-hybrid/39b9864/ : SUCCESS in 1h 49m 38s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-dsvm-full-kafka-centos-7/7ef3d92/ : SUCCESS in 2h 11m 08s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-grenade-dsvm/821555e/ : SUCCESS in 58m 50s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/3/check/oslo.messaging-src-grenade-dsvm-multinode/6567705/ : SUCCESS in 1h 03m 34s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/3/check/oslo.messaging-telemetry-dsvm-integration-rabbit/abbedda/ : FAILURE in 48m 17s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/3/check/oslo.messaging-telemetry-dsvm-integration-amqp1/2e1f425/ : FAILURE in 43m 06s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/3/check/oslo.messaging-telemetry-dsvm-integration-kafka/89b92a2/ : FAILURE in 46m 49s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"3ef07713f144a1e23156e95359e5ed206f9b5e53","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-05 10:32:57.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"9e466385020297d05e7027c394825876f13267c8","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-05 10:35:28.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"ea01ea3d33e9c4c977c49d3daaab66b1f28c0c80","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-05 13:46:39.000000000","message":"Patch Set 5: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/5/check/tempest-full/eec40a0/ : TIMED_OUT in 2h 02m 36s\n- tempest-full-py3 http://logs.openstack.org/74/663074/5/check/tempest-full-py3/18b6aab/ : FAILURE in 42m 11s\n- openstack-tox-cover http://logs.openstack.org/74/663074/5/check/openstack-tox-cover/f891d9f/ : TIMED_OUT in 50m 59s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/5/check/openstack-tox-lower-constraints/a303922/ : TIMED_OUT in 40m 56s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/5/check/openstack-tox-pep8/064bcee/ : FAILURE in 4m 13s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/5/check/openstack-tox-py27/5a71fa3/ : TIMED_OUT in 40m 55s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/5/check/openstack-tox-py36/5da8e86/ : TIMED_OUT in 40m 55s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/5/check/openstack-tox-py37/ea3f704/ : TIMED_OUT in 40m 53s\n- openstack-tox-docs http://logs.openstack.org/74/663074/5/check/openstack-tox-docs/31dedc9/html/ : SUCCESS in 2m 47s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/5/check/oslo.messaging-tox-py27-func-rabbit/b19a659/ : TIMED_OUT in 40m 52s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/5/check/oslo.messaging-tox-py27-func-amqp1/0a3d705/ : FAILURE in 11m 39s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/5/check/oslo.messaging-tox-py27-func-kafka/880d345/ : FAILURE in 5m 06s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/5/check/oslo.messaging-tox-py36-func-rabbit/11bd1ea/ : TIMED_OUT in 41m 01s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/5/check/oslo.messaging-tox-py36-func-amqp1/55e35e4/ : FAILURE in 11m 29s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/5/check/oslo.messaging-tox-py36-func-kafka/499e45b/ : FAILURE in 4m 00s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/5/check/oslo.messaging-src-dsvm-full-rabbit/34d7310/ : FAILURE in 3h 04m 37s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/5/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/0be34a6/ : SUCCESS in 1h 55m 09s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/5/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/19ebf0e/ : FAILURE in 1h 39m 43s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/5/check/oslo.messaging-src-dsvm-full-kafka-hybrid/ac1092e/ : TIMED_OUT in 3h 08m 21s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/5/check/oslo.messaging-src-dsvm-full-kafka-centos-7/84b31db/ : FAILURE in 36m 46s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/5/check/oslo.messaging-src-grenade-dsvm/965511d/ : FAILURE in 54m 14s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/5/check/oslo.messaging-src-grenade-dsvm-multinode/7e96c95/ : FAILURE in 1h 04m 56s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/5/check/oslo.messaging-telemetry-dsvm-integration-rabbit/8423779/ : FAILURE in 2h 50m 52s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/5/check/oslo.messaging-telemetry-dsvm-integration-amqp1/1d0dbaf/ : FAILURE in 45m 02s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/5/check/oslo.messaging-telemetry-dsvm-integration-kafka/eedde96/ : FAILURE in 2h 51m 26s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"90febaac50b0fc2b8a70ac323d05da6dd2bdf5ed","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-05 14:10:39.000000000","message":"Uploaded patch set 6.","accounts_in_message":[],"_revision_number":6},{"id":"c152ba6f02fcfe692417fdf7c58c72a4967925bb","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-05 14:52:21.000000000","message":"Uploaded patch set 7: Commit message was updated.","accounts_in_message":[],"_revision_number":7},{"id":"e6258f1360371316408ca85721fed8078e68471c","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-05 15:57:30.000000000","message":"Uploaded patch set 8.","accounts_in_message":[],"_revision_number":8},{"id":"d49a20e1b12a7fd9b39e664d0c603230051a0531","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-05 16:51:43.000000000","message":"Uploaded patch set 9.","accounts_in_message":[],"_revision_number":9},{"id":"c90747c27e29726ed880f48d9d70a7c7ad4cc9ce","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-05 19:04:13.000000000","message":"Patch Set 9: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/9/check/tempest-full/d947615/ : SUCCESS in 1h 43m 36s\n- tempest-full-py3 http://logs.openstack.org/74/663074/9/check/tempest-full-py3/614a714/ : SUCCESS in 1h 48m 30s\n- openstack-tox-cover http://logs.openstack.org/74/663074/9/check/openstack-tox-cover/44ca8cd/ : TIMED_OUT in 51m 07s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/9/check/openstack-tox-lower-constraints/fbe8a15/ : TIMED_OUT in 40m 56s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/9/check/openstack-tox-pep8/982ac84/ : FAILURE in 4m 03s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/9/check/openstack-tox-py27/2d339a7/ : TIMED_OUT in 40m 56s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/9/check/openstack-tox-py36/eb18dde/ : TIMED_OUT in 40m 56s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/9/check/openstack-tox-py37/3cbbe23/ : TIMED_OUT in 40m 56s\n- openstack-tox-docs http://logs.openstack.org/74/663074/9/check/openstack-tox-docs/8b93a80/html/ : SUCCESS in 2m 59s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/9/check/oslo.messaging-tox-py27-func-rabbit/5091e3e/ : SUCCESS in 5m 13s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/9/check/oslo.messaging-tox-py27-func-amqp1/25b528f/ : FAILURE in 11m 37s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/9/check/oslo.messaging-tox-py27-func-kafka/600bd8b/ : FAILURE in 3m 52s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/9/check/oslo.messaging-tox-py36-func-rabbit/5058d1e/ : SUCCESS in 7m 05s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/9/check/oslo.messaging-tox-py36-func-amqp1/9206412/ : FAILURE in 11m 46s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/9/check/oslo.messaging-tox-py36-func-kafka/ad5cb20/ : FAILURE in 4m 10s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/9/check/oslo.messaging-src-dsvm-full-rabbit/f344bb5/ : SUCCESS in 1h 36m 47s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/9/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/ad0e6e6/ : SUCCESS in 1h 36m 46s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/9/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/133c631/ : POST_FAILURE in 2h 06m 26s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/9/check/oslo.messaging-src-dsvm-full-kafka-hybrid/5b59aed/ : SUCCESS in 1h 51m 44s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/9/check/oslo.messaging-src-dsvm-full-kafka-centos-7/6032dfe/ : FAILURE in 1h 54m 19s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/9/check/oslo.messaging-src-grenade-dsvm/41d939d/ : SUCCESS in 1h 01m 13s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/9/check/oslo.messaging-src-grenade-dsvm-multinode/c3c4502/ : SUCCESS in 1h 14m 16s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/9/check/oslo.messaging-telemetry-dsvm-integration-rabbit/684bcf8/ : FAILURE in 45m 44s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/9/check/oslo.messaging-telemetry-dsvm-integration-amqp1/4f82e50/ : FAILURE in 44m 38s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/9/check/oslo.messaging-telemetry-dsvm-integration-kafka/cd01e9a/ : FAILURE in 44m 40s (non-voting)","accounts_in_message":[],"_revision_number":9},{"id":"a2de5fa07456692629ec893463f8bc04260cfc79","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-18 16:19:49.000000000","message":"Uploaded patch set 10.","accounts_in_message":[],"_revision_number":10},{"id":"f4696a4f4b96dd141b426b32e8853ad866b90fea","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-18 18:40:14.000000000","message":"Patch Set 10: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/10/check/tempest-full/bce4675/ : SUCCESS in 1h 46m 23s\n- tempest-full-py3 http://logs.openstack.org/74/663074/10/check/tempest-full-py3/9cd7932/ : SUCCESS in 1h 32m 00s\n- openstack-tox-cover http://logs.openstack.org/74/663074/10/check/openstack-tox-cover/bc9c951/ : TIMED_OUT in 50m 58s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/10/check/openstack-tox-lower-constraints/6d5806a/ : TIMED_OUT in 40m 51s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/10/check/openstack-tox-pep8/fcbbfad/ : SUCCESS in 4m 41s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/10/check/openstack-tox-py27/acabe7a/ : TIMED_OUT in 40m 57s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/10/check/openstack-tox-py36/b1d25b7/ : TIMED_OUT in 41m 03s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/10/check/openstack-tox-py37/c61cb8f/ : TIMED_OUT in 41m 01s\n- openstack-tox-docs http://logs.openstack.org/74/663074/10/check/openstack-tox-docs/f8ab200/html/ : SUCCESS in 3m 45s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py27-func-rabbit/431f300/ : SUCCESS in 5m 21s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py27-func-amqp1/612cd23/ : FAILURE in 11m 52s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py27-func-kafka/efeebfe/ : SUCCESS in 5m 15s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py36-func-rabbit/3e44c37/ : TIMED_OUT in 40m 57s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py36-func-amqp1/25b15dd/ : FAILURE in 10m 55s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py36-func-kafka/8a5fa39/ : SUCCESS in 5m 29s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-rabbit/42b6101/ : SUCCESS in 2h 01m 20s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/40946ed/ : FAILURE in 35m 24s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/29954a5/ : FAILURE in 41m 57s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-kafka-hybrid/e2cbb53/ : SUCCESS in 1h 43m 26s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-kafka-centos-7/8d5e534/ : SUCCESS in 2h 13m 50s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-grenade-dsvm/0a2482d/ : SUCCESS in 1h 02m 54s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-grenade-dsvm-multinode/3304298/ : SUCCESS in 1h 16m 37s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-telemetry-dsvm-integration-rabbit/22ad69c/ : FAILURE in 39m 52s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/10/check/oslo.messaging-telemetry-dsvm-integration-amqp1/b6a184f/ : FAILURE in 45m 07s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/10/check/oslo.messaging-telemetry-dsvm-integration-kafka/5a7b789/ : FAILURE in 45m 21s (non-voting)","accounts_in_message":[],"_revision_number":10},{"id":"5333dc2387388e592da49e0133f8dbd47a5f3342","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-19 08:56:25.000000000","message":"Patch Set 10:\n\nrecheck","accounts_in_message":[],"_revision_number":10},{"id":"c0b9934f07cd6d3a1b549d0abb77e582fdc8496e","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-19 12:41:36.000000000","message":"Patch Set 10:\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/10/check/tempest-full/36a806b/ : SUCCESS in 2h 00m 25s\n- tempest-full-py3 http://logs.openstack.org/74/663074/10/check/tempest-full-py3/3a124d4/ : SUCCESS in 1h 38m 19s\n- openstack-tox-cover http://logs.openstack.org/74/663074/10/check/openstack-tox-cover/54f320a/ : TIMED_OUT in 51m 10s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/10/check/openstack-tox-lower-constraints/cabaef4/ : TIMED_OUT in 41m 00s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/10/check/openstack-tox-pep8/db041d8/ : SUCCESS in 5m 17s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/10/check/openstack-tox-py27/6ca9f22/ : TIMED_OUT in 40m 41s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/10/check/openstack-tox-py36/c3b1e85/ : TIMED_OUT in 40m 51s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/10/check/openstack-tox-py37/88d34c0/ : TIMED_OUT in 41m 05s\n- openstack-tox-docs http://logs.openstack.org/74/663074/10/check/openstack-tox-docs/1ed4071/html/ : SUCCESS in 3m 08s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py27-func-rabbit/86dd290/ : TIMED_OUT in 41m 09s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py27-func-amqp1/0d041ea/ : FAILURE in 11m 35s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py27-func-kafka/44803a8/ : FAILURE in 5m 35s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py36-func-rabbit/8b08478/ : SUCCESS in 5m 38s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py36-func-amqp1/3e2840f/ : RETRY_LIMIT in 40m 55s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/10/check/oslo.messaging-tox-py36-func-kafka/babffb6/ : FAILURE in 7m 10s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-rabbit/d34f62c/ : SUCCESS in 1h 47m 33s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/37ad820/ : FAILURE in 28m 51s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/ddd2996/ : POST_FAILURE in 39m 45s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-kafka-hybrid/45fdf3e/ : SUCCESS in 1h 53m 06s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-dsvm-full-kafka-centos-7/dd2ac52/ : RETRY_LIMIT in 52m 49s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-grenade-dsvm/e40618d/ : SUCCESS in 1h 04m 01s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/10/check/oslo.messaging-src-grenade-dsvm-multinode/9c41bda/ : SUCCESS in 1h 10m 58s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/10/check/oslo.messaging-telemetry-dsvm-integration-rabbit/edb0122/ : FAILURE in 50m 26s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/10/check/oslo.messaging-telemetry-dsvm-integration-amqp1/9737acc/ : FAILURE in 51m 34s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/10/check/oslo.messaging-telemetry-dsvm-integration-kafka/aff6b29/ : FAILURE in 48m 41s (non-voting)","accounts_in_message":[],"_revision_number":10},{"id":"53cb27b2d4e46b67832555109f6387bb9942cfea","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-19 14:07:27.000000000","message":"Uploaded patch set 11.","accounts_in_message":[],"_revision_number":11},{"id":"8bcf26cf4f3970507d2bf634256b5861d5f8b03f","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-19 14:08:32.000000000","message":"Patch Set 11:\n\nAs I can observe on the unit test side we always monkey patch the stdlib at start https://github.com/openstack/oslo.messaging/commit/d64d05f3e07f08518faf83c6684da47998535678\n\nSo in tests like this one \u003d\u003e https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/tests/drivers/test_impl_rabbit.py#L52 where we try to test the heartbeat, we are in a sort a bastard execution model, where in the real driver we avoid to use monkey patched threading module, and where in the unit test side we do not take care about the execution model and where threading events are inherited for the previous monkey patched... \n\nWell... I need to continue to dig to found and avoid the unit test timeout issue... I think it\u0027s the most funny part of this patch...","accounts_in_message":[],"_revision_number":11},{"id":"9e84b3d48d05af5833f9a3fa4e6be05ddf3033fc","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-19 16:20:13.000000000","message":"Patch Set 11: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/11/check/tempest-full/de70526/ : SUCCESS in 1h 46m 31s\n- tempest-full-py3 http://logs.openstack.org/74/663074/11/check/tempest-full-py3/62fac38/ : SUCCESS in 1h 49m 38s\n- openstack-tox-cover http://logs.openstack.org/74/663074/11/check/openstack-tox-cover/b6cfde7/ : TIMED_OUT in 51m 16s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/11/check/openstack-tox-lower-constraints/97a11cd/ : TIMED_OUT in 41m 05s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/11/check/openstack-tox-pep8/5a0386d/ : FAILURE in 4m 18s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/11/check/openstack-tox-py27/6b3d8ea/ : TIMED_OUT in 40m 55s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/11/check/openstack-tox-py36/e53883d/ : TIMED_OUT in 41m 07s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/11/check/openstack-tox-py37/2646c38/ : TIMED_OUT in 41m 11s\n- openstack-tox-docs http://logs.openstack.org/74/663074/11/check/openstack-tox-docs/095491d/html/ : SUCCESS in 4m 53s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/11/check/oslo.messaging-tox-py27-func-rabbit/9ed4920/ : SUCCESS in 5m 34s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/11/check/oslo.messaging-tox-py27-func-amqp1/6000fbf/ : FAILURE in 12m 21s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/11/check/oslo.messaging-tox-py27-func-kafka/9f426e2/ : SUCCESS in 6m 12s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/11/check/oslo.messaging-tox-py36-func-rabbit/a9ea519/ : SUCCESS in 5m 42s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/11/check/oslo.messaging-tox-py36-func-amqp1/a6f6da3/ : FAILURE in 12m 36s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/11/check/oslo.messaging-tox-py36-func-kafka/09c4c2f/ : FAILURE in 5m 32s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/11/check/oslo.messaging-src-dsvm-full-rabbit/8c7cc51/ : SUCCESS in 1h 45m 03s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/11/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/f54c1bb/ : FAILURE in 30m 40s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/11/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/1f40284/ : FAILURE in 42m 26s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/11/check/oslo.messaging-src-dsvm-full-kafka-hybrid/be5f8ad/ : SUCCESS in 1h 52m 32s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/11/check/oslo.messaging-src-dsvm-full-kafka-centos-7/0a5304e/ : SUCCESS in 2h 05m 34s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/11/check/oslo.messaging-src-grenade-dsvm/159ccf2/ : SUCCESS in 1h 02m 47s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/11/check/oslo.messaging-src-grenade-dsvm-multinode/4ca0ff8/ : SUCCESS in 1h 18m 26s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/11/check/oslo.messaging-telemetry-dsvm-integration-rabbit/def5edd/ : FAILURE in 52m 36s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/11/check/oslo.messaging-telemetry-dsvm-integration-amqp1/3bbf784/ : FAILURE in 46m 33s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/11/check/oslo.messaging-telemetry-dsvm-integration-kafka/c6f49d5/ : FAILURE in 49m 54s (non-voting)","accounts_in_message":[],"_revision_number":11},{"id":"aeba2e7dc1bb9cbc9330feb3d368aaa09d51bd63","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-19 17:54:20.000000000","message":"Uploaded patch set 12.","accounts_in_message":[],"_revision_number":12},{"id":"982b0ee77437f1f8d5da7ac88998846a70c567c4","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-19 20:06:01.000000000","message":"Patch Set 12: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/12/check/tempest-full/7328cf6/ : SUCCESS in 1h 50m 47s\n- tempest-full-py3 http://logs.openstack.org/74/663074/12/check/tempest-full-py3/3022b51/ : SUCCESS in 1h 42m 03s\n- openstack-tox-cover http://logs.openstack.org/74/663074/12/check/openstack-tox-cover/90d5c69/cover/ : SUCCESS in 5m 32s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/12/check/openstack-tox-lower-constraints/fa01363/ : SUCCESS in 7m 43s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/12/check/openstack-tox-pep8/764b035/ : FAILURE in 5m 04s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/12/check/openstack-tox-py27/f5a2708/ : SUCCESS in 5m 18s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/12/check/openstack-tox-py36/8bea369/ : SUCCESS in 5m 29s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/12/check/openstack-tox-py37/457096b/ : SUCCESS in 6m 25s\n- openstack-tox-docs http://logs.openstack.org/74/663074/12/check/openstack-tox-docs/57380fe/html/ : SUCCESS in 3m 31s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/12/check/oslo.messaging-tox-py27-func-rabbit/50b311f/ : SUCCESS in 6m 00s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/12/check/oslo.messaging-tox-py27-func-amqp1/6b273c7/ : FAILURE in 11m 42s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/12/check/oslo.messaging-tox-py27-func-kafka/0e3a1b2/ : SUCCESS in 5m 08s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/12/check/oslo.messaging-tox-py36-func-rabbit/2e13d56/ : TIMED_OUT in 40m 56s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/12/check/oslo.messaging-tox-py36-func-amqp1/64f2beb/ : FAILURE in 11m 49s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/12/check/oslo.messaging-tox-py36-func-kafka/bc84a82/ : FAILURE in 7m 54s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/12/check/oslo.messaging-src-dsvm-full-rabbit/2804ff0/ : SUCCESS in 1h 41m 35s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/12/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/14c569c/ : FAILURE in 29m 16s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/12/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/c20443f/ : FAILURE in 39m 19s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/12/check/oslo.messaging-src-dsvm-full-kafka-hybrid/7d82147/ : SUCCESS in 1h 56m 01s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/12/check/oslo.messaging-src-dsvm-full-kafka-centos-7/7e2a88f/ : SUCCESS in 2h 04m 48s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/12/check/oslo.messaging-src-grenade-dsvm/2e3fc9c/ : FAILURE in 12m 22s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/12/check/oslo.messaging-src-grenade-dsvm-multinode/b693e24/ : SUCCESS in 1h 10m 18s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/12/check/oslo.messaging-telemetry-dsvm-integration-rabbit/6a75d10/ : FAILURE in 13m 00s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/12/check/oslo.messaging-telemetry-dsvm-integration-amqp1/73cde36/ : FAILURE in 42m 08s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/12/check/oslo.messaging-telemetry-dsvm-integration-kafka/93803e1/ : FAILURE in 50m 24s (non-voting)","accounts_in_message":[],"_revision_number":12},{"id":"f696c6adef7368ec29dfef91eeca36e19371a418","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-19 23:47:19.000000000","message":"Uploaded patch set 13.","accounts_in_message":[],"_revision_number":13},{"id":"f97f469cd20ce12b76b571dc1a822585639e5b95","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-06-20 02:13:35.000000000","message":"Patch Set 13: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full http://logs.openstack.org/74/663074/13/check/tempest-full/80671ae/ : SUCCESS in 1h 52m 19s\n- tempest-full-py3 http://logs.openstack.org/74/663074/13/check/tempest-full-py3/4ef8779/ : SUCCESS in 1h 33m 41s\n- openstack-tox-cover http://logs.openstack.org/74/663074/13/check/openstack-tox-cover/46d259b/cover/ : SUCCESS in 6m 19s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/13/check/openstack-tox-lower-constraints/a8d8903/ : SUCCESS in 5m 54s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/13/check/openstack-tox-pep8/f9db2f4/ : SUCCESS in 4m 21s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/13/check/openstack-tox-py27/8f9eea6/ : SUCCESS in 5m 19s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/13/check/openstack-tox-py36/72b50d5/ : SUCCESS in 5m 09s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/13/check/openstack-tox-py37/fe97926/ : SUCCESS in 5m 43s\n- openstack-tox-docs http://logs.openstack.org/74/663074/13/check/openstack-tox-docs/a0ae369/html/ : SUCCESS in 3m 23s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/13/check/oslo.messaging-tox-py27-func-rabbit/b57b04e/ : SUCCESS in 5m 18s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/13/check/oslo.messaging-tox-py27-func-amqp1/cb98838/ : FAILURE in 12m 10s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/13/check/oslo.messaging-tox-py27-func-kafka/9824c0b/ : SUCCESS in 5m 17s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/13/check/oslo.messaging-tox-py36-func-rabbit/ae63310/ : SUCCESS in 5m 22s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/13/check/oslo.messaging-tox-py36-func-amqp1/d8c4937/ : FAILURE in 11m 28s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/13/check/oslo.messaging-tox-py36-func-kafka/7d636b7/ : SUCCESS in 5m 34s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/13/check/oslo.messaging-src-dsvm-full-rabbit/074787e/ : SUCCESS in 1h 51m 40s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/13/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/656c581/ : FAILURE in 30m 35s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/13/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/a521abd/ : FAILURE in 41m 21s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/13/check/oslo.messaging-src-dsvm-full-kafka-hybrid/a790b24/ : SUCCESS in 1h 57m 57s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/13/check/oslo.messaging-src-dsvm-full-kafka-centos-7/a232e74/ : SUCCESS in 2h 18m 42s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/13/check/oslo.messaging-src-grenade-dsvm/0263eb2/ : SUCCESS in 57m 45s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/13/check/oslo.messaging-src-grenade-dsvm-multinode/007793a/ : SUCCESS in 1h 11m 08s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/13/check/oslo.messaging-telemetry-dsvm-integration-rabbit/fa1c6f6/ : FAILURE in 43m 33s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/13/check/oslo.messaging-telemetry-dsvm-integration-amqp1/7d65502/ : FAILURE in 46m 18s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/13/check/oslo.messaging-telemetry-dsvm-integration-kafka/c177538/ : FAILURE in 47m 35s (non-voting)","accounts_in_message":[],"_revision_number":13},{"id":"4e422ff64b1a1a12b779ccefe4dd56132bd86c8e","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-20 09:47:17.000000000","message":"Patch Set 13:\n\nI\u0027m not sure if the ConnectionLock can be used from any other code than the rabbit driver... I guess it doesn\u0027t make sense to use it in a standalone way but if this is a real use case then we have a design issue on the rabbit driver.\n\nMy other question is, if the rabbit driver need to by async and the heartbeat choose to use the native python threads and if they share the lock... then I guess we also a design issue.\n\nMy thoughts are related to [1].\n\nI need reviews from peoples who have more experiences with the rabbit driver.\n\n[1] https://review.opendev.org/#/c/663074/13/oslo_messaging/tests/drivers/test_impl_rabbit.py","accounts_in_message":[],"_revision_number":13},{"id":"f342c4a4595caafe8649e87746c319e810810a55","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-06-20 09:51:53.000000000","message":"Patch Set 13:\n\n\u003e I\u0027m not sure if the ConnectionLock can be used from any other code\n \u003e than the rabbit driver... I guess it doesn\u0027t make sense to use it\n \u003e in a standalone way but if this is a real use case then we have a\n \u003e design issue on the rabbit driver.\n\nApparently the ConnectionLock is only used inside the rabbit driver [1] to lock to protect the kombu connection, so I guess it\u0027s not an issue.\n\nThoughts?\n\n[1] https://github.com/search?q\u003dorg%3Aopenstack+ConnectionLock\u0026type\u003dCode\n \u003e \n \u003e My other question is, if the rabbit driver need to by async and the\n \u003e heartbeat choose to use the native python threads and if they share\n \u003e the lock... then I guess we also a design issue.\n \u003e \n \u003e My thoughts are related to [1].\n \u003e \n \u003e I need reviews from peoples who have more experiences with the\n \u003e rabbit driver.\n \u003e \n \u003e [1] https://review.opendev.org/#/c/663074/13/oslo_messaging/tests/drivers/test_impl_rabbit.py","accounts_in_message":[],"_revision_number":13},{"id":"7bd0173b0dd113be0d6af09c52a539cc4d43f054","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2019-07-01 14:58:35.000000000","message":"Patch Set 13: Code-Review+2","accounts_in_message":[],"_revision_number":13},{"id":"5316efaa02b334e0405e91a946cb2de3307bbe6c","author":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"date":"2019-07-03 21:41:19.000000000","message":"Patch Set 13:\n\nI still haven\u0027t had a chance to look closely at this, but I just wanted to provide some initial thoughts since I\u0027m about to leave for a long weekend.\n\nMy main concern with this is that we explicitly recommended against mixing unpatched threads with eventlet in http://specs.openstack.org/openstack/openstack-specs/specs/eventlet-best-practices.html\n\nI guess since this is working it must not be a problem in any of our ci uses of this code, but concurrency issues are tricky like that so it doesn\u0027t give me complete confidence either.\n\nOne other question I had: What specifically broke with the tpool version of this? Both of the Tempest jobs passed as did the rabbit functional tests, which suggests that it was working to some extent.","accounts_in_message":[],"_revision_number":13},{"id":"4dd447f8d1e31a649d362705740f93b6038df967","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-04 08:40:33.000000000","message":"Patch Set 13:\n\n\u003e I still haven\u0027t had a chance to look closely at this, but I just\n \u003e wanted to provide some initial thoughts since I\u0027m about to leave\n \u003e for a long weekend.\n \u003e \n \u003e My main concern with this is that we explicitly recommended against\n \u003e mixing unpatched threads with eventlet in http://specs.openstack.org/openstack/openstack-specs/specs/eventlet-best-practices.html\n \u003e \n \u003e I guess since this is working it must not be a problem in any of\n \u003e our ci uses of this code, but concurrency issues are tricky like\n \u003e that so it doesn\u0027t give me complete confidence either.\n \u003e \n \u003e One other question I had: What specifically broke with the tpool\n \u003e version of this? Both of the Tempest jobs passed as did the rabbit\n \u003e functional tests, which suggests that it was working to some\n \u003e extent.\n\nIt\u0027s not a CI issue with tpool, the main problem with tpool is when I run tests inside an apache environment where the heartbeat continue to exist with the tpool version.\n\nAs I can see during my tests the heartbeat disappear after few minutes:\n\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging \ngit review -d 663074,9 # to setup the patch set number 9 \ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat \npipenv run ./setup-containers.sh # if rabbit server is not running pipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n# then submit a request and observe  http://127.0.0.1:15672/#/connections\ncurl 0.0.0.0:8000\n```\n\nMaybe I\u0027m wrong and my tpool implementation (patch set 9) is not correct, but with tpool version you can observe deconnection after few minutes.\n\nI guess the main reason is that we use the apache MPM `prefork` module (like upstream) and it doesn\u0027t use epoll etc... eventlet/greenlet use epoll and libevent and it\u0027s not compatible with the `prefork` module . The simple fact to use eventlet under apache with the prefork engine is an issue and can\u0027t work at all in this way due to synchronism, idle phase, etc.. on the engine side.\n\nDon\u0027t hesitate to tests with my POC and send me a feedback.\n\nThanks.","accounts_in_message":[],"_revision_number":13},{"id":"b8093fc556cfdf52ff7513e8f52d9ccf81bf0800","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2019-07-04 12:47:18.000000000","message":"Topic changed from bug/1826281-patch3 to bug/1826281","accounts_in_message":[],"_revision_number":13},{"id":"27f0f0cb93fb2a3355c7b6d8c50c9c88c4d90a25","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2019-07-04 13:02:22.000000000","message":"Patch Set 13: Code-Review-1\n\n(7 comments)","accounts_in_message":[],"_revision_number":13},{"id":"e3a88d86c81972269d836160750012b771da0ff1","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-05 14:46:50.000000000","message":"Uploaded patch set 14.","accounts_in_message":[],"_revision_number":14},{"id":"9702740f93a4e99fc054b01c83cbb53e0884d0c9","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-05 15:35:18.000000000","message":"Patch Set 14:\n\n\u003e (7 comments)\n\nI completely reword the commit message to be more clear about what\u0027s the issue and why it occur.","accounts_in_message":[],"_revision_number":14},{"id":"b9a3c664c7dfa0fb6801f324a19a2ebe553c7029","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2019-07-05 16:29:57.000000000","message":"Patch Set 14:\n\n(6 comments)\n\nThis is much better. I still have some complaints but they\u0027re mostly my opinion so no -1. I do need more time to look through this before giving a +2 though","accounts_in_message":[],"_revision_number":14},{"id":"a6808baa2d9fbdde620f818bf050d56fcd4325d4","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-05 17:25:41.000000000","message":"Patch Set 14: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/14/check/tempest-full/bd5000c/ : SUCCESS in 1h 45m 28s\n- tempest-full-py3 http://logs.openstack.org/74/663074/14/check/tempest-full-py3/8faaf22/ : SUCCESS in 1h 39m 36s\n- openstack-tox-cover http://logs.openstack.org/74/663074/14/check/openstack-tox-cover/3b8c35c/cover/ : SUCCESS in 4m 58s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/14/check/openstack-tox-lower-constraints/acfba43/ : SUCCESS in 7m 09s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/14/check/openstack-tox-pep8/3280977/ : SUCCESS in 4m 27s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/14/check/openstack-tox-py27/0ff476d/ : TIMED_OUT in 41m 02s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/14/check/openstack-tox-py36/01cd09f/ : SUCCESS in 6m 56s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/14/check/openstack-tox-py37/41b7e21/ : SUCCESS in 8m 29s\n- openstack-tox-docs http://logs.openstack.org/74/663074/14/check/openstack-tox-docs/e998d1e/html/ : SUCCESS in 3m 05s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/14/check/oslo.messaging-tox-py27-func-rabbit/11b67c7/ : SUCCESS in 5m 47s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/14/check/oslo.messaging-tox-py27-func-amqp1/a59a068/ : FAILURE in 11m 24s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/14/check/oslo.messaging-tox-py27-func-kafka/0053bf1/ : SUCCESS in 5m 28s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/14/check/oslo.messaging-tox-py36-func-rabbit/45d5ac8/ : TIMED_OUT in 41m 00s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/14/check/oslo.messaging-tox-py36-func-amqp1/061f5c7/ : FAILURE in 14m 26s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/14/check/oslo.messaging-tox-py36-func-kafka/1eae415/ : SUCCESS in 5m 05s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/14/check/oslo.messaging-src-dsvm-full-rabbit/3189c79/ : SUCCESS in 2h 00m 08s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/14/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/4c924e0/ : SUCCESS in 1h 46m 24s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/14/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/077e01a/ : SUCCESS in 2h 31m 48s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/14/check/oslo.messaging-src-dsvm-full-kafka-hybrid/176faaf/ : SUCCESS in 1h 51m 51s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/14/check/oslo.messaging-src-dsvm-full-kafka-centos-7/c29e5b7/ : SUCCESS in 2h 12m 56s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/14/check/oslo.messaging-src-grenade-dsvm/df65eb7/ : SUCCESS in 1h 04m 03s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/14/check/oslo.messaging-src-grenade-dsvm-multinode/be89126/ : SUCCESS in 1h 37m 19s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/14/check/oslo.messaging-telemetry-dsvm-integration-rabbit/dd54eb3/ : FAILURE in 43m 58s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/14/check/oslo.messaging-telemetry-dsvm-integration-amqp1/4302620/ : FAILURE in 47m 37s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/14/check/oslo.messaging-telemetry-dsvm-integration-kafka/78136ef/ : FAILURE in 45m 42s (non-voting)","accounts_in_message":[],"_revision_number":14},{"id":"f25a0f116dac014b9f1342ecf54e53dc2e4fea91","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-06 10:38:56.000000000","message":"Uploaded patch set 15: Commit message was updated.","accounts_in_message":[],"_revision_number":15},{"id":"2490e6e1ce1d4a4c838b2d7f002aeb02a9c21665","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-06 10:41:07.000000000","message":"Patch Set 14:\n\n\u003e (6 comments)\n \u003e \n \u003e This is much better. I still have some complaints but they\u0027re\n \u003e mostly my opinion so no -1. I do need more time to look through\n \u003e this before giving a +2 though\n\nThanks Stephen for your useful comments.\n\nI keep the POC part here for the moment to let the reviewers play with it if needed, I\u0027ll add this part to the LP side too, and to the POC project too ASAP.","accounts_in_message":[],"_revision_number":14},{"id":"b5fddde9dab1793e9f144dacbd9e0c6f4fbc8882","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-06 12:41:16.000000000","message":"Patch Set 15: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/15/check/tempest-full/e9e6206/ : SUCCESS in 1h 32m 00s\n- tempest-full-py3 http://logs.openstack.org/74/663074/15/check/tempest-full-py3/f1079c1/ : SUCCESS in 1h 24m 48s\n- openstack-tox-cover http://logs.openstack.org/74/663074/15/check/openstack-tox-cover/5a1e455/cover/ : SUCCESS in 7m 39s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/15/check/openstack-tox-lower-constraints/9365f9a/ : SUCCESS in 6m 03s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/15/check/openstack-tox-pep8/a070ef8/ : SUCCESS in 4m 21s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/15/check/openstack-tox-py27/5cc5602/ : SUCCESS in 7m 27s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/15/check/openstack-tox-py36/f0d6b2e/ : SUCCESS in 5m 39s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/15/check/openstack-tox-py37/0bb915e/ : SUCCESS in 7m 38s\n- openstack-tox-docs http://logs.openstack.org/74/663074/15/check/openstack-tox-docs/4e4e9fc/html/ : SUCCESS in 4m 28s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py27-func-rabbit/c4df767/ : SUCCESS in 6m 51s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py27-func-amqp1/c1c6b92/ : FAILURE in 13m 06s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py27-func-kafka/37fcbc5/ : FAILURE in 5m 26s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py36-func-rabbit/a52b29b/ : SUCCESS in 8m 07s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py36-func-amqp1/51ffef8/ : FAILURE in 11m 34s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py36-func-kafka/0e3698e/ : SUCCESS in 6m 01s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-rabbit/e21d27d/ : FAILURE in 1h 27m 40s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/92e7ec7/ : SUCCESS in 1h 37m 19s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/3da572e/ : SUCCESS in 1h 47m 06s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-kafka-hybrid/3cf65f2/ : SUCCESS in 1h 48m 57s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-kafka-centos-7/66afecb/ : SUCCESS in 1h 57m 06s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-grenade-dsvm/a77e70a/ : SUCCESS in 58m 43s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-grenade-dsvm-multinode/6c5a848/ : SUCCESS in 1h 08m 14s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-telemetry-dsvm-integration-rabbit/3c08966/ : FAILURE in 50m 18s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/15/check/oslo.messaging-telemetry-dsvm-integration-amqp1/7102169/ : FAILURE in 43m 07s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/15/check/oslo.messaging-telemetry-dsvm-integration-kafka/56dcc92/ : FAILURE in 47m 03s (non-voting)","accounts_in_message":[],"_revision_number":15},{"id":"01582000a4999d47829a35e8a409e6640b78cfde","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-08 07:46:06.000000000","message":"Patch Set 15:\n\nrecheck","accounts_in_message":[],"_revision_number":15},{"id":"4108c609503d2c7dfd03e1ee88c71edeaf011f63","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-08 10:07:18.000000000","message":"Patch Set 15: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full http://logs.openstack.org/74/663074/15/check/tempest-full/f1c32bb/ : SUCCESS in 1h 53m 16s\n- tempest-full-py3 http://logs.openstack.org/74/663074/15/check/tempest-full-py3/c4d4d78/ : SUCCESS in 1h 34m 38s\n- openstack-tox-cover http://logs.openstack.org/74/663074/15/check/openstack-tox-cover/4dc7b1a/cover/ : SUCCESS in 5m 53s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/15/check/openstack-tox-lower-constraints/d72dea7/ : SUCCESS in 5m 41s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/15/check/openstack-tox-pep8/1e48e13/ : SUCCESS in 4m 51s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/15/check/openstack-tox-py27/978ae91/ : SUCCESS in 6m 09s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/15/check/openstack-tox-py36/89641e3/ : SUCCESS in 5m 33s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/15/check/openstack-tox-py37/351f1b7/ : SUCCESS in 6m 34s\n- openstack-tox-docs http://logs.openstack.org/74/663074/15/check/openstack-tox-docs/4f1ba76/html/ : SUCCESS in 3m 33s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py27-func-rabbit/224c589/ : SUCCESS in 6m 12s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py27-func-amqp1/0cb1a34/ : FAILURE in 11m 18s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py27-func-kafka/a09375e/ : SUCCESS in 5m 32s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py36-func-rabbit/6f1a9e3/ : TIMED_OUT in 40m 56s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py36-func-amqp1/b4049de/ : FAILURE in 11m 07s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/15/check/oslo.messaging-tox-py36-func-kafka/d9d0e7a/ : SUCCESS in 5m 18s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-rabbit/22bcb84/ : SUCCESS in 1h 49m 55s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/d3dced3/ : SUCCESS in 1h 55m 51s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/3901eb1/ : SUCCESS in 2h 03m 21s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-kafka-hybrid/76c2528/ : SUCCESS in 1h 57m 11s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-dsvm-full-kafka-centos-7/0cd52fb/ : SUCCESS in 2h 07m 44s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-grenade-dsvm/27847fd/ : SUCCESS in 57m 52s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/15/check/oslo.messaging-src-grenade-dsvm-multinode/d1c2516/ : SUCCESS in 1h 17m 02s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/15/check/oslo.messaging-telemetry-dsvm-integration-rabbit/4012341/ : FAILURE in 46m 06s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/15/check/oslo.messaging-telemetry-dsvm-integration-amqp1/83348ea/ : FAILURE in 44m 31s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/15/check/oslo.messaging-telemetry-dsvm-integration-kafka/6da99de/ : FAILURE in 48m 31s (non-voting)","accounts_in_message":[],"_revision_number":15},{"id":"6a9d5cde2b80f60e9a7abb0a258e6fa75150dd03","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2019-07-08 14:34:38.000000000","message":"Patch Set 15:\n\nHi Hervé,\n\nWould you be acceptable to the idea that we make this an \u0027opt-in\u0027 feature?   In other words, applications have to explicitly enable the use of the heartbeat native thread?\n\nAt least for now - maybe mark it as an experimental feature for the next release and let Nova turn it on.  Once they\u0027ve used it for awhile without issue we can then enable it by default in the following release?\n\neventlet is like the \"3rd rail\" of python threading - we\u0027ve been burned by it in the past.\n\nJust a thought....","accounts_in_message":[],"_revision_number":15},{"id":"f9ad118887dd9e3e11a768e68b3316c6a78958c4","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-08 17:32:18.000000000","message":"Patch Set 15:\n\n\u003e Hi Hervé,\n \u003e \n \u003e Would you be acceptable to the idea that we make this an \u0027opt-in\u0027\n \u003e feature?   In other words, applications have to explicitly enable\n \u003e the use of the heartbeat native thread?\n \u003e \n \u003e At least for now - maybe mark it as an experimental feature for the\n \u003e next release and let Nova turn it on.  Once they\u0027ve used it for\n \u003e awhile without issue we can then enable it by default in the\n \u003e following release?\n \u003e \n \u003e eventlet is like the \"3rd rail\" of python threading - we\u0027ve been\n \u003e burned by it in the past.\n \u003e \n \u003e Just a thought....\n\nThis that I\u0027ve proposed in the first design/discussions (https://review.opendev.org/#/c/661314/1), so I\u0027m ok with the option/experimental feature style... \n\nI wanted to let the users choose the right mode to use. \n\nIn other words I wanted to allow users to choose pthread by passing an option at the driver at run if needed or by default continue to use a green thread.\n\nSo it make sense to me to let the introduce these changes as an experimental feature in a first time and when we will acquire more feedback then maybe choose to only keep one execution model.\n\nI will appreciate to also have the bnemec opinion about this.","accounts_in_message":[],"_revision_number":15},{"id":"a36cb075cbe3929b564535a55e227dd748dd6ab7","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-10 12:06:04.000000000","message":"Uploaded patch set 16.","accounts_in_message":[],"_revision_number":16},{"id":"42ad456f1fe3e323a9cb54e89eeb8ea9137f6f45","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-10 12:08:38.000000000","message":"Uploaded patch set 17: Commit message was updated.","accounts_in_message":[],"_revision_number":17},{"id":"e3e5797130f41d6a0d244ccfca497cec61094d45","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-10 12:11:35.000000000","message":"Patch Set 17:\n\nI\u0027ve added the necessary config to activate this feature if needed (experimental mode).\n\nI need to update my POC to pass this config too and use this new execution mode.","accounts_in_message":[],"_revision_number":17},{"id":"0ef7b924894d72ea7a7e2642967f69e8bb6c7dcb","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-10 14:19:11.000000000","message":"Uploaded patch set 18: Commit message was updated.","accounts_in_message":[],"_revision_number":18},{"id":"a7c072755e1bdd8b9c6e3725f3e8d1a292b970b9","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-10 17:05:36.000000000","message":"Patch Set 18: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/18/check/tempest-full/7733782/ : SUCCESS in 2h 07m 14s\n- tempest-full-py3 http://logs.openstack.org/74/663074/18/check/tempest-full-py3/1e82d2c/ : SUCCESS in 1h 38m 39s\n- openstack-tox-cover http://logs.openstack.org/74/663074/18/check/openstack-tox-cover/b538abd/cover/ : SUCCESS in 6m 53s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/18/check/openstack-tox-lower-constraints/6f9c8dd/ : SUCCESS in 7m 17s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/18/check/openstack-tox-pep8/69c7d55/ : FAILURE in 4m 40s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/18/check/openstack-tox-py27/d1b2d03/ : SUCCESS in 6m 46s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/18/check/openstack-tox-py36/6c43eee/ : SUCCESS in 6m 51s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/18/check/openstack-tox-py37/158f630/ : SUCCESS in 5m 42s\n- openstack-tox-docs http://logs.openstack.org/74/663074/18/check/openstack-tox-docs/5788a7d/html/ : SUCCESS in 4m 30s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/18/check/oslo.messaging-tox-py27-func-rabbit/ddc5ec4/ : SUCCESS in 5m 29s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/18/check/oslo.messaging-tox-py27-func-amqp1/c25b8b1/ : FAILURE in 11m 50s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/18/check/oslo.messaging-tox-py27-func-kafka/4c4769a/ : SUCCESS in 8m 01s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/18/check/oslo.messaging-tox-py36-func-rabbit/86c234e/ : SUCCESS in 6m 59s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/18/check/oslo.messaging-tox-py36-func-amqp1/de5a2b3/ : FAILURE in 13m 22s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/18/check/oslo.messaging-tox-py36-func-kafka/3669149/ : SUCCESS in 7m 43s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/18/check/oslo.messaging-src-dsvm-full-rabbit/d93d5ac/ : SUCCESS in 1h 36m 26s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/18/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/d3f4e17/ : SUCCESS in 2h 01m 50s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/18/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/55dd9c5/ : SUCCESS in 2h 40m 09s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/18/check/oslo.messaging-src-dsvm-full-kafka-hybrid/98ee652/ : SUCCESS in 2h 00m 35s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/18/check/oslo.messaging-src-dsvm-full-kafka-centos-7/0727a05/ : SUCCESS in 2h 30m 02s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/18/check/oslo.messaging-src-grenade-dsvm/1bb66fa/ : SUCCESS in 59m 27s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/18/check/oslo.messaging-src-grenade-dsvm-multinode/16e632f/ : SUCCESS in 1h 25m 56s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/18/check/oslo.messaging-telemetry-dsvm-integration-rabbit/f813b1a/ : FAILURE in 46m 09s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/18/check/oslo.messaging-telemetry-dsvm-integration-amqp1/089fbf0/ : FAILURE in 45m 40s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/18/check/oslo.messaging-telemetry-dsvm-integration-kafka/8beb100/ : FAILURE in 47m 09s (non-voting)","accounts_in_message":[],"_revision_number":18},{"id":"6fd08b2f0fe32f7ee293ba9bb0520b13795cbdcd","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-11 06:43:00.000000000","message":"Uploaded patch set 19.","accounts_in_message":[],"_revision_number":19},{"id":"8afb319464832c2cb91544eb6b473d6adb059a1b","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-11 09:19:09.000000000","message":"Patch Set 19: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full http://logs.openstack.org/74/663074/19/check/tempest-full/4fe2047/ : TIMED_OUT in 2h 06m 09s\n- tempest-full-py3 http://logs.openstack.org/74/663074/19/check/tempest-full-py3/62ff1d6/ : SUCCESS in 1h 43m 48s\n- openstack-tox-cover http://logs.openstack.org/74/663074/19/check/openstack-tox-cover/26b47f6/cover/ : SUCCESS in 5m 08s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/19/check/openstack-tox-lower-constraints/46e79a2/ : SUCCESS in 5m 31s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/19/check/openstack-tox-pep8/8b16ba3/ : SUCCESS in 3m 32s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/19/check/openstack-tox-py27/5583c1f/ : SUCCESS in 6m 20s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/19/check/openstack-tox-py36/2fec4e8/ : SUCCESS in 6m 59s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/19/check/openstack-tox-py37/252297b/ : SUCCESS in 7m 30s\n- openstack-tox-docs http://logs.openstack.org/74/663074/19/check/openstack-tox-docs/9a6503e/html/ : SUCCESS in 3m 49s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py27-func-rabbit/060d2f0/ : SUCCESS in 5m 26s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py27-func-amqp1/0459561/ : FAILURE in 12m 01s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py27-func-kafka/1a37f75/ : FAILURE in 7m 49s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py36-func-rabbit/eaaa689/ : SUCCESS in 5m 25s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py36-func-amqp1/955d4ce/ : FAILURE in 11m 55s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py36-func-kafka/e008494/ : SUCCESS in 7m 02s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-rabbit/38aad53/ : SUCCESS in 1h 44m 30s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/ffa7381/ : SUCCESS in 1h 42m 20s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/fb1a4ee/ : SUCCESS in 2h 22m 53s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-kafka-hybrid/d11cc81/ : FAILURE in 1h 43m 55s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-kafka-centos-7/0d591cb/ : SUCCESS in 2h 00m 15s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-grenade-dsvm/89ad51a/ : SUCCESS in 58m 49s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-grenade-dsvm-multinode/2732345/ : SUCCESS in 1h 22m 17s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-telemetry-dsvm-integration-rabbit/2ec9c36/ : FAILURE in 51m 40s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/19/check/oslo.messaging-telemetry-dsvm-integration-amqp1/a7df8d9/ : FAILURE in 52m 36s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/19/check/oslo.messaging-telemetry-dsvm-integration-kafka/be732a8/ : FAILURE in 53m 28s (non-voting)","accounts_in_message":[],"_revision_number":19},{"id":"7b12260c255ed7fd7c7cdf1bcbc2134a59fedb9d","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-11 09:28:32.000000000","message":"Patch Set 19:\n\nrecheck","accounts_in_message":[],"_revision_number":19},{"id":"13fde3b71feeba0b2b3e14fd2e60d3b1d85b8bc3","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-11 11:56:44.000000000","message":"Patch Set 19: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full http://logs.openstack.org/74/663074/19/check/tempest-full/8fa9ab0/ : SUCCESS in 1h 50m 06s\n- tempest-full-py3 http://logs.openstack.org/74/663074/19/check/tempest-full-py3/16ce79e/ : SUCCESS in 1h 33m 37s\n- openstack-tox-cover http://logs.openstack.org/74/663074/19/check/openstack-tox-cover/ef7c5f3/cover/ : SUCCESS in 6m 26s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/19/check/openstack-tox-lower-constraints/5eb874b/ : SUCCESS in 6m 13s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/19/check/openstack-tox-pep8/0da1028/ : SUCCESS in 4m 11s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/19/check/openstack-tox-py27/a05a589/ : SUCCESS in 10m 42s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/19/check/openstack-tox-py36/7af8513/ : SUCCESS in 9m 08s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/19/check/openstack-tox-py37/d787fc4/ : SUCCESS in 5m 49s\n- openstack-tox-docs http://logs.openstack.org/74/663074/19/check/openstack-tox-docs/83153bd/html/ : SUCCESS in 3m 26s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py27-func-rabbit/bdb6273/ : SUCCESS in 5m 28s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py27-func-amqp1/ee4c301/ : FAILURE in 11m 48s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py27-func-kafka/dc39593/ : SUCCESS in 5m 36s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py36-func-rabbit/c8bc2cb/ : SUCCESS in 5m 49s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py36-func-amqp1/d4dd328/ : FAILURE in 12m 19s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/19/check/oslo.messaging-tox-py36-func-kafka/df8cd35/ : SUCCESS in 5m 19s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-rabbit/1befbfd/ : SUCCESS in 1h 42m 56s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/1f919f3/ : SUCCESS in 1h 40m 19s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/edf2f43/ : SUCCESS in 2h 21m 30s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-kafka-hybrid/443a02e/ : SUCCESS in 1h 51m 15s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-dsvm-full-kafka-centos-7/07e576e/ : SUCCESS in 2h 12m 24s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-grenade-dsvm/0a79bc6/ : SUCCESS in 56m 29s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/19/check/oslo.messaging-src-grenade-dsvm-multinode/f680c7c/ : SUCCESS in 1h 12m 31s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/19/check/oslo.messaging-telemetry-dsvm-integration-rabbit/9a5a089/ : FAILURE in 47m 22s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/19/check/oslo.messaging-telemetry-dsvm-integration-amqp1/70be6c9/ : FAILURE in 1h 02m 26s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/19/check/oslo.messaging-telemetry-dsvm-integration-kafka/b803d3e/ : FAILURE in 49m 50s (non-voting)","accounts_in_message":[],"_revision_number":19},{"id":"38974096d9cc0a466aedc62f0663364e7a594feb","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2019-07-22 12:46:21.000000000","message":"Patch Set 19: Code-Review+1\n\nTentative +1 - I\u0027ll upgrade this to +2 but I\u0027d like to get Gabriel\u0027s input, as well as let Stephen time to take a closer look.\n\nIt would be ideal to follow up this patch with a separate patch that adds a rabbit specific functional test (see tests/functional/test_rabbitmq.py) for this feature.  That way it gets tested against rabbitmq running in the gate.","accounts_in_message":[],"_revision_number":19},{"id":"4372822c8ba7b19bfc845445301b01a775fa0c90","author":{"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},"date":"2019-07-22 13:46:11.000000000","message":"Patch Set 19: Code-Review+1\n\nI agree to have a separate thread for the heartbeat instead of a green thread. I will put +1, even if I don\u0027t have a lot of experience with eventlet. The code is safe and also configurable. So I don\u0027t see any problem with this PR.","accounts_in_message":[],"_revision_number":19},{"id":"899594cf3724e15c7337d487b3f476f8ed36cb4a","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-22 14:02:22.000000000","message":"Patch Set 19:\n\n\u003e Tentative +1 - I\u0027ll upgrade this to +2 but I\u0027d like to get\n \u003e Gabriel\u0027s input, as well as let Stephen time to take a closer look.\n \u003e \n \u003e It would be ideal to follow up this patch with a separate patch\n \u003e that adds a rabbit specific functional test (see tests/functional/test_rabbitmq.py)\n \u003e for this feature.  That way it gets tested against rabbitmq running\n \u003e in the gate.\n\nI can try to add some functional tests in a follow up but I\u0027m not sure that it\u0027s really relevant since the issue depends on the fact that we run the rabbitmq driver under apache and mod_wsgi, so it\u0027s not a functional issue between a rabbitmq server and our code, it\u0027s an issue between us and our execution environment.\n\nDo you\u0027ve some ideas of implementable tests?","accounts_in_message":[],"_revision_number":19},{"id":"b4912babfa2ff3c12c3c23726530b4fea2d906bf","author":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"date":"2019-07-24 19:19:06.000000000","message":"Patch Set 19:\n\n(2 comments)\n\nSome thoughts inline. If what I\u0027m proposing is unnecessary (like we only create connections rarely) then I\u0027m okay with the patch as-is.\n\nI\u0027ll note that this still scares the wits out of me, but it\u0027s optional and experimental so it provides a way to get the behavior out in the wild and test it in real-world environments that are encountering this bug.\n\nAs I said a long time ago, if this proves to be the right thing to do I would like to see us switch it on by default at some point, but given my reservations about mixing patched and unpatched threading code I\u0027m okay with it being optional for now.","accounts_in_message":[],"_revision_number":19},{"id":"418fca62fcb11468b42795b1d9da00692f9ac581","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-29 09:56:52.000000000","message":"Uploaded patch set 20.","accounts_in_message":[],"_revision_number":20},{"id":"2f8fa8d691008feebc94fc2b84ffaf211e508e25","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-07-29 09:58:17.000000000","message":"Patch Set 20:\n\n\u003e (2 comments)\n \u003e \n \u003e Some thoughts inline. If what I\u0027m proposing is unnecessary (like we\n \u003e only create connections rarely) then I\u0027m okay with the patch as-is.\n \u003e \n \u003e I\u0027ll note that this still scares the wits out of me, but it\u0027s\n \u003e optional and experimental so it provides a way to get the behavior\n \u003e out in the wild and test it in real-world environments that are\n \u003e encountering this bug.\n \u003e \n \u003e As I said a long time ago, if this proves to be the right thing to\n \u003e do I would like to see us switch it on by default at some point,\n \u003e but given my reservations about mixing patched and unpatched\n \u003e threading code I\u0027m okay with it being optional for now.\n\nThanks for your feedback, I\u0027ve update my code to introduce a more efficient import step.","accounts_in_message":[],"_revision_number":20},{"id":"e931a44a4e0060c2ae2581b07d30306ebd9a6702","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-07-29 12:19:14.000000000","message":"Patch Set 20: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full http://logs.openstack.org/74/663074/20/check/tempest-full/9be98c1/ : SUCCESS in 1h 42m 16s\n- tempest-full-py3 http://logs.openstack.org/74/663074/20/check/tempest-full-py3/7de0753/ : SUCCESS in 1h 25m 30s\n- openstack-tox-cover http://logs.openstack.org/74/663074/20/check/openstack-tox-cover/091c267/cover/ : SUCCESS in 6m 12s\n- openstack-tox-lower-constraints http://logs.openstack.org/74/663074/20/check/openstack-tox-lower-constraints/94f8d5f/ : SUCCESS in 6m 56s\n- openstack-tox-pep8 http://logs.openstack.org/74/663074/20/check/openstack-tox-pep8/e630e0d/ : SUCCESS in 3m 05s\n- openstack-tox-py27 http://logs.openstack.org/74/663074/20/check/openstack-tox-py27/5fa6e05/ : SUCCESS in 7m 00s\n- openstack-tox-py36 http://logs.openstack.org/74/663074/20/check/openstack-tox-py36/b4e456e/ : SUCCESS in 6m 15s\n- openstack-tox-py37 http://logs.openstack.org/74/663074/20/check/openstack-tox-py37/84d6560/ : SUCCESS in 6m 23s\n- openstack-tox-docs http://logs.openstack.org/74/663074/20/check/openstack-tox-docs/3e87c3f/html/ : SUCCESS in 4m 38s\n- oslo.messaging-tox-py27-func-rabbit http://logs.openstack.org/74/663074/20/check/oslo.messaging-tox-py27-func-rabbit/9728787/ : SUCCESS in 5m 53s\n- oslo.messaging-tox-py27-func-amqp1 http://logs.openstack.org/74/663074/20/check/oslo.messaging-tox-py27-func-amqp1/69ca1a0/ : FAILURE in 12m 26s (non-voting)\n- oslo.messaging-tox-py27-func-kafka http://logs.openstack.org/74/663074/20/check/oslo.messaging-tox-py27-func-kafka/dd3cc47/ : SUCCESS in 5m 25s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit http://logs.openstack.org/74/663074/20/check/oslo.messaging-tox-py36-func-rabbit/43a2a16/ : SUCCESS in 6m 03s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 http://logs.openstack.org/74/663074/20/check/oslo.messaging-tox-py36-func-amqp1/7ac74eb/ : FAILURE in 12m 35s (non-voting)\n- oslo.messaging-tox-py36-func-kafka http://logs.openstack.org/74/663074/20/check/oslo.messaging-tox-py36-func-kafka/86c8f00/ : SUCCESS in 6m 10s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit http://logs.openstack.org/74/663074/20/check/oslo.messaging-src-dsvm-full-rabbit/87b3b28/ : SUCCESS in 1h 40m 21s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid http://logs.openstack.org/74/663074/20/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/c985a38/ : SUCCESS in 1h 33m 57s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 http://logs.openstack.org/74/663074/20/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/7453a06/ : SUCCESS in 1h 59m 05s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid http://logs.openstack.org/74/663074/20/check/oslo.messaging-src-dsvm-full-kafka-hybrid/568277f/ : SUCCESS in 1h 33m 06s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 http://logs.openstack.org/74/663074/20/check/oslo.messaging-src-dsvm-full-kafka-centos-7/c052cc0/ : SUCCESS in 2h 17m 01s (non-voting)\n- oslo.messaging-src-grenade-dsvm http://logs.openstack.org/74/663074/20/check/oslo.messaging-src-grenade-dsvm/4c068ae/ : SUCCESS in 1h 02m 07s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode http://logs.openstack.org/74/663074/20/check/oslo.messaging-src-grenade-dsvm-multinode/4760ced/ : SUCCESS in 1h 22m 34s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit http://logs.openstack.org/74/663074/20/check/oslo.messaging-telemetry-dsvm-integration-rabbit/294b889/ : FAILURE in 47m 59s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 http://logs.openstack.org/74/663074/20/check/oslo.messaging-telemetry-dsvm-integration-amqp1/b513f8f/ : FAILURE in 49m 43s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka http://logs.openstack.org/74/663074/20/check/oslo.messaging-telemetry-dsvm-integration-kafka/8d40324/ : FAILURE in 53m 34s (non-voting)","accounts_in_message":[],"_revision_number":20},{"id":"403ef4549541e1b3b3f7769d006ba5d20802bd54","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2019-07-29 15:17:14.000000000","message":"Patch Set 20: Code-Review+2","accounts_in_message":[],"_revision_number":20},{"id":"7947310827de93edf0da15c8947629b349a293ec","author":{"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},"date":"2019-08-01 10:55:38.000000000","message":"Patch Set 20: Code-Review+1","accounts_in_message":[],"_revision_number":20},{"id":"84608a157db3f5170fba7c9a98a86237b1f6e6c0","author":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"date":"2019-08-07 14:17:53.000000000","message":"Patch Set 20:\n\n(4 comments)\n\nA few nits, and one potential issue inline. It would be nice to make this safe so if someone sets this in a non-eventlet service it doesn\u0027t blow up.","accounts_in_message":[],"_revision_number":20},{"id":"c4ed3da0a98e45698422508b2f6d283ef5bd43a2","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-08-07 15:24:01.000000000","message":"Uploaded patch set 21.","accounts_in_message":[],"_revision_number":21},{"id":"da491f42c2b21107c5511a4ff675bd064d59ab73","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2019-08-07 15:27:07.000000000","message":"Patch Set 21:\n\n\u003e (4 comments)\n \u003e \n \u003e A few nits, and one potential issue inline. It would be nice to\n \u003e make this safe so if someone sets this in a non-eventlet service it\n \u003e doesn\u0027t blow up.\n\nDone.\n\nI\u0027ve reword the commit msg to provide a more lean and focused explaination.\n\nI\u0027ve handle the case we will run the driver outside a monkey patched environment with a driver config who ask for a standard python thread.","accounts_in_message":[],"_revision_number":21},{"id":"d44b47231b7e9865b3f859b10ab2fea5a2b41bde","author":{"_account_id":29222,"name":"Natal Ngétal","email":"hobbestigrou@erakis.eu","username":"hobbestigrou"},"date":"2019-08-07 15:37:11.000000000","message":"Patch Set 21: Code-Review+1","accounts_in_message":[],"_revision_number":21},{"id":"88c109d95c5d7e515f6e6d4eb55a9a391ecf98fe","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-07 17:50:41.000000000","message":"Patch Set 21: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full https://logs.opendev.org/74/663074/21/check/tempest-full/c2850c8/ : FAILURE in 1h 25m 01s\n- tempest-full-py3 https://logs.opendev.org/74/663074/21/check/tempest-full-py3/a0cea2d/ : SUCCESS in 1h 54m 18s\n- openstack-tox-cover https://logs.opendev.org/74/663074/21/check/openstack-tox-cover/a537222/cover/ : SUCCESS in 6m 36s\n- openstack-tox-lower-constraints https://logs.opendev.org/74/663074/21/check/openstack-tox-lower-constraints/3a0cdfe/ : SUCCESS in 7m 46s\n- openstack-tox-pep8 https://logs.opendev.org/74/663074/21/check/openstack-tox-pep8/23562ec/ : SUCCESS in 4m 38s\n- openstack-tox-py27 https://logs.opendev.org/74/663074/21/check/openstack-tox-py27/5fb2741/ : SUCCESS in 6m 05s\n- openstack-tox-py36 https://logs.opendev.org/74/663074/21/check/openstack-tox-py36/48b05fd/ : SUCCESS in 6m 01s\n- openstack-tox-py37 https://logs.opendev.org/74/663074/21/check/openstack-tox-py37/27d5276/ : SUCCESS in 6m 45s\n- openstack-tox-docs https://logs.opendev.org/74/663074/21/check/openstack-tox-docs/2534fa4/html/ : SUCCESS in 3m 09s\n- oslo.messaging-tox-py27-func-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py27-func-rabbit/ffdcafc/ : FAILURE in 7m 38s\n- oslo.messaging-tox-py27-func-amqp1 https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py27-func-amqp1/738ab8b/ : FAILURE in 13m 17s (non-voting)\n- oslo.messaging-tox-py27-func-kafka https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py27-func-kafka/3590c6a/ : SUCCESS in 5m 53s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py36-func-rabbit/87b8b92/ : SUCCESS in 5m 29s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py36-func-amqp1/435ced9/ : FAILURE in 11m 44s (non-voting)\n- oslo.messaging-tox-py36-func-kafka https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py36-func-kafka/21c482c/ : FAILURE in 4m 57s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-rabbit/b92be70/ : SUCCESS in 1h 36m 24s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/b345b0e/ : SUCCESS in 1h 46m 31s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/553da56/ : SUCCESS in 1h 54m 25s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-kafka-hybrid/20e797d/ : SUCCESS in 1h 40m 51s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-kafka-centos-7/ef5f66b/ : SUCCESS in 2h 19m 47s (non-voting)\n- oslo.messaging-src-grenade-dsvm https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-grenade-dsvm/c680e5f/ : SUCCESS in 59m 30s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-grenade-dsvm-multinode/69690e8/ : SUCCESS in 1h 20m 58s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-telemetry-dsvm-integration-rabbit/52d92a7/ : FAILURE in 49m 21s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 https://logs.opendev.org/74/663074/21/check/oslo.messaging-telemetry-dsvm-integration-amqp1/5f10ba8/ : FAILURE in 46m 26s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka https://logs.opendev.org/74/663074/21/check/oslo.messaging-telemetry-dsvm-integration-kafka/cf6a6b6/ : FAILURE in 45m 20s (non-voting)","accounts_in_message":[],"_revision_number":21},{"id":"f6343cc0ed7befafcb9845e6bfac7d9731fa57fc","author":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"date":"2019-08-07 20:48:09.000000000","message":"Patch Set 21: Code-Review+2\n\nrecheck\n\nTempest failure appears to be a cinder-backup heisenbug, not sure about the functional test, but it seems unlikely the voting py27 job failed and the non-voting py36 passed for a legit failure.\n\nI guess the spec is kind of the \"bug\" for tracking this, so we should probably get that updated to reflect the current state.","accounts_in_message":[],"_revision_number":21},{"id":"dcec0adfe6555834ecca575d198aa47a6be2bff5","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-07 23:10:58.000000000","message":"Patch Set 21: Verified+1\n\nBuild succeeded (check pipeline).\n\n- tempest-full https://logs.opendev.org/74/663074/21/check/tempest-full/c354058/ : SUCCESS in 1h 32m 55s\n- tempest-full-py3 https://logs.opendev.org/74/663074/21/check/tempest-full-py3/4f589a1/ : SUCCESS in 1h 16m 06s\n- openstack-tox-cover https://logs.opendev.org/74/663074/21/check/openstack-tox-cover/daa2734/cover/ : SUCCESS in 6m 20s\n- openstack-tox-lower-constraints https://logs.opendev.org/74/663074/21/check/openstack-tox-lower-constraints/8a210f0/ : SUCCESS in 6m 33s\n- openstack-tox-pep8 https://logs.opendev.org/74/663074/21/check/openstack-tox-pep8/830d849/ : SUCCESS in 4m 38s\n- openstack-tox-py27 https://logs.opendev.org/74/663074/21/check/openstack-tox-py27/49c8e28/ : SUCCESS in 6m 35s\n- openstack-tox-py36 https://logs.opendev.org/74/663074/21/check/openstack-tox-py36/60a0098/ : SUCCESS in 6m 09s\n- openstack-tox-py37 https://logs.opendev.org/74/663074/21/check/openstack-tox-py37/167935f/ : SUCCESS in 6m 57s\n- openstack-tox-docs https://logs.opendev.org/74/663074/21/check/openstack-tox-docs/cf660ee/html/ : SUCCESS in 4m 11s\n- oslo.messaging-tox-py27-func-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py27-func-rabbit/b5167dc/ : SUCCESS in 7m 37s\n- oslo.messaging-tox-py27-func-amqp1 https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py27-func-amqp1/c9b4b07/ : FAILURE in 12m 26s (non-voting)\n- oslo.messaging-tox-py27-func-kafka https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py27-func-kafka/5876fe9/ : FAILURE in 5m 20s (non-voting)\n- oslo.messaging-tox-py36-func-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py36-func-rabbit/c83da5b/ : SUCCESS in 6m 09s (non-voting)\n- oslo.messaging-tox-py36-func-amqp1 https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py36-func-amqp1/80fa202/ : FAILURE in 12m 48s (non-voting)\n- oslo.messaging-tox-py36-func-kafka https://logs.opendev.org/74/663074/21/check/oslo.messaging-tox-py36-func-kafka/dbe452d/ : SUCCESS in 5m 58s (non-voting)\n- oslo.messaging-src-dsvm-full-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-rabbit/0d33526/ : SUCCESS in 1h 31m 37s\n- oslo.messaging-src-dsvm-full-amqp1-hybrid https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-amqp1-hybrid/2c8a6b9/ : SUCCESS in 1h 37m 28s (non-voting)\n- oslo.messaging-src-dsvm-full-amqp1-centos-7 https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-amqp1-centos-7/7e991f7/ : SUCCESS in 2h 17m 48s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-hybrid https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-kafka-hybrid/6599bb4/ : SUCCESS in 1h 34m 07s (non-voting)\n- oslo.messaging-src-dsvm-full-kafka-centos-7 https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-dsvm-full-kafka-centos-7/134febf/ : SUCCESS in 2h 03m 09s (non-voting)\n- oslo.messaging-src-grenade-dsvm https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-grenade-dsvm/a632d4e/ : SUCCESS in 1h 08m 48s (non-voting)\n- oslo.messaging-src-grenade-dsvm-multinode https://logs.opendev.org/74/663074/21/check/oslo.messaging-src-grenade-dsvm-multinode/a699008/ : SUCCESS in 1h 11m 10s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-rabbit https://logs.opendev.org/74/663074/21/check/oslo.messaging-telemetry-dsvm-integration-rabbit/230f5a6/ : FAILURE in 47m 13s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-amqp1 https://logs.opendev.org/74/663074/21/check/oslo.messaging-telemetry-dsvm-integration-amqp1/8490e8e/ : FAILURE in 46m 52s (non-voting)\n- oslo.messaging-telemetry-dsvm-integration-kafka https://logs.opendev.org/74/663074/21/check/oslo.messaging-telemetry-dsvm-integration-kafka/f4aac08/ : FAILURE in 56m 47s (non-voting)","accounts_in_message":[],"_revision_number":21},{"id":"3efc4b7330d68aa0a6734ffb891bcdbe89b61abd","author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"date":"2019-08-08 13:59:01.000000000","message":"Patch Set 21: Code-Review+2 Workflow+1\n\nWFM","accounts_in_message":[],"_revision_number":21},{"id":"e37a3e2fa50abd203f36e18d473a685d1784d2d4","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-08 13:59:15.000000000","message":"Patch Set 21: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":21},{"id":"a6f97636da422e4b538a727f0b00d7e531a68cb5","author":{"_account_id":29364,"name":"Gabriele Santomaggio","email":"g.santomaggio@gmail.com","username":"gsantomaggio"},"date":"2019-08-08 14:07:39.000000000","message":"Patch Set 21: Code-Review+1","accounts_in_message":[],"_revision_number":21},{"id":"3fe571ea42a81c837a21dea1eed4d171a573d880","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-08 15:50:07.000000000","message":"Patch Set 21: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- tempest-full https://logs.opendev.org/74/663074/21/gate/tempest-full/4b111a2/ : SUCCESS in 1h 48m 07s\n- tempest-full-py3 https://logs.opendev.org/74/663074/21/gate/tempest-full-py3/b07c4cc/ : SUCCESS in 1h 28m 34s\n- openstack-tox-lower-constraints https://logs.opendev.org/74/663074/21/gate/openstack-tox-lower-constraints/a7eaadc/ : SUCCESS in 6m 33s\n- openstack-tox-pep8 https://logs.opendev.org/74/663074/21/gate/openstack-tox-pep8/b15ffca/ : SUCCESS in 5m 07s\n- openstack-tox-py27 https://logs.opendev.org/74/663074/21/gate/openstack-tox-py27/e9eb12d/ : SUCCESS in 5m 10s\n- openstack-tox-py36 https://logs.opendev.org/74/663074/21/gate/openstack-tox-py36/98ac95c/ : SUCCESS in 6m 54s\n- openstack-tox-py37 https://logs.opendev.org/74/663074/21/gate/openstack-tox-py37/acce311/ : SUCCESS in 10m 20s\n- openstack-tox-docs https://logs.opendev.org/74/663074/21/gate/openstack-tox-docs/1223475/html/ : SUCCESS in 4m 17s\n- oslo.messaging-tox-py27-func-rabbit https://logs.opendev.org/74/663074/21/gate/oslo.messaging-tox-py27-func-rabbit/233ae55/ : SUCCESS in 5m 48s\n- oslo.messaging-src-dsvm-full-rabbit https://logs.opendev.org/74/663074/21/gate/oslo.messaging-src-dsvm-full-rabbit/08c5392/ : SUCCESS in 1h 37m 18s","accounts_in_message":[],"_revision_number":21},{"id":"f8c3da130245b1107c4576d8421088ed87b5c9ae","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-08-08 15:50:07.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":21}],"current_revision_number":21,"current_revision":"22f240b82fffbd62be8568a7d0d3369134596ace","revisions":{"9f4d387e344c4ba0959cf0f1f39f0fa1b58688fd":{"kind":"REWORK","_number":1,"created":"2019-06-04 15:54:44.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/1"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\nPartial-Bug: #1826281\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/9f4d387e344c4ba0959cf0f1f39f0fa1b58688fd"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/9f4d387e344c4ba0959cf0f1f39f0fa1b58688fd"}]},"branch":"refs/heads/master"},"2bb037cbe4b352c485bf06d2ac6d53b470adc74f":{"kind":"NO_CODE_CHANGE","_number":2,"created":"2019-06-04 15:56:41.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/2"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:56:22.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/2bb037cbe4b352c485bf06d2ac6d53b470adc74f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/2bb037cbe4b352c485bf06d2ac6d53b470adc74f"}]},"branch":"refs/heads/master"},"47adbaaf5de55f502d16c0b2e00a6f80492be051":{"kind":"NO_CODE_CHANGE","_number":3,"created":"2019-06-04 17:00:39.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/3"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 16:58:11.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nYou can check the new behavior by using the following gist:\n- https://gist.github.com/4383/ad316b498b6d6b2beaed96f7a2f9b38c (without\nsimilar patch changes)\n\n- https://gist.github.com/4383/1ac3a2c2cbd6d7a54046d4585ecef151 (with\nchecks/changes who looks like to this patch)\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/47adbaaf5de55f502d16c0b2e00a6f80492be051"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/47adbaaf5de55f502d16c0b2e00a6f80492be051"}]},"branch":"refs/heads/master"},"322b1dea204d5f142b7d78f8677515ffaf07cdd2":{"kind":"REWORK","_number":4,"created":"2019-06-05 10:32:57.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/4"}}},"commit":{"parents":[{"commit":"da6caaef0701e46f24869a33c5875b36e0e0f9af","subject":"Add thread name to the RabbitMQ heartbeat thread","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/da6caaef0701e46f24869a33c5875b36e0e0f9af"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-05 10:32:32.000000000","tz":120},"subject":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.","message":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nYou can check the new behavior by using the following gist:\n- https://gist.github.com/4383/ad316b498b6d6b2beaed96f7a2f9b38c (without\nsimilar patch changes)\n\n- https://gist.github.com/4383/1ac3a2c2cbd6d7a54046d4585ecef151 (with\nchecks/changes who looks like to this patch)\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/322b1dea204d5f142b7d78f8677515ffaf07cdd2"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/322b1dea204d5f142b7d78f8677515ffaf07cdd2"}]},"branch":"refs/heads/master"},"de5b27d6007683047fb9d1603abc2396b953d271":{"kind":"REWORK","_number":5,"created":"2019-06-05 10:35:28.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/5"}}},"commit":{"parents":[{"commit":"da6caaef0701e46f24869a33c5875b36e0e0f9af","subject":"Add thread name to the RabbitMQ heartbeat thread","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/da6caaef0701e46f24869a33c5875b36e0e0f9af"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-05 10:35:19.000000000","tz":120},"subject":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.","message":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nYou can check the new behavior by using the following gist:\n- https://gist.github.com/4383/ad316b498b6d6b2beaed96f7a2f9b38c (without\nsimilar patch changes)\n\n- https://gist.github.com/4383/1ac3a2c2cbd6d7a54046d4585ecef151 (with\nchecks/changes who looks like to this patch)\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/de5b27d6007683047fb9d1603abc2396b953d271"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/de5b27d6007683047fb9d1603abc2396b953d271"}]},"branch":"refs/heads/master"},"b2d4cb46ed3a8ea7a4e8f103e0b6f0aa362eb3ba":{"kind":"REWORK","_number":6,"created":"2019-06-05 14:10:39.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/6"}}},"commit":{"parents":[{"commit":"da6caaef0701e46f24869a33c5875b36e0e0f9af","subject":"Add thread name to the RabbitMQ heartbeat thread","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/da6caaef0701e46f24869a33c5875b36e0e0f9af"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-05 14:10:23.000000000","tz":120},"subject":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.","message":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nYou can check the new behavior by using the following gist:\n- https://gist.github.com/4383/ad316b498b6d6b2beaed96f7a2f9b38c (without\nsimilar patch changes)\n\n- https://gist.github.com/4383/1ac3a2c2cbd6d7a54046d4585ecef151 (with\nchecks/changes who looks like to this patch)\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b2d4cb46ed3a8ea7a4e8f103e0b6f0aa362eb3ba"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b2d4cb46ed3a8ea7a4e8f103e0b6f0aa362eb3ba"}]},"branch":"refs/heads/master"},"76cf41f0c5e8c4da1bd9ad77b09a658e85d34a04":{"kind":"NO_CODE_CHANGE","_number":7,"created":"2019-06-05 14:52:21.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/7","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/7","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/7"}}},"commit":{"parents":[{"commit":"da6caaef0701e46f24869a33c5875b36e0e0f9af","subject":"Add thread name to the RabbitMQ heartbeat thread","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/da6caaef0701e46f24869a33c5875b36e0e0f9af"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-05 14:51:42.000000000","tz":120},"subject":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.","message":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/76cf41f0c5e8c4da1bd9ad77b09a658e85d34a04"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/76cf41f0c5e8c4da1bd9ad77b09a658e85d34a04"}]},"branch":"refs/heads/master"},"de5ac2850adc9b520be02c879ac38cf5d2877ab1":{"kind":"REWORK","_number":8,"created":"2019-06-05 15:57:30.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/8","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/8","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/8"}}},"commit":{"parents":[{"commit":"da6caaef0701e46f24869a33c5875b36e0e0f9af","subject":"Add thread name to the RabbitMQ heartbeat thread","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/da6caaef0701e46f24869a33c5875b36e0e0f9af"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-05 15:57:21.000000000","tz":120},"subject":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.","message":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/de5ac2850adc9b520be02c879ac38cf5d2877ab1"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/de5ac2850adc9b520be02c879ac38cf5d2877ab1"}]},"branch":"refs/heads/master"},"2251dc0d36e8d516ad20f861b7f6d904f5862c13":{"kind":"REWORK","_number":9,"created":"2019-06-05 16:51:43.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/9","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/9","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/9 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/9 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/9 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/9"}}},"commit":{"parents":[{"commit":"da6caaef0701e46f24869a33c5875b36e0e0f9af","subject":"Add thread name to the RabbitMQ heartbeat thread","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/da6caaef0701e46f24869a33c5875b36e0e0f9af"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-05 16:51:33.000000000","tz":120},"subject":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.","message":"[WIP] Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/2251dc0d36e8d516ad20f861b7f6d904f5862c13"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/2251dc0d36e8d516ad20f861b7f6d904f5862c13"}]},"branch":"refs/heads/master"},"b9afa43944bd1da18966f3f6ac195968892f0a9a":{"kind":"REWORK","_number":10,"created":"2019-06-18 16:19:49.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/10","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/10","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/10 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/10 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/10 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/10"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-18 16:13:46.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nTo fix this issue we have propose different approaches:\n- using a python thread (the changes of this patch set)\n- using eventlet tpool execute (patch set 9)\n\nThe only version which work like expected is this one (import the native python\nthread from eventlet)\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\nhttps://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/fakeservice/README.md\n\nWith this walkthrough you can observe the different the differents steps\nused by nova to use oslo.messaging.\n\nAlso, I would propose to you to compare the different solutions that I\u0027ve\ntested and propose to you to observe the behaviour.\n\nTo do this we will use some of the patches set related to this review.\n\nTo test the different solution we will play with git review and patches\nsets like:\n`git review -d 663074,\u003cpatch_set\u003e`\n\nBut in a first time we will setup a testing environment to reproduce the\nissue and observe the patches results.\n\n/## First setup the testing env\n\nSetup the testing environment:\n```\ncd /tmp\nTEST_DEBUG_ENV\u003d/tmp/663074\nmkdir ${TEST_DEBUG_ENV}\ncd ${TEST_DEBUG_ENV}\ngit clone git@github.com:openstack/oslo.messaging.git\ncd oslo.messaging\ngit review -s\ncd ${TEST_DEBUG_ENV}\ngit clone https://github.com/4383/pyamqp-heartbeat/ -b test_debug_env\n```\n\nNow the testing environment is setup.\n\nYou need to ensure to have pipenv installed on your laptop:\n```\npython3 -m pip install --user pipenv\n```\n\n/## Solution 1 - Using a native python thread\n\nWe will first check this patch set (the solution that works)(the latest\npatch set).\n\nhttps://review.opendev.org/#/c/663074/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work without issues and the\nconnection stay opened. Please observe that during more than 10 minutes.\n\nNow you can kill the running process.\n\n/## Solution 2 - Using eventlet tpool execute\n\nWe will now check how the things works if we use the eventlet tpool\nexecute functionality:\nhttps://eventlet.net/doc/threading.html#eventlet.tpool.execute\n\nhttps://review.opendev.org/#/c/663074/9/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074,9 # to setup the patch set number 9\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh # if rabbit server is not running\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work doesn\u0027t stream output to the\nterminal.\n\nYou also can observe on the rabbit dashboard that after some minutes\n(3 minutes on my side), all the connections are closed.\n\n/## More informations\n\n- During all these tests we force to run the heartbeat by using the send\n  purpose:\n  https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L30\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b9afa43944bd1da18966f3f6ac195968892f0a9a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b9afa43944bd1da18966f3f6ac195968892f0a9a"}]},"branch":"refs/heads/master"},"d50d2fa6c28a6e9956f7e83f8b11c66f9e64d21a":{"kind":"REWORK","_number":11,"created":"2019-06-19 14:07:27.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/11","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/11","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/11 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/11 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/11 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/11"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-19 14:07:16.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nTo fix this issue we have propose different approaches:\n- using a python thread (the changes of this patch set)\n- using eventlet tpool execute (patch set 9)\n\nThe only version which work like expected is this one (import the native python\nthread from eventlet)\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\nhttps://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/fakeservice/README.md\n\nWith this walkthrough you can observe the different the differents steps\nused by nova to use oslo.messaging.\n\nAlso, I would propose to you to compare the different solutions that I\u0027ve\ntested and propose to you to observe the behaviour.\n\nTo do this we will use some of the patches set related to this review.\n\nTo test the different solution we will play with git review and patches\nsets like:\n`git review -d 663074,\u003cpatch_set\u003e`\n\nBut in a first time we will setup a testing environment to reproduce the\nissue and observe the patches results.\n\n/## First setup the testing env\n\nSetup the testing environment:\n```\ncd /tmp\nTEST_DEBUG_ENV\u003d/tmp/663074\nmkdir ${TEST_DEBUG_ENV}\ncd ${TEST_DEBUG_ENV}\ngit clone git@github.com:openstack/oslo.messaging.git\ncd oslo.messaging\ngit review -s\ncd ${TEST_DEBUG_ENV}\ngit clone https://github.com/4383/pyamqp-heartbeat/ -b test_debug_env\n```\n\nNow the testing environment is setup.\n\nYou need to ensure to have pipenv installed on your laptop:\n```\npython3 -m pip install --user pipenv\n```\n\n/## Solution 1 - Using a native python thread\n\nWe will first check this patch set (the solution that works)(the latest\npatch set).\n\nhttps://review.opendev.org/#/c/663074/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work without issues and the\nconnection stay opened. Please observe that during more than 10 minutes.\n\nNow you can kill the running process.\n\n/## Solution 2 - Using eventlet tpool execute\n\nWe will now check how the things works if we use the eventlet tpool\nexecute functionality:\nhttps://eventlet.net/doc/threading.html#eventlet.tpool.execute\n\nhttps://review.opendev.org/#/c/663074/9/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074,9 # to setup the patch set number 9\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh # if rabbit server is not running\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work doesn\u0027t stream output to the\nterminal.\n\nYou also can observe on the rabbit dashboard that after some minutes\n(3 minutes on my side), all the connections are closed.\n\n/## More informations\n\n- During all these tests we force to run the heartbeat by using the send\n  purpose:\n  https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L30\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/d50d2fa6c28a6e9956f7e83f8b11c66f9e64d21a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/d50d2fa6c28a6e9956f7e83f8b11c66f9e64d21a"}]},"branch":"refs/heads/master"},"1660223ec0095540e7cc146f8ff862f6c7f4e6ee":{"kind":"REWORK","_number":12,"created":"2019-06-19 17:54:20.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/12","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/12","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/12 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/12 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/12 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/12"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-19 17:54:10.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nTo fix this issue we have propose different approaches:\n- using a python thread (the changes of this patch set)\n- using eventlet tpool execute (patch set 9)\n\nThe only version which work like expected is this one (import the native python\nthread from eventlet)\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\nhttps://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/fakeservice/README.md\n\nWith this walkthrough you can observe the different the differents steps\nused by nova to use oslo.messaging.\n\nAlso, I would propose to you to compare the different solutions that I\u0027ve\ntested and propose to you to observe the behaviour.\n\nTo do this we will use some of the patches set related to this review.\n\nTo test the different solution we will play with git review and patches\nsets like:\n`git review -d 663074,\u003cpatch_set\u003e`\n\nBut in a first time we will setup a testing environment to reproduce the\nissue and observe the patches results.\n\n/## First setup the testing env\n\nSetup the testing environment:\n```\ncd /tmp\nTEST_DEBUG_ENV\u003d/tmp/663074\nmkdir ${TEST_DEBUG_ENV}\ncd ${TEST_DEBUG_ENV}\ngit clone git@github.com:openstack/oslo.messaging.git\ncd oslo.messaging\ngit review -s\ncd ${TEST_DEBUG_ENV}\ngit clone https://github.com/4383/pyamqp-heartbeat/ -b test_debug_env\n```\n\nNow the testing environment is setup.\n\nYou need to ensure to have pipenv installed on your laptop:\n```\npython3 -m pip install --user pipenv\n```\n\n/## Solution 1 - Using a native python thread\n\nWe will first check this patch set (the solution that works)(the latest\npatch set).\n\nhttps://review.opendev.org/#/c/663074/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work without issues and the\nconnection stay opened. Please observe that during more than 10 minutes.\n\nNow you can kill the running process.\n\n/## Solution 2 - Using eventlet tpool execute\n\nWe will now check how the things works if we use the eventlet tpool\nexecute functionality:\nhttps://eventlet.net/doc/threading.html#eventlet.tpool.execute\n\nhttps://review.opendev.org/#/c/663074/9/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074,9 # to setup the patch set number 9\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh # if rabbit server is not running\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work doesn\u0027t stream output to the\nterminal.\n\nYou also can observe on the rabbit dashboard that after some minutes\n(3 minutes on my side), all the connections are closed.\n\n/## More informations\n\n- During all these tests we force to run the heartbeat by using the send\n  purpose:\n  https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L30\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/1660223ec0095540e7cc146f8ff862f6c7f4e6ee"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/1660223ec0095540e7cc146f8ff862f6c7f4e6ee"}]},"branch":"refs/heads/master"},"82f5a53b40214e120814a5daf2a2af81f2968abc":{"kind":"REWORK","_number":13,"created":"2019-06-19 23:47:19.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/13","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/13","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/13 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/13 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/13 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/13"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-19 23:47:05.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues who run\nheartbeat under mod_wsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat.\n\nTo fix this issue we have propose different approaches:\n- using a python thread (the changes of this patch set)\n- using eventlet tpool execute (patch set 9)\n\nThe only version which work like expected is this one (import the native python\nthread from eventlet)\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\nhttps://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/fakeservice/README.md\n\nWith this walkthrough you can observe the different the differents steps\nused by nova to use oslo.messaging.\n\nAlso, I would propose to you to compare the different solutions that I\u0027ve\ntested and propose to you to observe the behaviour.\n\nTo do this we will use some of the patches set related to this review.\n\nTo test the different solution we will play with git review and patches\nsets like:\n`git review -d 663074,\u003cpatch_set\u003e`\n\nBut in a first time we will setup a testing environment to reproduce the\nissue and observe the patches results.\n\n/## First setup the testing env\n\nSetup the testing environment:\n```\ncd /tmp\nTEST_DEBUG_ENV\u003d/tmp/663074\nmkdir ${TEST_DEBUG_ENV}\ncd ${TEST_DEBUG_ENV}\ngit clone git@github.com:openstack/oslo.messaging.git\ncd oslo.messaging\ngit review -s\ncd ${TEST_DEBUG_ENV}\ngit clone https://github.com/4383/pyamqp-heartbeat/ -b test_debug_env\n```\n\nNow the testing environment is setup.\n\nYou need to ensure to have pipenv installed on your laptop:\n```\npython3 -m pip install --user pipenv\n```\n\n/## Solution 1 - Using a native python thread\n\nWe will first check this patch set (the solution that works)(the latest\npatch set).\n\nhttps://review.opendev.org/#/c/663074/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work without issues and the\nconnection stay opened. Please observe that during more than 10 minutes.\n\nNow you can kill the running process.\n\n/## Solution 2 - Using eventlet tpool execute\n\nWe will now check how the things works if we use the eventlet tpool\nexecute functionality:\nhttps://eventlet.net/doc/threading.html#eventlet.tpool.execute\n\nhttps://review.opendev.org/#/c/663074/9/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074,9 # to setup the patch set number 9\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh # if rabbit server is not running\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in                                            /!\\\n/!\\ a tmux, resize a pane or create a new pane kill the container... so for /!\\\n/!\\ the next steps you need to create a second terminal to avoid tmux pane  /!\\\n/!\\ creation and similar things.                                            /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulate service monkey patch the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow In your open the rabbit management dashboard and observe the\nconnections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work doesn\u0027t stream output to the\nterminal.\n\nYou also can observe on the rabbit dashboard that after some minutes\n(3 minutes on my side), all the connections are closed.\n\n/## More informations\n\n- During all these tests we force to run the heartbeat by using the send\n  purpose:\n  https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L30\n\nSee the specifications for more informations:\nhttps://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/82f5a53b40214e120814a5daf2a2af81f2968abc"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/82f5a53b40214e120814a5daf2a2af81f2968abc"}]},"branch":"refs/heads/master"},"ed7625be665319fc5b608dbb79163fd96d06c39e":{"kind":"REWORK","_number":14,"created":"2019-07-05 14:46:50.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/14","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/14","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/14 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/14 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/14 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/14"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-07-05 13:31:39.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [6] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\neventlet is based on epoll or libevent [7][9], in an environment based on\napache MPM `prefork` module who doesn\u0027t support epoll and recent kernel\nfeatures.\n\nThe MPM `prefork` apache module is appropriate for sites that\nneed to avoid threading for compatibility with non-thread-safe\nlibraries [1].\n\nPossible proposed solutions\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo fix this issue we have proposed different approaches during discusses [8]:\n- using a python thread (the changes of this patch set)\n- using eventlet tpool execute (patch set 9)\n\nThe only version which work like expected is this patch set who\ncorrespond to use a native python thread and not eventlet.\n\nI think the tpool solution doesn\u0027t work correctly due to the apache\nengine in use currently (`prefork`), and using tpool keep us to using\neventlet and so libevent or epoll too.\n\nI would propose to you to compare the different solutions that I\u0027ve\ntested and I want allow to you to observe the behaviours.\n\nBut before starting the comparison I\u0027ll describe a little bit the chosen\nsolution.\n\nChosen solution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nSo we need to ensure to run heartbeat through a classical python thread\nnot build over epoll and libevent to avoid issue with environment who\ndoesn\u0027t support these kernel features.\n\nAlso we try to switch [4][5] from the apache `prefork` module to the `event`\nmodule which support non blocking sockets, use modern kernel features\nlike epoll through APR [3].\n\nPOC\n\u003d\u003d\u003d\n\nTo do this comparison we will use:\n- some of the patches set of this review\n- the POC section bellow\n- and the POC project \u003d\u003e https://github.com/4383/pyamqp-heartbeat/\n\nTo test the different solutions we will play with git review and patches\nsets by using commands like `git review -d 663074,\u003cpatch_set\u003e`\n\nBut in a first time we will setup a testing environment to reproduce the\nissue and observe the patches results.\n\nSetup the testing environment\n-----------------------------\n\nThe testing env is based on https://github.com/4383/pyamqp-heartbeat/\n\nIt was designed to help to reproduce the issue easily and to reproduce\nthe original environment as much as possible.\n\nSo, now setup your env by using:\n\n```\ncd /tmp\nTEST_DEBUG_ENV\u003d/tmp/663074\nmkdir ${TEST_DEBUG_ENV}\ncd ${TEST_DEBUG_ENV}\ngit clone git@github.com:openstack/oslo.messaging.git\ncd oslo.messaging\ngit review -s\ncd ${TEST_DEBUG_ENV}\ngit clone https://github.com/4383/pyamqp-heartbeat/\n```\n\nNow the testing environment is setup.\n\nYou need to ensure to have pipenv installed on your laptop:\n```\npython3 -m pip install --user pipenv\n```\n\nReproduce the original issue with non patched code\n--------------------------------------------------\n\nYou can test the original issue with a non patched code by simply switch\nto the `master` branch of your oslo.messaging clone that you have\nrealize during the previous setup step:\n\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit checkout master\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\nSolution 1 - Using a native python thread\n-----------------------------------------\n\nWe will first check this patch set (the solution that works)(the latest\npatch set https://review.opendev.org/#/c/663074/).\n\nHow can test these changes by using the following commands:\n\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in  a tmux with have an issue with the        /!\\\n/!\\ resize of a tmux pane or when we create a new pane, the both kill       /!\\\n/!\\ the container... so for the next steps you need to create a second      /!\\\n/!\\ terminal or tmux window to avoid tmux pane creation and similar things. /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulated service have monkey patched the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow open the rabbit management dashboard and observe the connections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal like described in\nthe previous warning, where you need to send a request to\nthe service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work without issues and the\nconnection stay opened indefinitely.\nPlease observe that during more than 10 minutes.\n\nNow you can kill the running process.\n\nSolution 2 - Using eventlet tpool execute\n-----------------------------------------\n\nWe will now check how the things works if we use the eventlet tpool\nexecute functionality:\nhttps://eventlet.net/doc/threading.html#eventlet.tpool.execute\n\nThis test correspond to the patch set https://review.opendev.org/#/c/663074/9/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074,9 # to setup the patch set number 9\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh # if rabbit server is not running\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in  a tmux with have an issue with the        /!\\\n/!\\ resize of a tmux pane or when we create a new pane, the both kill       /!\\\n/!\\ the container... so for the next steps you need to create a second      /!\\\n/!\\ terminal or tmux window to avoid tmux pane creation and similar things. /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulated service have monkey patched the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow open the rabbit management dashboard and observe the connections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe on the rabbit dashboard that after some minutes\n(3 minutes on my side), all the connections are closed.\n\nConclusion\n----------\n\nWith standard python thread connections stay opened indefinitely\nWith tpool connections still to disappears after few minutes like the\noriginal issue.\n\nAdditional informations\n-----------------------\n\nDuring all these tests:\n- we force to run the heartbeat by using the send purpose\n  https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L30\n- we run oslo.messaging under a monkey patched environment\n\nFurther reading\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\n- https://github.com/4383/pyamqp-heartbeat/tree/master/fakeservice\n\nWith this walkthrough you can observe the differents steps\nused by nova to use oslo.messaging and the different components in use\nto obtain the big picture of the stack.\n\nThe previous POC is based on this walkthrough as much as possible to\nreproduce the original issue.\n\nAlso you can retrieve online discussion on the mailing list about this\ntopic:\n- http://lists.openstack.org/pipermail/openstack-discuss/2019-April/005310.html\n\nLibevent support epoll too [9] but I don\u0027t know in which case eventlet use it.\n\nReferences\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://httpd.apache.org/docs/2.4/fr/mod/event.html\n[3] https://httpd.apache.org/docs/2.4/fr/glossary.html#apr\n[4] https://review.opendev.org/#/c/668862/\n[5] https://review.opendev.org/#/c/669178/\n[6] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n[7] https://github.com/eventlet/eventlet/blame/master/README.rst#L3\n[8] https://review.opendev.org/#/c/661314/\n[9] https://libevent.org/\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\nPartial-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/ed7625be665319fc5b608dbb79163fd96d06c39e"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/ed7625be665319fc5b608dbb79163fd96d06c39e"}]},"branch":"refs/heads/master"},"18f610345114d1af5e2a93f245fe5c7124dfa27f":{"kind":"NO_CODE_CHANGE","_number":15,"created":"2019-07-06 10:38:56.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/15","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/15","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/15 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/15 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/15 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/15"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-07-06 10:35:25.000000000","tz":120},"subject":"Always run the rabbitmq heartbeat inside a standard pthread.","message":"Always run the rabbitmq heartbeat inside a standard pthread.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to always run the rabbitmq health check heartbeat in a\nstandard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [6] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\neventlet is based on epoll or libevent [7][9], in an environment based on\napache MPM `prefork` module who doesn\u0027t support epoll and recent kernel\nfeatures.\n\nThe MPM `prefork` apache module is appropriate for sites that\nneed to avoid threading for compatibility with non-thread-safe\nlibraries [1].\n\nPossible proposed solutions\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo fix this issue we have proposed different approaches during discussions [8]:\n- using a python thread (this change)\n- using eventlet tpool execute (https://review.opendev.org/#/c/663074/9/)\n\nThe only version which work like expected is this patch set who\ncorrespond to use a native python thread and not eventlet.\n\nI think the tpool solution doesn\u0027t work correctly due to the apache\nengine in use currently (`prefork`), and using tpool keep us to using\neventlet and so libevent or epoll too.\n\nI would propose to you to compare the different solutions that I\u0027ve\ntested and I want allow to you to observe the behaviours.\n\nBut before starting the comparison I\u0027ll describe a little bit the chosen\nsolution.\n\nChosen solution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to isolate the heartbeat execution model from the parent process\ninherited execution model.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nWe want to force to use the python stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nSo we need to ensure to run heartbeat through a classical python thread\nnot build over epoll and libevent to avoid issue with environment who\ndoesn\u0027t support these kernel features.\n\nAlso we try to switch [4][5] from the apache `prefork` module to the `event`\nmodule which support non blocking sockets, use modern kernel features\nlike epoll through APR [3].\n\nPOC\n\u003d\u003d\u003d\n\nTo do this comparison we will use:\n- some of the patches set of this review\n- the POC section bellow\n- and the POC project \u003d\u003e https://github.com/4383/pyamqp-heartbeat/\n\nTo test the different solutions we will play with git review and patches\nsets by using commands like `git review -d 663074,\u003cpatch_set\u003e`\n\nBut in a first time we will setup a testing environment to reproduce the\nissue and observe the patches results.\n\nSetup the testing environment\n-----------------------------\n\nThe testing env is based on https://github.com/4383/pyamqp-heartbeat/\n\nIt was designed to help to reproduce the issue easily and to reproduce\nthe original environment as much as possible.\n\nSo, now setup your env by using:\n\n```\ncd /tmp\nTEST_DEBUG_ENV\u003d/tmp/663074\nmkdir ${TEST_DEBUG_ENV}\ncd ${TEST_DEBUG_ENV}\ngit clone git@github.com:openstack/oslo.messaging.git\ncd oslo.messaging\ngit review -s\ncd ${TEST_DEBUG_ENV}\ngit clone https://github.com/4383/pyamqp-heartbeat/\n```\n\nNow the testing environment is setup.\n\nYou need to ensure to have pipenv installed on your laptop:\n```\npython3 -m pip install --user pipenv\n```\n\nReproduce the original issue with non patched code\n--------------------------------------------------\n\nYou can test the original issue with a non patched code by simply switch\nto the `master` branch of your oslo.messaging clone that you have\nrealize during the previous setup step:\n\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit checkout master\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\nSolution 1 - Using a native python thread\n-----------------------------------------\n\nWe will first check this patch set (the solution that works)(the latest\npatch set https://review.opendev.org/#/c/663074/).\n\nHow can test these changes by using the following commands:\n\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in  a tmux with have an issue with the        /!\\\n/!\\ resize of a tmux pane or when we create a new pane, the both kill       /!\\\n/!\\ the container... so for the next steps you need to create a second      /!\\\n/!\\ terminal or tmux window to avoid tmux pane creation and similar things. /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulated service have monkey patched the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow open the rabbit management dashboard and observe the connections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal like described in\nthe previous warning, where you need to send a request to\nthe service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe that the heartbeat work without issues and the\nconnection stay opened indefinitely.\nPlease observe that during more than 10 minutes.\n\nNow you can kill the running process.\n\nSolution 2 - Using eventlet tpool execute\n-----------------------------------------\n\nWe will now check how the things works if we use the eventlet tpool\nexecute functionality:\nhttps://eventlet.net/doc/threading.html#eventlet.tpool.execute\n\nThis test correspond to the patch set https://review.opendev.org/#/c/663074/9/\n\nHow can test these changes by using the following commands:\n```\ncd ${TEST_DEBUG_ENV}/oslo.messaging\ngit review -d 663074,9 # to setup the patch set number 9\ncd ${TEST_DEBUG_ENV}/pyamqp-heartbeat\npipenv run ./setup-containers.sh # if rabbit server is not running\npipenv run ./start-oslo-mod_wsgi.sh ${TEST_DEBUG_ENV}/oslo.messaging\n```\n\n/!\\-------------------------------------------------------------------------/!\\\n/!\\ For some obscure reasons when we run this container                     /!\\\n/!\\ (who emulate the service) in  a tmux with have an issue with the        /!\\\n/!\\ resize of a tmux pane or when we create a new pane, the both kill       /!\\\n/!\\ the container... so for the next steps you need to create a second      /!\\\n/!\\ terminal or tmux window to avoid tmux pane creation and similar things. /!\\\n/!\\-------------------------------------------------------------------------/!\\\n\nYour emulated service is now running.\n\nThe emulated service have monkey patched the stdlib at start like nova do the\nthing.\n\nFor further reading about the emulated service and monkey patch:\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/Dockerfile#L13\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server-eventlet.wsgi#L10\n- https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L45\n\nNow open the rabbit management dashboard and observe the connections:\nhttp://127.0.0.1:15672/#/connections\nuser: guest\npwd: guest\n\nIn a second time you need to open a second terminal where you need to send\na request to this service by using:\n```\ncurl 0.0.0.0:8000\n```\n\nThe service will return \"it\u0027s work\" and you now can observe the\nestablished connections (related to the heartbeat) in the rabbit\ndashboard previously opened.\n\nYou can observe on the rabbit dashboard that after some minutes\n(3 minutes on my side), all the connections are closed.\n\nConclusion\n----------\n\nWith standard python thread connections stay opened indefinitely\nWith tpool connections still to disappears after few minutes like the\noriginal issue.\n\nAdditional informations\n-----------------------\n\nDuring all these tests:\n- we force to run the heartbeat by using the send purpose\n  https://github.com/4383/pyamqp-heartbeat/blob/test_debug_env/server.py#L30\n- we run oslo.messaging under a monkey patched environment\n\nFurther reading\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\n- https://github.com/4383/pyamqp-heartbeat/tree/master/fakeservice\n\nWith this walkthrough you can observe the differents steps\nused by nova to use oslo.messaging and the different components in use\nto obtain the big picture of the stack.\n\nThe previous POC is based on this walkthrough as much as possible to\nreproduce the original issue.\n\nAlso you can retrieve online discussion on the mailing list about this\ntopic:\n- http://lists.openstack.org/pipermail/openstack-discuss/2019-April/005310.html\n\nLibevent support epoll too [9] but I don\u0027t know in which case eventlet use it.\n\nReferences\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://httpd.apache.org/docs/2.4/fr/mod/event.html\n[3] https://httpd.apache.org/docs/2.4/fr/glossary.html#apr\n[4] https://review.opendev.org/#/c/668862/\n[5] https://review.opendev.org/#/c/669178/\n[6] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n[7] https://github.com/eventlet/eventlet/blame/master/README.rst#L3\n[8] https://review.opendev.org/#/c/661314/\n[9] https://libevent.org/\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\nCloses-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/18f610345114d1af5e2a93f245fe5c7124dfa27f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/18f610345114d1af5e2a93f245fe5c7124dfa27f"}]},"branch":"refs/heads/master"},"b7a08986ff9d11d2b61e558c55feaec50f71b663":{"kind":"REWORK","_number":16,"created":"2019-07-10 12:06:04.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/16","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/16","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/16 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/16 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/16 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/16"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-07-10 11:54:27.000000000","tz":120},"subject":"Allow users run the rabbitmq heartbeat inside a standard pthread.","message":"Allow users run the rabbitmq heartbeat inside a standard pthread.\n\nThis is an experimental feature.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to allow user to choose to run the rabbitmq health check\nheartbeat in a standard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [6] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\neventlet is based on epoll or libevent [7][9], in an environment based on\napache MPM `prefork` module who doesn\u0027t support epoll and recent kernel\nfeatures.\n\nThe MPM `prefork` apache module is appropriate for sites that\nneed to avoid threading for compatibility with non-thread-safe\nlibraries [1].\n\nPossible proposed solutions\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo fix this issue we have proposed different approaches during discussions [8]:\n- using a python thread (this change)\n- using eventlet tpool execute (https://review.opendev.org/#/c/663074/9/)\n\nThe only version which work like expected is this patch set who\ncorrespond to use a native python thread and not eventlet.\n\nI think the tpool solution doesn\u0027t work correctly due to the apache\nengine in use currently (`prefork`), and using tpool keep us to using\neventlet and so libevent or epoll too.\n\nI would propose to you to compare the different solutions that I\u0027ve\ntested and I want allow to you to observe the behaviours.\n\nBut before starting the comparison I\u0027ll describe a little bit the chosen\nsolution.\n\nChosen solution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to allow user to isolate the heartbeat execution model\nfrom the parent process inherited execution model by passing the\n`heartbeat_in_pthread` option through the driver config.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nIf the `heartbeat_in_pthread` option is given we want to force to use the\npython stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nThis is an experimental feature, we can help us to ensure to run heartbeat\nthrough a classical python thread\nnot build over epoll and libevent to avoid issue with environment who\ndoesn\u0027t support these kernel features.\n\nAlso we try to switch [4][5] from the apache `prefork` module to the `event`\nmodule which support non blocking sockets, use modern kernel features\nlike epoll through APR [3].\n\nPOC\n\u003d\u003d\u003d\n\nA POC is available at https://github.com/4383/pyamqp-heartbeat/ you can\ntest this feature and compare execution changes with the current code base.\n\nFurther reading\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\n- https://github.com/4383/pyamqp-heartbeat/tree/master/fakeservice\n\nWith this walkthrough you can observe the differents steps\nused by nova to use oslo.messaging and the different components in use\nto obtain the big picture of the stack.\n\nThe previous POC is based on this walkthrough as much as possible to\nreproduce the original issue.\n\nAlso you can retrieve online discussion on the mailing list about this\ntopic:\n- http://lists.openstack.org/pipermail/openstack-discuss/2019-April/005310.html\n\nLibevent support epoll too [9] but I don\u0027t know in which case eventlet use it.\n\nReferences\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://httpd.apache.org/docs/2.4/fr/mod/event.html\n[3] https://httpd.apache.org/docs/2.4/fr/glossary.html#apr\n[4] https://review.opendev.org/#/c/668862/\n[5] https://review.opendev.org/#/c/669178/\n[6] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n[7] https://github.com/eventlet/eventlet/blame/master/README.rst#L3\n[8] https://review.opendev.org/#/c/661314/\n[9] https://libevent.org/\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\nCloses-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b7a08986ff9d11d2b61e558c55feaec50f71b663"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/b7a08986ff9d11d2b61e558c55feaec50f71b663"}]},"branch":"refs/heads/master"},"95f5327d68b5aff201bbe45da966dd01f920716f":{"kind":"NO_CODE_CHANGE","_number":17,"created":"2019-07-10 12:08:38.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/17","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/17","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/17 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/17 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/17 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/17"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-07-10 12:07:49.000000000","tz":120},"subject":"Allow users run the rabbitmq heartbeat inside a standard pthread.","message":"Allow users run the rabbitmq heartbeat inside a standard pthread.\n\nThis is an experimental feature.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to allow user to choose to run the rabbitmq health check\nheartbeat in a standard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [6] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\neventlet is based on epoll or libevent [7][9], in an environment based on\napache MPM `prefork` module who doesn\u0027t support epoll and recent kernel\nfeatures.\n\nThe MPM `prefork` apache module is appropriate for sites that\nneed to avoid threading for compatibility with non-thread-safe\nlibraries [1].\n\nPossible proposed solutions\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo fix this issue we have proposed different approaches during discussions [8]:\n- using a python thread (this change)\n- using eventlet tpool execute (https://review.opendev.org/#/c/663074/9/)\n\nThe only version which work like expected is this patch set who\ncorrespond to use a native python thread and not eventlet.\n\nI think the tpool solution doesn\u0027t work correctly due to the apache\nengine in use currently (`prefork`), and using tpool keep us to using\neventlet and so libevent or epoll too.\n\nI would propose to you to compare the different solutions that I\u0027ve\ntested and I want allow to you to observe the behaviours.\n\nBut before starting the comparison I\u0027ll describe a little bit the chosen\nsolution.\n\nChosen solution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to allow user to isolate the heartbeat execution model\nfrom the parent process inherited execution model by passing the\n`heartbeat_in_pthread` option through the driver config.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nIf the `heartbeat_in_pthread` option is given we want to force to use the\npython stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nThis is an experimental feature, we can help us to ensure to run heartbeat\nthrough a classical python thread\nnot build over epoll and libevent to avoid issue with environment who\ndoesn\u0027t support these kernel features.\n\nAlso we try to switch [4][5] from the apache `prefork` module to the `event`\nmodule which support non blocking sockets, use modern kernel features\nlike epoll through APR [3].\n\nPOC\n\u003d\u003d\u003d\n\nA POC is available at:\nhttps://github.com/4383/pyamqp-heartbeat/blob/master/POC.md\n\nYou can test this feature and compare execution changes\nwith the current code base.\n\nFurther reading\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\n- https://github.com/4383/pyamqp-heartbeat/tree/master/fakeservice\n\nWith this walkthrough you can observe the differents steps\nused by nova to use oslo.messaging and the different components in use\nto obtain the big picture of the stack.\n\nThe previous POC is based on this walkthrough as much as possible to\nreproduce the original issue.\n\nAlso you can retrieve online discussion on the mailing list about this\ntopic:\n- http://lists.openstack.org/pipermail/openstack-discuss/2019-April/005310.html\n\nLibevent support epoll too [9] but I don\u0027t know in which case eventlet use it.\n\nReferences\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://httpd.apache.org/docs/2.4/fr/mod/event.html\n[3] https://httpd.apache.org/docs/2.4/fr/glossary.html#apr\n[4] https://review.opendev.org/#/c/668862/\n[5] https://review.opendev.org/#/c/669178/\n[6] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n[7] https://github.com/eventlet/eventlet/blame/master/README.rst#L3\n[8] https://review.opendev.org/#/c/661314/\n[9] https://libevent.org/\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\nCloses-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/95f5327d68b5aff201bbe45da966dd01f920716f"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/95f5327d68b5aff201bbe45da966dd01f920716f"}]},"branch":"refs/heads/master"},"318a9b102cf532841a1cd69346a36a01253c5220":{"kind":"NO_CODE_CHANGE","_number":18,"created":"2019-07-10 14:19:11.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/18","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/18","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/18 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/18 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/18 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/18"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-07-10 14:10:31.000000000","tz":120},"subject":"Allow users run the rabbitmq heartbeat inside a standard pthread.","message":"Allow users run the rabbitmq heartbeat inside a standard pthread.\n\nThis is an experimental feature.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to allow user to choose to run the rabbitmq health check\nheartbeat in a standard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [6] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\neventlet is based on epoll or libevent [7][9], in an environment based on\napache MPM `prefork` module who doesn\u0027t support epoll and recent kernel\nfeatures.\n\nThe MPM `prefork` apache module is appropriate for sites that\nneed to avoid threading for compatibility with non-thread-safe\nlibraries [1].\n\nPossible proposed solutions\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo fix this issue we have proposed different approaches during discussions [8]:\n- using a python thread (this change)\n- using eventlet tpool execute (https://review.opendev.org/#/c/663074/9/)\n\nThe only version which work like expected is this patch set who\ncorrespond to use a native python thread and not eventlet.\n\nI think the tpool solution doesn\u0027t work correctly due to the apache\nengine in use currently (`prefork`), and using tpool keep us to using\neventlet and so libevent or epoll too.\n\nI would propose to you to compare the different solutions that I\u0027ve\ntested and I want allow to you to observe the behaviours.\n\nBut before starting the comparison I\u0027ll describe a little bit the chosen\nsolution.\n\nChosen solution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to allow user to isolate the heartbeat execution model\nfrom the parent process inherited execution model by passing the\n`heartbeat_in_pthread` option through the driver config.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nIf the `heartbeat_in_pthread` option is given we want to force to use the\npython stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nThis is an experimental feature, we can help us to ensure to run heartbeat\nthrough a classical python thread\nnot build over epoll and libevent to avoid issue with environment who\ndoesn\u0027t support these kernel features.\n\nAlso we try to switch [4][5] from the apache `prefork` module to the `event`\nmodule which support non blocking sockets, use modern kernel features\nlike epoll through APR [3].\n\nPOC\n\u003d\u003d\u003d\n\nSome POC are available at:\nhttps://github.com/4383/pyamqp-heartbeat/blob/master/POC.md\n\nYou can test behaviors between execution models and the current code base.\n\nThese POCs are not compatible with the patches set higher than the patch\nset 15.\n\nThey don\u0027t reflect the latest changes and especially the\n`heartbeat_in_pthread` option and the possibility to turn on/off the\nfeature. In other words on these POCs we always force to use pthreads\nand these POCs allow you to compare the rabbitmq heartbeat connection\nwith pthread and green thread.\n\nFurther reading\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\n- https://github.com/4383/pyamqp-heartbeat/tree/master/fakeservice\n\nWith this walkthrough you can observe the differents steps\nused by nova to use oslo.messaging and the different components in use\nto obtain the big picture of the stack.\n\nThe previous POC is based on this walkthrough as much as possible to\nreproduce the original issue.\n\nAlso you can retrieve online discussion on the mailing list about this\ntopic:\n- http://lists.openstack.org/pipermail/openstack-discuss/2019-April/005310.html\n\nLibevent support epoll too [9] but I don\u0027t know in which case eventlet use it.\n\nReferences\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://httpd.apache.org/docs/2.4/fr/mod/event.html\n[3] https://httpd.apache.org/docs/2.4/fr/glossary.html#apr\n[4] https://review.opendev.org/#/c/668862/\n[5] https://review.opendev.org/#/c/669178/\n[6] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n[7] https://github.com/eventlet/eventlet/blame/master/README.rst#L3\n[8] https://review.opendev.org/#/c/661314/\n[9] https://libevent.org/\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\nCloses-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/318a9b102cf532841a1cd69346a36a01253c5220"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/318a9b102cf532841a1cd69346a36a01253c5220"}]},"branch":"refs/heads/master"},"197e4f47af5d0cea7e408ac4136bb18b2f8f1ac8":{"kind":"REWORK","_number":19,"created":"2019-07-11 06:43:00.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/19","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/19","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/19 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/19 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/19 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/19"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-07-11 06:42:42.000000000","tz":120},"subject":"Allow users run the rabbitmq heartbeat inside a standard pthread.","message":"Allow users run the rabbitmq heartbeat inside a standard pthread.\n\nThis is an experimental feature.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to allow user to choose to run the rabbitmq health check\nheartbeat in a standard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [6] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\neventlet is based on epoll or libevent [7][9], in an environment based on\napache MPM `prefork` module who doesn\u0027t support epoll and recent kernel\nfeatures.\n\nThe MPM `prefork` apache module is appropriate for sites that\nneed to avoid threading for compatibility with non-thread-safe\nlibraries [1].\n\nPossible proposed solutions\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo fix this issue we have proposed different approaches during discussions [8]:\n- using a python thread (this change)\n- using eventlet tpool execute (https://review.opendev.org/#/c/663074/9/)\n\nThe only version which work like expected is this patch set who\ncorrespond to use a native python thread and not eventlet.\n\nI think the tpool solution doesn\u0027t work correctly due to the apache\nengine in use currently (`prefork`), and using tpool keep us to using\neventlet and so libevent or epoll too.\n\nI would propose to you to compare the different solutions that I\u0027ve\ntested and I want allow to you to observe the behaviours.\n\nBut before starting the comparison I\u0027ll describe a little bit the chosen\nsolution.\n\nChosen solution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to allow user to isolate the heartbeat execution model\nfrom the parent process inherited execution model by passing the\n`heartbeat_in_pthread` option through the driver config.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nIf the `heartbeat_in_pthread` option is given we want to force to use the\npython stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nThis is an experimental feature, we can help us to ensure to run heartbeat\nthrough a classical python thread\nnot build over epoll and libevent to avoid issue with environment who\ndoesn\u0027t support these kernel features.\n\nAlso we try to switch [4][5] from the apache `prefork` module to the `event`\nmodule which support non blocking sockets, use modern kernel features\nlike epoll through APR [3].\n\nPOC\n\u003d\u003d\u003d\n\nSome POC are available at:\nhttps://github.com/4383/pyamqp-heartbeat/blob/master/POC.md\n\nYou can test behaviors between execution models and the current code base.\n\nThese POCs are not compatible with the patches set higher than the patch\nset 15.\n\nThey don\u0027t reflect the latest changes and especially the\n`heartbeat_in_pthread` option and the possibility to turn on/off the\nfeature. In other words on these POCs we always force to use pthreads\nand these POCs allow you to compare the rabbitmq heartbeat connection\nwith pthread and green thread.\n\nFurther reading\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\n- https://github.com/4383/pyamqp-heartbeat/tree/master/fakeservice\n\nWith this walkthrough you can observe the differents steps\nused by nova to use oslo.messaging and the different components in use\nto obtain the big picture of the stack.\n\nThe previous POC is based on this walkthrough as much as possible to\nreproduce the original issue.\n\nAlso you can retrieve online discussion on the mailing list about this\ntopic:\n- http://lists.openstack.org/pipermail/openstack-discuss/2019-April/005310.html\n\nLibevent support epoll too [9] but I don\u0027t know in which case eventlet use it.\n\nReferences\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://httpd.apache.org/docs/2.4/fr/mod/event.html\n[3] https://httpd.apache.org/docs/2.4/fr/glossary.html#apr\n[4] https://review.opendev.org/#/c/668862/\n[5] https://review.opendev.org/#/c/669178/\n[6] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n[7] https://github.com/eventlet/eventlet/blame/master/README.rst#L3\n[8] https://review.opendev.org/#/c/661314/\n[9] https://libevent.org/\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\nCloses-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/197e4f47af5d0cea7e408ac4136bb18b2f8f1ac8"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/197e4f47af5d0cea7e408ac4136bb18b2f8f1ac8"}]},"branch":"refs/heads/master"},"e589f93ab360ad6dd0ba313c3805b407c1eebd3a":{"kind":"REWORK","_number":20,"created":"2019-07-29 09:56:52.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/20","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/20","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/20 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/20 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/20 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/20"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-07-29 09:56:38.000000000","tz":120},"subject":"Allow users run the rabbitmq heartbeat inside a standard pthread.","message":"Allow users run the rabbitmq heartbeat inside a standard pthread.\n\nThis is an experimental feature.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to allow user to choose to run the rabbitmq health check\nheartbeat in a standard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [6] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\neventlet is based on epoll or libevent [7][9], in an environment based on\napache MPM `prefork` module who doesn\u0027t support epoll and recent kernel\nfeatures.\n\nThe MPM `prefork` apache module is appropriate for sites that\nneed to avoid threading for compatibility with non-thread-safe\nlibraries [1].\n\nPossible proposed solutions\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo fix this issue we have proposed different approaches during discussions [8]:\n- using a python thread (this change)\n- using eventlet tpool execute (https://review.opendev.org/#/c/663074/9/)\n\nThe only version which work like expected is this patch set who\ncorrespond to use a native python thread and not eventlet.\n\nI think the tpool solution doesn\u0027t work correctly due to the apache\nengine in use currently (`prefork`), and using tpool keep us to using\neventlet and so libevent or epoll too.\n\nI would propose to you to compare the different solutions that I\u0027ve\ntested and I want allow to you to observe the behaviours.\n\nBut before starting the comparison I\u0027ll describe a little bit the chosen\nsolution.\n\nChosen solution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to allow user to isolate the heartbeat execution model\nfrom the parent process inherited execution model by passing the\n`heartbeat_in_pthread` option through the driver config.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nIf the `heartbeat_in_pthread` option is given we want to force to use the\npython stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nThis is an experimental feature, we can help us to ensure to run heartbeat\nthrough a classical python thread\nnot build over epoll and libevent to avoid issue with environment who\ndoesn\u0027t support these kernel features.\n\nAlso we try to switch [4][5] from the apache `prefork` module to the `event`\nmodule which support non blocking sockets, use modern kernel features\nlike epoll through APR [3].\n\nPOC\n\u003d\u003d\u003d\n\nSome POC are available at:\nhttps://github.com/4383/pyamqp-heartbeat/blob/master/POC.md\n\nYou can test behaviors between execution models and the current code base.\n\nThese POCs are not compatible with the patches set higher than the patch\nset 15.\n\nThey don\u0027t reflect the latest changes and especially the\n`heartbeat_in_pthread` option and the possibility to turn on/off the\nfeature. In other words on these POCs we always force to use pthreads\nand these POCs allow you to compare the rabbitmq heartbeat connection\nwith pthread and green thread.\n\nFurther reading\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nTo understand how services consume the oslo.messaging drivers,\nand especially how to nova consome the oslo.messaging rabbitmq driver,\nplease take a look to:\n- https://github.com/4383/pyamqp-heartbeat/tree/master/fakeservice\n\nWith this walkthrough you can observe the differents steps\nused by nova to use oslo.messaging and the different components in use\nto obtain the big picture of the stack.\n\nThe previous POC is based on this walkthrough as much as possible to\nreproduce the original issue.\n\nAlso you can retrieve online discussion on the mailing list about this\ntopic:\n- http://lists.openstack.org/pipermail/openstack-discuss/2019-April/005310.html\n\nLibevent support epoll too [9] but I don\u0027t know in which case eventlet use it.\n\nReferences\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://httpd.apache.org/docs/2.4/fr/mod/event.html\n[3] https://httpd.apache.org/docs/2.4/fr/glossary.html#apr\n[4] https://review.opendev.org/#/c/668862/\n[5] https://review.opendev.org/#/c/669178/\n[6] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n[7] https://github.com/eventlet/eventlet/blame/master/README.rst#L3\n[8] https://review.opendev.org/#/c/661314/\n[9] https://libevent.org/\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\nCloses-Bug: #1826281\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e589f93ab360ad6dd0ba313c3805b407c1eebd3a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/e589f93ab360ad6dd0ba313c3805b407c1eebd3a"}]},"branch":"refs/heads/master"},"22f240b82fffbd62be8568a7d0d3369134596ace":{"kind":"REWORK","_number":21,"created":"2019-08-07 15:24:01.000000000","uploader":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"ref":"refs/changes/74/663074/21","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.messaging","ref":"refs/changes/74/663074/21","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/21 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/21 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/21 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.messaging refs/changes/74/663074/21"}}},"commit":{"parents":[{"commit":"fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d","subject":"Merge \"Fix switch connection destination when a rabbitmq cluster node disappear\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/fe0ac3195ef1dcb1b481f0f5cf6ff2b55ca6056d"}]}],"author":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-06-04 15:45:59.000000000","tz":120},"committer":{"name":"Hervé Beraud","email":"hberaud@redhat.com","date":"2019-08-07 15:17:02.000000000","tz":120},"subject":"Allow users run the rabbitmq heartbeat inside a standard pthread.","message":"Allow users run the rabbitmq heartbeat inside a standard pthread.\n\nThis is an experimental feature.\n\nThe proposed changes will fix related issues when we run\nheartbeat under apache/httpd enviornment with the apache MPM `prefork`\n[1] engine and mod_wsgi or uwsgi in a monkey patched environment.\n\nPropose changes to allow user to choose to run the rabbitmq health check\nheartbeat in a standard python thread.\n\nIssue\n\u003d\u003d\u003d\u003d\u003d\n\nWe facing an issue with the rabbitmq driver heartbeat\nunder apache MPM `prefork` module and mod_wsgi when nova_api monkey\npatched the stdlib by using eventlet.\n\nnova_api calling eventlet.monkey_patch() [2] when it runs under mod_wsgi.\n\nThis impacts the AMQP heartbeat thread,\nwhich is meant to be a native thread. Instead of checking AMQP sockets\nevery 15s, It is now suspended and resumed by eventlet. However,\nresuming greenthreads can take a very long time if mod_wsgi isn\u0027t\nprocessing traffic regularly, which can cause rabbitmq to close the AMQP\nconnection.\n\nRoot Cause\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nThe oslo.messaging RabbitMQ driver and especially the heartbeat\nsuffer to inherit the execution model of the service which consume him.\n\nIn this scenario nova_api need green threads to manage cells and edge\nfeatures so nova_api monkey patch the stdlib to obtain async features,\nand the oslo.messaging rabbitmq driver endure these changes.\n\nI think the main issue here is that nova_api want async and use eventlet green\nthreads to obtain it.\n\nSolution\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWe want to allow user to isolate the heartbeat execution model\nfrom the parent process inherited execution model by passing the\n`heartbeat_in_pthread` option through the driver config.\n\nWhile we use MPM `prefork` we want to avoid to use libevent and epoll.\n\nIf the `heartbeat_in_pthread` option is given we want to force to use the\npython stdlib threading module to run the\nrabbitmq heartbeat to avoid issue related to a non \"standard\"\nenvironment. I mean \"standard\" because async features isn\u0027t the default\nconfig in mostly case, starting by apache which define `prefork` is the\ndefault engine.\n\nThis is an experimental feature, we can help us to ensure to run heartbeat\nthrough a classical python thread\n\nSpecifications\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n- https://review.opendev.org/661314\n\n[1] https://httpd.apache.org/docs/2.4/fr/mod/prefork.html\n[2] https://github.com/openstack/nova/commit/3c5e2b0e9fac985294a949852bb8c83d4ed77e04\n\nChange-Id: If8846599efc48fe18ecfb99c04e2c38f9a45b9ed\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/22f240b82fffbd62be8568a7d0d3369134596ace"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.messaging/commit/22f240b82fffbd62be8568a7d0d3369134596ace"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}