)]}'
{"id":"openstack%2Fkeystone~754488","triplet_id":"openstack%2Fkeystone~master~Ia45a45ff852d0d4e3a713dae07a46d4ff8d370f3","project":"openstack/keystone","branch":"master","hashtags":[],"change_id":"Ia45a45ff852d0d4e3a713dae07a46d4ff8d370f3","subject":"Implement more robust connection handling for asynchronous LDAP calls","status":"MERGED","created":"2020-09-26 01:50:21.000000000","updated":"2020-10-15 19:57:18.000000000","submitted":"2020-10-15 19:55:24.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":4,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"754488-1602791724865-34ab9269","meta_rev_id":"a2453aafcb38d649d62e3f17f6124d607b38906d","_number":754488,"virtual_id_number":754488,"owner":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},{"tag":"autogenerated:zuul:gate","value":2,"date":"2020-10-15 19:55:24.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"date":"2020-09-30 22:17:11.000000000","_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"value":0,"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"value":0,"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"value":0,"date":"2020-10-15 13:52:09.000000000","_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"},{"value":0,"_account_id":9954,"name":"Lance Bragstad","username":"lbragstad","inactive":true}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"all":[{"value":2,"date":"2020-10-14 21:14:02.000000000","_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"value":2,"date":"2020-10-13 14:01:13.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"value":0,"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"value":0,"_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"},{"value":0,"_account_id":9954,"name":"Lance Bragstad","username":"lbragstad","inactive":true}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"all":[{"value":1,"date":"2020-10-14 21:14:02.000000000","_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"value":0,"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"value":0,"date":"2020-10-13 13:05:30.000000000","_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"value":0,"_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"},{"value":0,"_account_id":9954,"name":"Lance Bragstad","username":"lbragstad","inactive":true}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"_account_id":9954,"name":"Lance Bragstad","username":"lbragstad","inactive":true},{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-09-26 01:50:21.000000000","updated_by":{"_account_id":9954,"name":"Lance Bragstad","username":"lbragstad","inactive":true},"reviewer":{"_account_id":9954,"name":"Lance Bragstad","username":"lbragstad","inactive":true},"state":"REVIEWER"},{"updated":"2020-10-13 13:05:30.000000000","updated_by":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"reviewer":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"state":"REVIEWER"},{"updated":"2020-10-13 14:01:13.000000000","updated_by":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"reviewer":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"state":"REVIEWER"},{"updated":"2020-10-14 21:14:02.000000000","updated_by":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"reviewer":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"state":"REVIEWER"},{"updated":"2020-10-15 13:52:09.000000000","updated_by":{"_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"},"reviewer":{"_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"},"state":"REVIEWER"},{"updated":"2020-10-15 19:55:24.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"fe3b16f2c86a884af8f97ab274a48e718a089c43","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2020-09-26 01:50:21.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"804b7b83d671334357fbf428e56e27d367b10145","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-26 02:56:08.000000000","message":"Patch Set 1: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/f819ce83597e4768909fa3be4058bac5 : FAILURE in 43m 20s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/d9782737ef90411db448478a6ef8abc8 : FAILURE in 10m 09s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/4c1902077d404a7cb6b9cb361c05779a : FAILURE in 5m 25s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/9ede1e98e7834d948cd4d7392a93af6a : FAILURE in 12m 57s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/e7133509dea947109f8b8f526ff71416 : FAILURE in 11m 56s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/0c4b9b38953140ee929385054b5ef231 : SUCCESS in 11m 36s\n- grenade https://zuul.opendev.org/t/openstack/build/595060ac0a6c4ebf97a2a98b69721cd8 : SUCCESS in 42m 21s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/7b50a82b21804ef4a1e29c1556fcb8e5 : FAILURE in 1h 04m 09s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/5cbaeab46b3c41729f5403c886c843c1 : SUCCESS in 23m 48s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal https://zuul.opendev.org/t/openstack/build/b800d47856a142e2a37371ec3e79f110 : SUCCESS in 22m 49s (non-voting)\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/5b291a4744924e4d98378eb37c00f137 : SUCCESS in 23m 40s\n- keystoneclient-devstack-functional https://zuul.opendev.org/t/openstack/build/aa2a982ef48e4c9082b959c77019eaed : SUCCESS in 12m 17s (non-voting)\n- keystone-dsvm-ldap-domain-specific-driver https://zuul.opendev.org/t/openstack/build/6a7b620ddea94cac92be921f85e5e73a : FAILURE in 11m 20s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/a6e2c10f55634c6288086d3d2183ed64 : SUCCESS in 58m 11s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/45f411636653476b82532a65afced575 : SUCCESS in 22m 40s","accounts_in_message":[],"_revision_number":1},{"id":"93dfaa4f014ef025afbc1e192fd71501d45e1c94","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2020-09-27 01:10:16.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"d616e85e04afdd628ee3969875a48858ede1bd97","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2020-09-28 21:38:58.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"bc92352a7c86ed9fd5a756d8700c437685c02de0","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2020-09-28 21:39:47.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"b5f8aed9996953450af55252eb0e664bf6a23c16","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-28 23:20:11.000000000","message":"Patch Set 2: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/90c2c80fa4e0405781f34ed4dcb3acac : SUCCESS in 17m 00s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/364d228af4634a3daae09335a631edfa : SUCCESS in 12m 03s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2d704ffff9e9486991a548d58a591158 : FAILURE in 4m 41s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/e120f492f51e4feb83dbe0c3a3a83528 : SUCCESS in 13m 25s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/03549e4dfa814789bed7b2d3d9e104e8 : SUCCESS in 12m 03s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/255622c025cd4999b77599e6fa0c47c0 : SUCCESS in 12m 57s\n- grenade https://zuul.opendev.org/t/openstack/build/dbd4bbd838c4486d8551dc332eb0b690 : SUCCESS in 57m 46s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/04a030f27f714146b2a31f3cba7cf6ea : SUCCESS in 1h 33m 23s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/1711becb09f24684af5d9c01dd0105c2 : SUCCESS in 33m 43s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal https://zuul.opendev.org/t/openstack/build/8674950fcc59471db717a519ae00ef40 : SUCCESS in 34m 28s (non-voting)\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/0fcd02cdf7d949c598d291e05c8f5a22 : SUCCESS in 36m 43s\n- keystoneclient-devstack-functional https://zuul.opendev.org/t/openstack/build/d75a5fedae8843ea8bff6b450c1d9a9a : SUCCESS in 17m 18s (non-voting)\n- keystone-dsvm-ldap-domain-specific-driver https://zuul.opendev.org/t/openstack/build/c07dcfebfebb4e93a6c389f252854f8f : FAILURE in 15m 27s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/1604629988bb4334a60522dc97ded8b6 : SUCCESS in 1h 00m 02s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/5d86e7f06f2a4d2abbe154a5cc549e54 : SUCCESS in 37m 19s","accounts_in_message":[],"_revision_number":2},{"id":"01d20817cd81b02c8a8900c578e30d6a8ca13fcb","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2020-09-29 15:55:45.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"f6e01a5519befde4c92ef8f90654bc00de225f6d","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-29 18:10:21.000000000","message":"Patch Set 3: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/814886789418470dbff34f58d6d808cc : SUCCESS in 20m 31s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/1a1d4f01475f4f48880c506c57c3c202 : SUCCESS in 14m 44s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/cb83dddde91b4c70b4aee4e1c7ce195b : SUCCESS in 5m 41s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/2eff84b10e524335b574ce11ee47e86d : SUCCESS in 21m 37s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/efa439f7164448a380191e1d1adc0f57 : SUCCESS in 13m 18s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/6e1bc8f7123f45f0ae3ef07323f049b8 : FAILURE in 6m 49s\n- grenade https://zuul.opendev.org/t/openstack/build/e7c82876c8614b428ef21390f15b17e0 : SUCCESS in 1h 00m 46s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/ab8b3b76420b43ccb7712914f63d7253 : SUCCESS in 1h 14m 06s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/e3489149d35b493186d583230f57124e : SUCCESS in 6m 14s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/9bc2deec04ca4d34913e3e1de2382d75 : SUCCESS in 30m 04s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal https://zuul.opendev.org/t/openstack/build/7aae15a30271481bbb1db7f2b3f6cf39 : SUCCESS in 32m 30s (non-voting)\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/7bfca5fb2e5f490d860d9538eabce6a5 : SUCCESS in 32m 45s\n- keystoneclient-devstack-functional https://zuul.opendev.org/t/openstack/build/6b28646f304b4ac0b2f0373aa877084c : SUCCESS in 14m 08s (non-voting)\n- keystone-dsvm-ldap-domain-specific-driver https://zuul.opendev.org/t/openstack/build/195edd1f0e974128ab2d2cd58f0857a0 : FAILURE in 11m 13s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/158810ef37474d2197d6c5b1a03d4879 : SUCCESS in 1h 00m 56s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/0a9411d252bc4281a9bf6fd5b46180f1 : SUCCESS in 37m 31s","accounts_in_message":[],"_revision_number":3},{"id":"96381344cdd6e9cefbfdb3dae335fb8089373ba4","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2020-09-30 13:28:03.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"41e31172de632c07e82c129127ba0729348314c1","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-09-30 19:08:11.000000000","message":"Patch Set 4: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/cc86502012994788860a41f5a7da8022 : SUCCESS in 17m 06s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/38309090c622435fb7a5d18a387dfb12 : SUCCESS in 13m 02s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/1aedaa2520c14b79a1b8269f6c4f31a0 : SUCCESS in 7m 05s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/0ff110872c944cf7a7e737e8d6b0a0b8 : SUCCESS in 24m 37s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/0fb89eff6c514fe7b4d240ae69c55b21 : SUCCESS in 14m 22s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/42a2ad9eaa7a4d1cae78af6039408a79 : SUCCESS in 13m 22s\n- grenade https://zuul.opendev.org/t/openstack/build/03bb8f10fe404d37a9bcb23cb082438d : SUCCESS in 1h 01m 41s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/2d920b680a524814ad2bf5cd04c176bc : SUCCESS in 1h 12m 21s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/e7643ddc90e54f75a4e588f8d9e27e0a : SUCCESS in 7m 18s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/20db7c7323764bffb382584a950af59d : FAILURE in 23m 09s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal https://zuul.opendev.org/t/openstack/build/184e59f59a804e06b263778a1253ec18 : SUCCESS in 32m 00s (non-voting)\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/6257a3500b294d0588a1341deec81b33 : SUCCESS in 44m 06s\n- keystoneclient-devstack-functional https://zuul.opendev.org/t/openstack/build/3e0a43f6368e4ac1b4d8212db6586a6f : SUCCESS in 15m 13s (non-voting)\n- keystone-dsvm-ldap-domain-specific-driver https://zuul.opendev.org/t/openstack/build/d7460de382584c0c81e195e1fdc8b5e1 : FAILURE in 17m 31s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/5dec055614564a6cbeb9ace5551650ff : SUCCESS in 1h 07m 28s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/0c623bf91e3b4b0e81e62c19dbc2dead : SUCCESS in 37m 32s","accounts_in_message":[],"_revision_number":4},{"id":"37fd316b0b5813578cf0ef94b7c4cc7f982e4cb0","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2020-09-30 22:17:11.000000000","message":"Patch Set 4:\n\nrecheck","accounts_in_message":[],"_revision_number":4},{"id":"79cde804643a89052f6932fb70f52392fc7b4027","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-01 00:36:38.000000000","message":"Patch Set 4: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/810db2bd5c944228ba41b01e89f09f14 : SUCCESS in 15m 51s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/e0444ef20ad34dba8b9f8455a8868fbb : SUCCESS in 13m 35s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/97621bcb5af844b281ce4afe4e17c30e : SUCCESS in 5m 52s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/370288da77864df1b14b5009878f5f84 : SUCCESS in 13m 00s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/fc1ac0a0263641abb6883079175e61f0 : SUCCESS in 13m 55s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/e5a1c0cba9d64707a19b0938aefa040e : SUCCESS in 12m 14s\n- grenade https://zuul.opendev.org/t/openstack/build/36706ee53cc242e78c87b9567d80b5b8 : SUCCESS in 1h 00m 28s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/5c05cb5b652a4d7f8f1e32973a05027f : SUCCESS in 1h 19m 44s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/469fc1fd229e45fe8a0a4e8029c49c21 : SUCCESS in 7m 26s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/14fbf781c47c428aa4c423b01b7ea584 : SUCCESS in 32m 27s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal https://zuul.opendev.org/t/openstack/build/d0e4eef5c5e64d4d9aeea296def60b05 : SUCCESS in 38m 03s (non-voting)\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/eedd0bae951442e9b07a537fb6743792 : SUCCESS in 24m 02s\n- keystoneclient-devstack-functional https://zuul.opendev.org/t/openstack/build/140bec7d1c9a46d98457085f857ffcae : SUCCESS in 19m 49s (non-voting)\n- keystone-dsvm-ldap-domain-specific-driver https://zuul.opendev.org/t/openstack/build/4888284af3d543c79ea04bf50807b8d3 : FAILURE in 16m 58s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/111112ee3ca84d24a3cfba6a1d9f2bb0 : SUCCESS in 1h 17m 22s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/24d040010fc84424b00dd4fad5f1b02a : SUCCESS in 22m 54s","accounts_in_message":[],"_revision_number":4},{"id":"57f19caea862e21a2e1914d3925669fd09b67d45","author":{"_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"},"date":"2020-10-02 09:21:43.000000000","message":"Patch Set 4: Cherry Picked\n\nThis patchset was cherry picked to branch stable/victoria as commit a26a40d441e7f211fd68e0c1b9c779d965c19dea","accounts_in_message":[],"_revision_number":4},{"id":"c33d0a76c95ca4253d352d8c0ceb88f32ba66f47","author":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"date":"2020-10-13 14:01:13.000000000","message":"Patch Set 4: Code-Review+2","accounts_in_message":[],"_revision_number":4},{"id":"e6c442ba18ddf0b41c7c3d267f93e38d89b72d1a","author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"date":"2020-10-14 21:14:02.000000000","message":"Patch Set 4: Code-Review+2 Workflow+1","accounts_in_message":[],"_revision_number":4},{"id":"1b0001d43ce58b7ae2fbf394eb96488c30f9ee8d","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-14 21:21:50.000000000","message":"Patch Set 4: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":4},{"id":"736155d747615ea7819067d9a739d074de31e2c5","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-15 01:07:20.000000000","message":"Patch Set 4: Verified-2\n\nBuild failed (gate pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/a5b12c0cba8743ff9c08421a23f5ffb3 : SUCCESS in 14m 57s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/67636510252746a09226ef0d42a81d2f : SUCCESS in 5m 55s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/d6425f09397d4cb5a339139bd86f6ce7 : SUCCESS in 14m 32s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/823cd7b3cce3482ebf4bbe0c40d9c9d2 : TIMED_OUT in 41m 29s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/9ace4f514d354f78b90589b7cb1be50b : SUCCESS in 12m 25s\n- grenade https://zuul.opendev.org/t/openstack/build/4060221dbe874bf7964c2bcd13f6c051 : SUCCESS in 1h 06m 42s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/8aab766aaddc4a5a84e612d5b0163f05 : TIMED_OUT in 2h 06m 06s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/9fe8f86f65a74ab7bc2f8a7680b67ab2 : SUCCESS in 8m 56s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/ddb3fbd132764cec8d562c8548e558e3 : POST_FAILURE in 1h 13m 21s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/8bf168c3abdb49fb917674c18896e255 : SUCCESS in 39m 18s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/702750b16d7e4b13837f13e3dc841e9e : SUCCESS in 59m 24s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/5f808476f1914e8ea6eba54e96d9deba : SUCCESS in 38m 22s","accounts_in_message":[],"_revision_number":4},{"id":"ace04c863cf4970ed54ea65638375d3adbffaf03","author":{"_account_id":27954,"name":"Moisés Guimarães de Medeiros","email":"guimaraes@pm.me","username":"moguimar"},"date":"2020-10-15 13:52:09.000000000","message":"Patch Set 4:\n\nrecheck","accounts_in_message":[],"_revision_number":4},{"id":"c6f44c65e8765be18f6816fc4e45d9d3ada53092","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-15 18:30:17.000000000","message":"Patch Set 4: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/475f583796424ecdb339b293fa7b920c : SUCCESS in 16m 29s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/40f31a70cac24970ae72ca5de1c17f1c : SUCCESS in 14m 28s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/758aee43163e40c4aa7aa767f9cdeaa0 : SUCCESS in 6m 06s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/c498a7d27e8d4280a9b302f0ea8c8d5a : SUCCESS in 16m 59s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/60b5ef1fe0af430b8a34ab01ad8b44eb : SUCCESS in 13m 35s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/93248bb6dad14ecfbe0122a7ffa2561d : SUCCESS in 12m 41s\n- grenade https://zuul.opendev.org/t/openstack/build/66e84ff3ccd64696a85ca1b14f959904 : SUCCESS in 1h 04m 41s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/989e077e4e9d49419016a1e7c7d15d41 : SUCCESS in 1h 15m 50s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/f60cda327e92436a80830cda84248bbf : SUCCESS in 8m 10s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/fc14f1db12d04042944151c2eccdf633 : SUCCESS in 37m 25s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal https://zuul.opendev.org/t/openstack/build/f5513868e5d5484c8c4abdf3bc05cf94 : SUCCESS in 54m 28s (non-voting)\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/e12c4771b920414bba0e96926df416d7 : SUCCESS in 36m 30s\n- keystoneclient-devstack-functional https://zuul.opendev.org/t/openstack/build/6da4a0e687e14a938cf604fbb792d081 : FAILURE in 21m 24s (non-voting)\n- keystone-dsvm-ldap-domain-specific-driver https://zuul.opendev.org/t/openstack/build/29398ab3b2b84d97bdf7d71f57fd2b08 : FAILURE in 12m 19s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/fd9fe056034a4a56a7a5da21ccbf0da2 : SUCCESS in 1h 02m 49s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/087115f1246c49ca9fbb8ee7d9dbcea7 : SUCCESS in 38m 19s","accounts_in_message":[],"_revision_number":4},{"id":"4b00b17fc605e482fcf9303cd0502b65639e5b30","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-15 18:31:15.000000000","message":"Patch Set 4: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":4},{"id":"fc973cef3c2d57b013487639e079b4b80c1e7420","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-15 19:55:24.000000000","message":"Patch Set 4: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/c090bc904f724b92b5c631e25f2d43cc : SUCCESS in 12m 16s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2bd0b6e33b4442e19b9c4542149b2faa : SUCCESS in 6m 06s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/c1264ecf77284bda80707546f7128092 : SUCCESS in 15m 33s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/e6cef04b74384668869df0b28c1eccb2 : SUCCESS in 20m 40s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/6d56172b2d284207b67c1750799ee6b6 : SUCCESS in 15m 04s\n- grenade https://zuul.opendev.org/t/openstack/build/25e64333a9f54de49a1e88bd473dfcfa : SUCCESS in 1h 07m 09s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/4974a233d0944cd0b2c6f528d6243055 : SUCCESS in 1h 20m 30s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/6286175d27ed452e9b784ce45213949d : SUCCESS in 7m 49s\n- keystone-dsvm-py3-functional https://zuul.opendev.org/t/openstack/build/6f0b0c8e032249f48982829bd75c0b43 : SUCCESS in 31m 51s\n- keystone-dsvm-py3-functional-federation-ubuntu-focal-k2k https://zuul.opendev.org/t/openstack/build/2a8ed319d5a64aef9aac833ae44ee184 : SUCCESS in 33m 47s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/1b2c852287da46ffb8540c549f35573f : SUCCESS in 1h 02m 11s\n- keystone-tox-protection https://zuul.opendev.org/t/openstack/build/6308447b866649daabcfdaefb3c68598 : SUCCESS in 46m 17s","accounts_in_message":[],"_revision_number":4},{"id":"b0f5018cc9c3f9a21b0b5cc9bad11f8d083e0ea9","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-15 19:55:24.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":4},{"id":"a2453aafcb38d649d62e3f17f6124d607b38906d","tag":"autogenerated:zuul:promote","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-15 19:57:18.000000000","message":"Patch Set 4:\n\nBuild succeeded (promote pipeline).\n\n- promote-openstack-tox-docs https://zuul.opendev.org/t/openstack/build/4df1cc00ec404d5fb6395d6379ad701e : SUCCESS in 1m 23s\n- promote-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/a089014427ca4026a54b4564b69afbed : SUCCESS in 58s","accounts_in_message":[],"_revision_number":4}],"current_revision_number":4,"current_revision":"e98d1ac622f74cbbb41872226ab755bf4ff3ed84","revisions":{"257bfce206c5c41495ff0e37e9674d5abb5dad01":{"kind":"REWORK","_number":1,"created":"2020-09-26 01:50:21.000000000","uploader":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"ref":"refs/changes/88/754488/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystone","ref":"refs/changes/88/754488/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystone refs/changes/88/754488/1"}}},"commit":{"parents":[{"commit":"db25e505a30b10ed8a2a66c4674e20130dd5d5e0","subject":"[goal] Migrate testing to ubuntu focal","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/db25e505a30b10ed8a2a66c4674e20130dd5d5e0"}]}],"author":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-26 01:48:59.000000000","tz":-300},"committer":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-26 01:48:59.000000000","tz":-300},"subject":"WIP: Properly cleanup ConnectionManager context managers","message":"WIP: Properly cleanup ConnectionManager context managers\n\nThis is a hypothesis. Please see the comment in common.py for more\ndetails. I\u0027m currently stress testing this in a deployment to see if it\nresolved the memory leak.\n\nChange-Id: Ia45a45ff852d0d4e3a713dae07a46d4ff8d370f3\nCloses-Bug: 1896125\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/257bfce206c5c41495ff0e37e9674d5abb5dad01"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/257bfce206c5c41495ff0e37e9674d5abb5dad01"}]},"branch":"refs/heads/master"},"776859329b388397664dfc3cfa5f813cd48ab1ee":{"kind":"REWORK","_number":2,"created":"2020-09-28 21:38:58.000000000","uploader":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"ref":"refs/changes/88/754488/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystone","ref":"refs/changes/88/754488/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystone refs/changes/88/754488/2"}}},"commit":{"parents":[{"commit":"db25e505a30b10ed8a2a66c4674e20130dd5d5e0","subject":"[goal] Migrate testing to ubuntu focal","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/db25e505a30b10ed8a2a66c4674e20130dd5d5e0"}]}],"author":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-28 20:41:28.000000000","tz":-300},"committer":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-28 21:38:16.000000000","tz":-300},"subject":"WIP: Attempt to cleanup context managers","message":"WIP: Attempt to cleanup context managers\n\nKeystone\u0027s paging implementation contains a memory leak. The issue is\nnoticeable if you integrate keystone with an LDAP server that supports\npaging and set `keystone.conf [ldap] page_size` to a low integer (e.g.,\n5).\n\nKeystone\u0027s LDAP backend uses `python-ldap` to interact with LDAP\nservers. For paged requests, it uses `search_ext()`, which is an\nasynchronous API [0]. The server responds with message IDs, which the\nclient uses to retrieve all data for the request. In keystone\u0027s case,\nthe `search_ext` method is invoked with a page control that tells the\nserver to deliver responses in increments according to the page size. So\nlong as the client has the original connection used to fetch the message\nID, it can request the rest of the information associated to the\nrequest.\n\nKeystone\u0027s paging implementation loops continuously for paged requests.\nIt takes the message ID it gets from `search_ext()` and calls\n`result3()`, asking the server for the data associated with that\nspecific message. Keystone continues to do this until the server sends\nan indicator that it has no more data relevant to the query (via a\ncookie). The LDAP driver must have the connection used to get the\nmessage ID in order to fetch the remaining data. The `search_ext()` and\n`result3()` methods must use the same LDAP connection.\n\nGiven the above information, keystone uses context managers to manage\nconnections. This is relevant when deploying connection pools, where\ncertain connections are re-used. Keystone relies on Python context\nmanagers to handle connections, which is pretty typical for connection\nuse-cases. Connection managers allow us to do the following (assuming\npseudocode)\n\n  with self.get_connection as conn:\n      response \u003d conn.search_s()\n      return format(response)\n\nThe above snippet assumes the `get_connection` method provides a\nconnection object and a callable that implements `search_s`. Upon\nexiting the `with` statement, the connection is disconnected, or put\nback into the pool, or whatever the implementation of the handler\ndecides to do. Most connections in the LDAP backend are handled in this\nfashion.\n\nUnfortunately, the LDAP driver is somewhat oblivious to paging, it\u0027s\ncontrol implementation, or the fact that it uses an asynchronous API.\nInstead, the driver leaves it up to the handler objects it uses for\nconnections to determine if the request should be controlled via paging.\nThis is an anti-pattern since the backend establishes the connection for\nthe request but doesn\u0027t ensure that connection is safely handled for\nasynchronous APIs.\n\nThis causes the implementation of the PooledLDAPHandler to know how to\nmanager context managers. The current code tried to clean up the\ncontext manager responsible for connections after the results are\ncollected from the server using the message ID. I believe it does this\nbecause it needs to get a new connection for each message in the paged\nresults, even though it already operation from within a connection\nestablished via a context manager. The code tries to use a weak\nreference to create a callback that tears down the context manager when\nnothing else references it. At a high-level, the idea is similar to the\nfollowing pseudocode:\n\n  with self.get_connection as conn:\n      while True:\n\tldap_data \u003d []\n\tcontext_manager \u003d self.get_connection()\n\tconnection \u003d context_manager.__enter__()\n\tmessage_id \u003d connection.search_ext()\n\tresults \u003d connection.result3(message_id)\n\tldap_data.append(results)\n\tcontext_manager.__exit__()\n\nI was unsuccessful in getting this code to work as described in\ncomments. I don\u0027t think the context managers are cleaned up, resulting\nin memory bloat, especially with low page sizes which results in more\nrequests. A weak reference invokes the callback when the weak reference\nis called, but there are no other references to the original object [1].\nIn our case, I don\u0027t think we invoke that path because we don\u0027t actually\ndo anything with the weak reference. We assume it\u0027s going to run the\ncallback when the object is garbage collected.\n\nThis commit attempts to address this issue by using the concept of a\nfinalizer [2], which was designed for similar cases.\n\nAn alternative approach would be to push more of the paging logic and\nimplementation up into the LDAP driver. This would make it easier to put\nthe entire asynchronous API flow for paging into a `with` statement and\nrelying on the normal behavior of context managers to clean up\naccordingly. This approach would remove the manual cleanup invocation,\nregardless of using weak references or finalizer objects.\n\n[0] https://www.python-ldap.org/en/python-ldap-3.3.0/reference/ldap.html#ldap.LDAPObject.search_ext\n[1] https://docs.python.org/3/library/weakref.html#weakref.ref\n[2] https://docs.python.org/3/library/weakref.html#finalizer-objects\n\nChange-Id: Ia45a45ff852d0d4e3a713dae07a46d4ff8d370f3\nCloses-Bug: 1896125\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/776859329b388397664dfc3cfa5f813cd48ab1ee"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/776859329b388397664dfc3cfa5f813cd48ab1ee"}]},"branch":"refs/heads/master"},"003c46fefc4a8863003a8d771622e6a87a8ab8c4":{"kind":"REWORK","_number":3,"created":"2020-09-29 15:55:45.000000000","uploader":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"ref":"refs/changes/88/754488/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystone","ref":"refs/changes/88/754488/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystone refs/changes/88/754488/3"}}},"commit":{"parents":[{"commit":"db25e505a30b10ed8a2a66c4674e20130dd5d5e0","subject":"[goal] Migrate testing to ubuntu focal","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/db25e505a30b10ed8a2a66c4674e20130dd5d5e0"}]}],"author":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-29 15:20:13.000000000","tz":-300},"committer":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-29 15:55:26.000000000","tz":-300},"subject":"Implement more robust connection handling for asynchronous LDAP calls","message":"Implement more robust connection handling for asynchronous LDAP calls\n\nKeystone\u0027s paging implementation contains a memory leak. The issue is\nnoticeable if you integrate keystone with an LDAP server that supports\npaging and set `keystone.conf [ldap] page_size` to a low integer\n(e.g., 5).\n\nKeystone\u0027s LDAP backend uses `python-ldap` to interact with LDAP\nservers. For paged requests, it uses `search_ext()`, which is an\nasynchronous API [0]. The server responds with a message ID, which the\nclient uses to retrieve all data for the request. In keystone\u0027s case,\nthe `search_ext()` method is invoked with a page control that tells\nthe server to deliver responses in increments according to the page\nsize configured with `keystone.conf [ldap] page_size`. So long as the\nclient has the original connection used to fetch the message ID, it\ncan request the rest of the information associated to the request.\n\nKeystone\u0027s paging implementation loops continuously for paged\nrequests. It takes the message ID it gets from `search_ext()` and\ncalls `result3()`, asking the server for the data associated with that\nspecific message. Keystone continues to do this until the server sends\nan indicator that it has no more data relevant to the query (via a\ncookie). The `search_ext()` and `result3()` methods must use the same\nLDAP connection.\n\nGiven the above information, keystone uses context managers to provide\nconnections. This is relevant when deploying connection pools, where\ncertain connections are re-used from a pool. Keystone relies on Python\ncontext managers to handle connections, which is pretty typical\nuse-case for context managers. Connection managers allow us to do the\nfollowing (assuming pseudocode):\n\n  with self.get_connection as conn:\n      response \u003d conn.search_s()\n      return format(response)\n\nThe above snippet assumes the `get_connection` method provides a\nconnection object and a callable that implements `search_s`. Upon\nexiting the `with` statement, the connection is disconnected, or put\nback into the pool, or whatever the implementation of the context\nmanager decides to do. Most connections in the LDAP backend are\nhandled in this fashion.\n\nUnfortunately, the LDAP driver is somewhat oblivious to paging, it\u0027s\ncontrol implementation, or the fact that it uses an asynchronous API.\nInstead, the driver leaves it up to the handler objects it uses for\nconnections to determine if the request should be controlled via\npaging. This is an anti-pattern since the backend establishes the\nconnection for the request but doesn\u0027t ensure that connection is\nsafely handled for asynchronous APIs.\n\nThis forces the `search_ext()` and `result3()` implementations in the\nPooledLDAPHandler to know how to handle connections and context\nmanagers, since it needs to ensure the same connection is used for\npaged requests. The current code tried to clean up the context\nmanager responsible for connections after the results are collected\nfrom the server using the message ID. I believe it does this because\nit needs to get a new connection for each message in the paged\nresults, even though it already operates from within a connection\nestablished via a context manager and the PooledLDAPHandler almost\nalways returns the same connection object from the pool. The code\ntries to use a weak reference to create a callback that tears down the\ncontext manager when nothing else references it. At a high-level, the\nidea is similar to the following pseudocode:\n\n  with self.get_connection as conn:\n      while True:\n\tldap_data \u003d []\n\tcontext_manager \u003d self.get_connection()\n\tconnection \u003d context_manager.__enter__()\n\tmessage_id \u003d connection.search_ext()\n\tresults \u003d connection.result3(message_id)\n\tldap_data.append(results)\n\tcontext_manager.__exit__()\n\nI wasn\u0027t able to see the callback get invoked or work as described in\ncomments, resulting in memory bloat, especially with low page sizes\nwhich results in more requests. A weak reference invokes the callback\nwhen the weak reference is called, but there are no other references\nto the original object [1]. In our case, I don\u0027t think we invoke that\npath because we don\u0027t actually do anything with the weak reference. We\nassume it\u0027s going to run the callback when the object is garbage\ncollected.\n\nThis commit attempts to address this issue by using the concept of a\nfinalizer [2], which was designed for similar cases. It also attempts\nto hide the cleanup implementation in the AsynchronousMessage object,\nso that callers don\u0027t have to worry about making sure they invoke the\nfinalizer.\n\nAn alternative approach would be to push more of the paging logic and\nimplementation up into the LDAP driver. This would make it easier to\nput the entire asynchronous API flow for paging into a `with`\nstatement and relying on the normal behavior of context managers to\nclean up accordingly. This approach would remove the manual cleanup\ninvocation, regardless of using weak references or finalizer objects.\nHowever, this approach would likely require a non-trivial amount of\ndesign work to refactor the entire LDAP backend. The LDAP backend has\nother issues that would complicate the re-design process:\n\n  - Handlers and connection are generalized to mean the same thing\n  - Method names don\u0027t follow a convention\n  - Domain-specific language from python-ldap bleeds into keystone\u0027s\n    implementation (e.g., get_all, _ldap_get_all, add_member) at\n    different points in the backend (e.g., UserApi (BaseLdap), GroupApi\n    (BaseLdap), KeystoneLDAPHandler, PooledLDAPHandler,\n    PythonLDAPHandler)\n  - Backend contains dead code from when keystone supported writeable\n    LDAP backends\n  - Responsibility for connections and connection handling is spread\n    across objects (BaseLdap, LDAPHandler)\n  - Handlers will invoke methods differently based on configuration at\n    runtime, which is a sign that the relationship between the driver,\n    handlers, and connection objects isn\u0027t truely polymorphic\n\nWhile keeping the logic for properly handling context managers and\nconnections in the Handlers might not be ideal, it is a relatively\nminimal fix in comparison to a re-design or backend refactor. These\nissues can be considered during a refactor of the LDAP backend if or\nwhen the community decides to re-design the LDAP backend.\n\n[0] https://www.python-ldap.org/en/python-ldap-3.3.0/reference/ldap.html#ldap.LDAPObject.search_ext\n[1] https://docs.python.org/3/library/weakref.html#weakref.ref\n[2] https://docs.python.org/3/library/weakref.html#finalizer-objects\n\nCloses-Bug: 1896125\nChange-Id: Ia45a45ff852d0d4e3a713dae07a46d4ff8d370f3\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/003c46fefc4a8863003a8d771622e6a87a8ab8c4"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/003c46fefc4a8863003a8d771622e6a87a8ab8c4"}]},"branch":"refs/heads/master"},"e98d1ac622f74cbbb41872226ab755bf4ff3ed84":{"kind":"REWORK","_number":4,"created":"2020-09-30 13:28:03.000000000","uploader":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"ref":"refs/changes/88/754488/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystone","ref":"refs/changes/88/754488/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystone refs/changes/88/754488/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystone refs/changes/88/754488/4"}}},"commit":{"parents":[{"commit":"db25e505a30b10ed8a2a66c4674e20130dd5d5e0","subject":"[goal] Migrate testing to ubuntu focal","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/db25e505a30b10ed8a2a66c4674e20130dd5d5e0"}]}],"author":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-29 15:20:13.000000000","tz":-300},"committer":{"name":"Lance Bragstad","email":"lbragstad@gmail.com","date":"2020-09-30 13:27:40.000000000","tz":-300},"subject":"Implement more robust connection handling for asynchronous LDAP calls","message":"Implement more robust connection handling for asynchronous LDAP calls\n\nKeystone\u0027s paging implementation contains a memory leak. The issue is\nnoticeable if you integrate keystone with an LDAP server that supports\npaging and set `keystone.conf [ldap] page_size` to a low integer\n(e.g., 5).\n\nKeystone\u0027s LDAP backend uses `python-ldap` to interact with LDAP\nservers. For paged requests, it uses `search_ext()`, which is an\nasynchronous API [0]. The server responds with a message ID, which the\nclient uses to retrieve all data for the request. In keystone\u0027s case,\nthe `search_ext()` method is invoked with a page control that tells\nthe server to deliver responses in increments according to the page\nsize configured with `keystone.conf [ldap] page_size`. So long as the\nclient has the original connection used to fetch the message ID, it\ncan request the rest of the information associated to the request.\n\nKeystone\u0027s paging implementation loops continuously for paged\nrequests. It takes the message ID it gets from `search_ext()` and\ncalls `result3()`, asking the server for the data associated with that\nspecific message. Keystone continues to do this until the server sends\nan indicator that it has no more data relevant to the query (via a\ncookie). The `search_ext()` and `result3()` methods must use the same\nLDAP connection.\n\nGiven the above information, keystone uses context managers to provide\nconnections. This is relevant when deploying connection pools, where\ncertain connections are re-used from a pool. Keystone relies on Python\ncontext managers to handle connections, which is pretty typical\nuse-case for context managers. Connection managers allow us to do the\nfollowing (assuming pseudocode):\n\n  with self.get_connection as conn:\n      response \u003d conn.search_s()\n      return format(response)\n\nThe above snippet assumes the `get_connection` method provides a\nconnection object and a callable that implements `search_s`. Upon\nexiting the `with` statement, the connection is disconnected, or put\nback into the pool, or whatever the implementation of the context\nmanager decides to do. Most connections in the LDAP backend are\nhandled in this fashion.\n\nUnfortunately, the LDAP driver is somewhat oblivious to paging, it\u0027s\ncontrol implementation, or the fact that it uses an asynchronous API.\nInstead, the driver leaves it up to the handler objects it uses for\nconnections to determine if the request should be controlled via\npaging. This is an anti-pattern since the backend establishes the\nconnection for the request but doesn\u0027t ensure that connection is\nsafely handled for asynchronous APIs.\n\nThis forces the `search_ext()` and `result3()` implementations in the\nPooledLDAPHandler to know how to handle connections and context\nmanagers, since it needs to ensure the same connection is used for\npaged requests. The current code tried to clean up the context\nmanager responsible for connections after the results are collected\nfrom the server using the message ID. I believe it does this because\nit needs to get a new connection for each message in the paged\nresults, even though it already operates from within a connection\nestablished via a context manager and the PooledLDAPHandler almost\nalways returns the same connection object from the pool. The code\ntries to use a weak reference to create a callback that tears down the\ncontext manager when nothing else references it. At a high-level, the\nidea is similar to the following pseudocode:\n\n  with self.get_connection as conn:\n      while True:\n\tldap_data \u003d []\n\tcontext_manager \u003d self.get_connection()\n\tconnection \u003d context_manager.__enter__()\n\tmessage_id \u003d connection.search_ext()\n\tresults \u003d connection.result3(message_id)\n\tldap_data.append(results)\n\tcontext_manager.__exit__()\n\nI wasn\u0027t able to see the callback get invoked or work as described in\ncomments, resulting in memory bloat, especially with low page sizes\nwhich results in more requests. A weak reference invokes the callback\nwhen the weak reference is called, but there are no other references\nto the original object [1]. In our case, I don\u0027t think we invoke that\npath because we don\u0027t actually do anything with the weak reference. We\nassume it\u0027s going to run the callback when the object is garbage\ncollected.\n\nThis commit attempts to address this issue by using the concept of a\nfinalizer [2], which was designed for similar cases. It also attempts\nto hide the cleanup implementation in the AsynchronousMessage object,\nso that callers don\u0027t have to worry about making sure they invoke the\nfinalizer.\n\nAn alternative approach would be to push more of the paging logic and\nimplementation up into the LDAP driver. This would make it easier to\nput the entire asynchronous API flow for paging into a `with`\nstatement and relying on the normal behavior of context managers to\nclean up accordingly. This approach would remove the manual cleanup\ninvocation, regardless of using weak references or finalizer objects.\nHowever, this approach would likely require a non-trivial amount of\ndesign work to refactor the entire LDAP backend. The LDAP backend has\nother issues that would complicate the re-design process:\n\n  - Handlers and connection are generalized to mean the same thing\n  - Method names don\u0027t follow a convention\n  - Domain-specific language from python-ldap bleeds into keystone\u0027s\n    implementation (e.g., get_all, _ldap_get_all, add_member) at\n    different points in the backend (e.g., UserApi (BaseLdap), GroupApi\n    (BaseLdap), KeystoneLDAPHandler, PooledLDAPHandler,\n    PythonLDAPHandler)\n  - Backend contains dead code from when keystone supported writeable\n    LDAP backends\n  - Responsibility for connections and connection handling is spread\n    across objects (BaseLdap, LDAPHandler)\n  - Handlers will invoke methods differently based on configuration at\n    runtime, which is a sign that the relationship between the driver,\n    handlers, and connection objects isn\u0027t truely polymorphic\n\nWhile keeping the logic for properly handling context managers and\nconnections in the Handlers might not be ideal, it is a relatively\nminimal fix in comparison to a re-design or backend refactor. These\nissues can be considered during a refactor of the LDAP backend if or\nwhen the community decides to re-design the LDAP backend.\n\n[0] https://www.python-ldap.org/en/python-ldap-3.3.0/reference/ldap.html#ldap.LDAPObject.search_ext\n[1] https://docs.python.org/3/library/weakref.html#weakref.ref\n[2] https://docs.python.org/3/library/weakref.html#finalizer-objects\n\nCloses-Bug: 1896125\nChange-Id: Ia45a45ff852d0d4e3a713dae07a46d4ff8d370f3\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/e98d1ac622f74cbbb41872226ab755bf4ff3ed84"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystone/commit/e98d1ac622f74cbbb41872226ab755bf4ff3ed84"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}