)]}'
{"id":"openstack%2Fkeystonemiddleware~773939","triplet_id":"openstack%2Fkeystonemiddleware~master~I0e96334b65a0bf369ebf1d88651d13feb8d2ecac","project":"openstack/keystonemiddleware","branch":"master","topic":"bug/1883659","hashtags":[],"change_id":"I0e96334b65a0bf369ebf1d88651d13feb8d2ecac","subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","status":"MERGED","created":"2021-02-03 16:22:16.000000000","updated":"2021-04-11 07:37:20.000000000","submitted":"2021-03-02 18:41:20.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":5,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"773939-bug/1883659","meta_rev_id":"8425f9d40ec55f0d637d0a41401140118be4099f","_number":773939,"virtual_id_number":773939,"owner":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"value":0,"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"value":0,"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":0,"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},{"tag":"autogenerated:zuul:gate","value":2,"date":"2021-03-02 18:41:19.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"all":[{"value":2,"date":"2021-02-23 14:38:59.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"value":2,"date":"2021-03-02 17:09:51.000000000","_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"value":1,"date":"2021-02-11 14:37:27.000000000","permitted_voting_range":{"min":1,"max":2},"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":2,"date":"2021-03-02 17:25:05.000000000","_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":1,"date":"2021-02-12 23:17:02.000000000","permitted_voting_range":{"min":1,"max":2},"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"all":[{"value":0,"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"value":1,"date":"2021-03-02 17:09:51.000000000","_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"value":0,"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"value":0,"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"value":0,"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},{"value":0,"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"}],"reviewers":{"CC":[{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"}],"REVIEWER":[{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2021-02-03 18:11:04.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2021-02-03 20:07:05.000000000","updated_by":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"reviewer":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"state":"REVIEWER"},{"updated":"2021-02-03 20:07:05.000000000","updated_by":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"reviewer":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"state":"REVIEWER"},{"updated":"2021-02-03 20:09:06.000000000","updated_by":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"reviewer":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"state":"CC"},{"updated":"2021-02-03 20:09:06.000000000","updated_by":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"reviewer":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"state":"CC"},{"updated":"2021-02-03 20:11:04.000000000","updated_by":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"reviewer":{"_account_id":11904,"name":"Sean McGinnis","email":"sean.mcginnis@gmail.com","username":"SeanM"},"state":"REVIEWER"},{"updated":"2021-02-08 12:31:57.000000000","updated_by":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"reviewer":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"state":"REVIEWER"},{"updated":"2021-02-08 16:29:14.000000000","updated_by":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"reviewer":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"state":"REVIEWER"},{"updated":"2021-02-12 23:17:02.000000000","updated_by":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"reviewer":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"state":"REVIEWER"},{"updated":"2021-02-23 14:38:59.000000000","updated_by":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"reviewer":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"state":"REVIEWER"},{"updated":"2021-02-23 14:39:34.000000000","updated_by":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"reviewer":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"state":"REVIEWER"},{"updated":"2021-02-23 14:39:34.000000000","updated_by":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"reviewer":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"state":"REVIEWER"},{"updated":"2021-03-02 17:09:51.000000000","updated_by":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"reviewer":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"state":"REVIEWER"}],"messages":[{"id":"e2fccc9f4576e269dc1445a838a7e5abd0f32459","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-03 16:22:16.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"fef0d411025b0a2ab22234cff832201804763e16","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-03 16:24:19.000000000","message":"Uploaded patch set 2: Commit message was updated.","accounts_in_message":[],"_revision_number":2},{"id":"8da2e7a85542332eeb637366abf27b0476cf71f6","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-02-03 18:11:04.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/43fa987b00b4440a8b8ac6f0d17d8f86 : SUCCESS in 3m 25s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/a70ed37d32e04694b92551fb42e05b13 : FAILURE in 2m 44s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/0cbc636279a7413a8a2790e554c1df48 : SUCCESS in 3m 51s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/cb98366ea183453390f2a650c89e1afd : SUCCESS in 3m 09s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/8143a88de3ad40cbaa5c323515cbb1db : SUCCESS in 3m 25s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/f2871ab9da564a9ca3d5ada7a2e2f68d : SUCCESS in 3m 41s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/6ce6ac2c6d604a32bc550ac3d2a10595 : SUCCESS in 6m 36s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/5cbb2149e4c64364804f501545c27dd3 : SUCCESS in 1h 32m 48s","accounts_in_message":[],"_revision_number":2},{"id":"cca52292de200ff0c8dfbe1ee04afa84fecaa9ae","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-03 19:03:37.000000000","message":"Patch Set 2:\n\nrecheck","accounts_in_message":[],"_revision_number":2},{"id":"2e91a6c99893be96b14764c819f48c86198e015c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-03 19:56:43.000000000","message":"Uploaded patch set 3: Patch Set 2 was rebased.","accounts_in_message":[],"_revision_number":3},{"id":"d33a35532505f52121b7449e0435b23055641220","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-02-03 21:08:08.000000000","message":"Patch Set 3: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/db522a51b32849279726089394f2c76c : SUCCESS in 3m 43s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/f23b41c3b7654d67be9afce97fa2478a : SUCCESS in 3m 03s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/4bcb274f3b8a402d9f9da02d9f8f91ad : SUCCESS in 3m 38s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/7884972b96d3493e8f53ece3d8773d27 : SUCCESS in 3m 57s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/79c270e12b4a48c6b43a61bbc3b57876 : SUCCESS in 3m 43s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/fcf39ad2ed3f45b18636e69c6a3a17cb : SUCCESS in 6m 05s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/1fd3a906448f4a82ac7ff1f0d3b53f55 : SUCCESS in 1h 08m 13s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/e92a3789dd344c9f9a1ee55d812f0d66 : FAILURE in 2m 32s (non-voting)","accounts_in_message":[],"_revision_number":3},{"id":"cf174d26fc6c67fa009bcba42c7cab69a3e8e9e9","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-03 21:46:16.000000000","message":"Uploaded patch set 4: Commit message was updated.","accounts_in_message":[],"_revision_number":4},{"id":"79470798dbba0fceea4cc8926f296f6bbb629d42","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-02-03 23:02:38.000000000","message":"Patch Set 4: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/e598e740b87545308392a845298b827c : SUCCESS in 4m 36s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/f3f1d66af47744f89214d68cca2ff46f : SUCCESS in 2m 38s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/6eca0d38d5d2427ebc6e05a318de25a0 : SUCCESS in 3m 06s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/8ec85d685f7546e08521723f057819fc : SUCCESS in 3m 55s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/cdc75bd04cab4a93b0ef5fca5866a3b0 : SUCCESS in 3m 15s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/272d43e90e2f45a8b3086fafe3200c3b : SUCCESS in 4m 44s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/bb2564cb753440b1987d8419e40f7ac3 : FAILURE in 1h 10m 01s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/01d65e02ea0f4b82929583bf4b07e533 : FAILURE in 2m 30s (non-voting)","accounts_in_message":[],"_revision_number":4},{"id":"59f898df1fc44c9016b49f3e2eec06ac3cc31ae3","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-04 06:04:23.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"d0b8f8c55edd79ccd937a00fb8d09963f0d57516","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-02-04 07:25:38.000000000","message":"Patch Set 5: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/d4803e025eca4aa5975d4750098ceb52 : SUCCESS in 7m 35s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/62206b0a892c483f83dbb05c2f6be691 : SUCCESS in 5m 07s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/0b0af2ceaabf4a3b9ec1518596fcc9bc : SUCCESS in 9m 16s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/00b3e638e32647069e5155670e323de9 : SUCCESS in 6m 09s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/7d9ecf5b664c4492bf002837e38998f2 : SUCCESS in 4m 54s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/3dc07a342e6345698459bd0f96e5060a : SUCCESS in 6m 02s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/bdbd2348994c443ca756aadf9aba2783 : SUCCESS in 1h 09m 46s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/2165b29b4c1b4c3cb72cfca55c6b3354 : FAILURE in 2m 51s (non-voting)","accounts_in_message":[],"_revision_number":5},{"id":"0e6475df365b7bc7cd6a7ec9c488aa1c27117333","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2021-02-04 10:54:43.000000000","message":"Patch Set 5:\n\n(1 comment)\n\nI wonder if this the right way to do this.\n\nCan\u0027t we follow/apply the way keystone define is cache module? [][]\n\nkeystonemiddleware\u0027s cache module seems really redundant to oslo.cache\u0027s code in general.\nI think we could drop many part of this module.\n\nWithout too digging many options seems duplicated here and are also available in oslo.cache, it would better to only rely on oslo.cache.\n\nLet\u0027s wait for the keystone team\u0027s feedbacks.\n\n[1] https://opendev.org/openstack/keystone/src/branch/master/keystone/common/cache/core.py\n[2] https://docs.openstack.org/oslo.cache/latest/user/usage.html","accounts_in_message":[],"_revision_number":5},{"id":"53cc4721a2dacfc7843b2765ec53261adf502a34","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-04 11:06:13.000000000","message":"Patch Set 5:\n\n(1 comment)","accounts_in_message":[],"_revision_number":5},{"id":"956af23aa0791124c88586a2c10eee74825c0af6","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-04 11:24:28.000000000","message":"Patch Set 5:\n\nWell, I am not core developer, I\u0027m just engineer who is using, fixing, operating OStack in production.\nBUT, From my perspective of view, you can\u0027t mix cache layer of keystone itself AND keystonemiddleware which is used by every service, can you ? \n\nKeystone cache layer is for keystone caching and it\u0027s modificable by config options - OK agree.\nBut then, there is every service which IS authenticating against keystone (keystonemiddleware) and this is ONLY token caching ..so that is the reason (i think) why it is splitted to keystonemiddleware (and also why options for memcache in keystone_authtoken are duplicated)\n\nAs I said above, now you are able to configure:\n 1. Token caching (via options in keystonemiddleware) per service\n 2. Global Cache (via oslo_cache) per service \n 3. Different behaviour for 1 and 2 \n\nToken via memcache for example\nCache data via redis for example.\n\nIf it will be all common, you can\u0027t split data and tokens into two caches ..for example ..\nSo, I think this is for more work and bigger discussion.\n\nBut yes, agree, let\u0027s wait for keystone feedback.. \nIf feedback will be positive, I will patch this into debian probably as it looks like our users would appreciate it, and it will solve a lot of problems with cache.","accounts_in_message":[],"_revision_number":5},{"id":"2c252bff8fda021b54becc10e2c68595fa572080","author":{"_account_id":28522,"name":"Hervé Beraud","email":"herveberaud.pro@gmail.com","username":"hberaud"},"date":"2021-02-08 12:31:57.000000000","message":"Patch Set 5: Code-Review+1\n\n\u003e Patch Set 5:\n\u003e \n\u003e Well, I am not core developer, I\u0027m just engineer who is using, fixing, operating OStack in production.\n\u003e BUT, From my perspective of view, you can\u0027t mix cache layer of keystone itself AND keystonemiddleware which is used by every service, can you ? \n\u003e \n\u003e Keystone cache layer is for keystone caching and it\u0027s modificable by config options - OK agree.\n\u003e But then, there is every service which IS authenticating against keystone (keystonemiddleware) and this is ONLY token caching ..so that is the reason (i think) why it is splitted to keystonemiddleware (and also why options for memcache in keystone_authtoken are duplicated)\n\u003e \n\u003e As I said above, now you are able to configure:\n\u003e  1. Token caching (via options in keystonemiddleware) per service\n\u003e  2. Global Cache (via oslo_cache) per service \n\u003e  3. Different behaviour for 1 and 2 \n\u003e \n\u003e Token via memcache for example\n\u003e Cache data via redis for example.\n\u003e \n\u003e If it will be all common, you can\u0027t split data and tokens into two caches ..for example ..\n\u003e So, I think this is for more work and bigger discussion.\n\u003e \n\u003e But yes, agree, let\u0027s wait for keystone feedback.. \n\u003e If feedback will be positive, I will patch this into debian probably as it looks like our users would appreciate it, and it will solve a lot of problems with cache.\n\nOk thanks for details.\n\nI\u0027m not core developer on keystone too, so I can\u0027t help you to move this patch further, however I\u0027ll +1 anyway as this patch LGTM and can\u0027t hurt.","accounts_in_message":[],"_revision_number":5},{"id":"7af1819c427d24d6ff95dc83198bf0bdd0ced9b6","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2021-02-08 16:29:14.000000000","message":"Patch Set 5: Code-Review-1\n\n(1 comment)\n\nRather than switching this as you\u0027ve done, could we not simply set the default value of the config option to True and mark it as deprecated? It sounds like the non-MemcacheClientPool path should be deprecated anyway.\n\nThis also needs a release note","accounts_in_message":[],"_revision_number":5},{"id":"77d583c505338a68e5d2a1acd0d31e3dfbe32872","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-10 10:25:36.000000000","message":"Patch Set 5:\n\n\u003e Patch Set 5: Code-Review-1\n\u003e \n\u003e (1 comment)\n\u003e \n\u003e Rather than switching this as you\u0027ve done, could we not simply set the default value of the config option to True and mark it as deprecated? It sounds like the non-MemcacheClientPool path should be deprecated anyway.\n\u003e \n\u003e This also needs a release note\n\nWell, It depends on you, if it will be better to just change default value to true , then OK, i will rework review.\n\nJust let me know and I will process it.","accounts_in_message":[],"_revision_number":5},{"id":"26af840ebe077a63ede7eee850a9a123624d15b4","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2021-02-10 11:01:51.000000000","message":"Patch Set 5:\n\n\u003e Patch Set 5:\n\u003e \n\u003e \u003e Patch Set 5: Code-Review-1\n\u003e \u003e \n\u003e \u003e (1 comment)\n\u003e \u003e \n\u003e \u003e Rather than switching this as you\u0027ve done, could we not simply set the default value of the config option to True and mark it as deprecated? It sounds like the non-MemcacheClientPool path should be deprecated anyway.\n\u003e \u003e \n\u003e \u003e This also needs a release note\n\u003e \n\u003e Well, It depends on you, if it will be better to just change default value to true , then OK, i will rework review.\n\u003e \n\u003e Just let me know and I will process it.\n\nYes, I think that would be wiser. We can remove this in a follow-up but it gives people time to switch back and report issues if there are any","accounts_in_message":[],"_revision_number":5},{"id":"d440f62763492db4b58b4efcbe04aeff47b96309","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-10 13:50:47.000000000","message":"Uploaded patch set 6.","accounts_in_message":[],"_revision_number":6},{"id":"cb32b435a1a0e4c957a0242f4a786f22d4b2c679","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2021-02-10 14:32:00.000000000","message":"Patch Set 6: Code-Review-1\n\nMuch better. Can you add a release note too? You can create one using \u0027reno\u0027","accounts_in_message":[],"_revision_number":6},{"id":"281934c081cd481aabd6e6c056a6cbdce32f14ba","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-02-10 15:31:16.000000000","message":"Patch Set 6: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/d242f56e303942099bfe78ba9c7dfa2e : SUCCESS in 4m 53s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/af05f042a77a4bbe91e9c0f3e8f7e0b3 : SUCCESS in 2m 56s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/b2f22acbe3eb4bb78e7b7ea219c98861 : SUCCESS in 3m 12s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/cf38b4a94fa144a593ecc39aa0321cf8 : SUCCESS in 3m 09s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/c5a4d790a5524630b49b0fb6665e13ce : SUCCESS in 3m 24s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/e3732e8b4f124a1d9ad3cdf838956d13 : SUCCESS in 5m 25s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/533afac480cc4a3fa3ca122ad648f02e : SUCCESS in 1h 38m 24s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/8460b6cf7c9c4ceba2af14626850b1fd : FAILURE in 2m 22s (non-voting)","accounts_in_message":[],"_revision_number":6},{"id":"6bb1c39f83b97e4f2908779b5cbd63133ceafbba","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-11 13:34:09.000000000","message":"Uploaded patch set 7.","accounts_in_message":[],"_revision_number":7},{"id":"de756ef12d39eb1f500c9047b6fb20d3b9e3fca1","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-11 13:37:04.000000000","message":"Patch Set 7:\n\n\u003e Patch Set 6: Code-Review-1\n\u003e \n\u003e Much better. Can you add a release note too? You can create one using \u0027reno\u0027\n\nI did, can u check it ? Honestly, I\u0027m not writing release notes every day :/ .","accounts_in_message":[],"_revision_number":7},{"id":"4e555b53d45d7b0df936bf2b4eb1697c18c5cf1f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2021-02-11 14:36:33.000000000","message":"Patch Set 8: Published edit on patch set 7.","accounts_in_message":[],"_revision_number":8},{"id":"b99651d4abe5cb7b53b373fb1372632c5089b7f6","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2021-02-11 14:37:27.000000000","message":"Patch Set 8: Code-Review+1\n\nI removed the prelude section since that should only be used for releases. This looks good to me now. Thanks for working through the issues. Hopefully we can find someone with +2 rights to review it now 😊","accounts_in_message":[],"_revision_number":8},{"id":"1651b6b4e6e74d594f617757b2ba46440ef9b264","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-11 15:39:06.000000000","message":"Patch Set 8:\n\n\u003e Patch Set 8: Code-Review+1\n\u003e \n\u003e I removed the prelude section since that should only be used for releases. This looks good to me now. Thanks for working through the issues. Hopefully we can find someone with +2 rights to review it now 😊\n\nAs I said, I\u0027m not good in release notes, but in original patchset I\u0027ve defined fixes: section and you replaced \"fixes\" with \"upgrades\", is it ok, ? (or it is typo ?)","accounts_in_message":[],"_revision_number":8},{"id":"6d2a5f2040bd5746ade77e28e332ddfd6bbd1a24","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-02-11 16:23:00.000000000","message":"Patch Set 8: Verified+1\n\nBuild succeeded (check pipeline).\n\n- openstack-tox-cover https://zuul.opendev.org/t/openstack/build/77c68a5dff2348c487ec0cacf7dad928 : SUCCESS in 3m 38s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/fe7d79225ef046a8b8482b7dfb9edeea : SUCCESS in 3m 37s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/d9e0290911054234a4c12f49751e54a1 : SUCCESS in 4m 31s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/ee5838ff9a01404f87035c08d2a0c9ae : SUCCESS in 3m 38s\n- openstack-tox-py38 https://zuul.opendev.org/t/openstack/build/81f59e17bd0b402bb370a30057105dc3 : SUCCESS in 3m 42s (non-voting)\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/ad24841ee4c44486b5603f131506be17 : SUCCESS in 5m 24s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/6696ef0f93f24e90ba16b78951857c37 : SUCCESS in 1h 43m 31s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/0b4597030a9a485684bc419cfdc64db9 : SUCCESS in 4m 53s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/f496eb399fc84682b1670c2c681d0f16 : POST_FAILURE in 18m 17s (non-voting)","accounts_in_message":[],"_revision_number":8},{"id":"cf37791f3c71f2430a27671f9b13005a5ab9301f","author":{"_account_id":14250,"name":"Grzegorz Grasza","email":"xek@redhat.com","username":"xek"},"date":"2021-02-12 23:17:02.000000000","message":"Patch Set 8: Code-Review+1","accounts_in_message":[],"_revision_number":8},{"id":"c6194606562777897475dd800d970aab40c36a55","author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"date":"2021-02-13 11:10:56.000000000","message":"Patch Set 8:\n\n\u003e Patch Set 8:\n\u003e \n\u003e \u003e Patch Set 8: Code-Review+1\n\u003e \u003e \n\u003e \u003e I removed the prelude section since that should only be used for releases. This looks good to me now. Thanks for working through the issues. Hopefully we can find someone with +2 rights to review it now 😊\n\u003e \n\u003e As I said, I\u0027m not good in release notes, but in original patchset I\u0027ve defined fixes: section and you replaced \"fixes\" with \"upgrades\", is it ok, ? (or it is typo ?)\n\nYes, upgrade implies the user must do something because there\u0027s been a change in behaviour. We changed the default so that is correct","accounts_in_message":[],"_revision_number":8},{"id":"240b681de90fe43bebd83463f0cefb294c7cbf2a","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-02-19 11:36:35.000000000","message":"Patch Set 8:\n\n(2 comments)","accounts_in_message":[],"_revision_number":8},{"id":"8ef176f81a4181babda0da2b77768bba8229069c","author":{"_account_id":16465,"name":"Kristi Nikolla","email":"knikolla@bu.edu","username":"knikolla"},"date":"2021-02-23 14:38:59.000000000","message":"Patch Set 8: Code-Review+2\n\nLooks good to me. Waiting on another core to +2 as I\u0027m not an expert on the caching side of keystone or ksm.","accounts_in_message":[],"_revision_number":8},{"id":"9147fcf203077162363d835bfbcff111bdb444dd","author":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"},"date":"2021-03-02 17:09:51.000000000","message":"Patch Set 8: Code-Review+2 Workflow+1","accounts_in_message":[],"_revision_number":8},{"id":"dd5ffa5e76db83d2770561a9005ebcd2236bdd07","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-03-02 17:10:48.000000000","message":"Patch Set 8: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":8},{"id":"4426ad56f91cc51811ae1acdebfca7265306f849","author":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"},"date":"2021-03-02 17:25:05.000000000","message":"Patch Set 8: Code-Review+2","accounts_in_message":[],"_revision_number":8},{"id":"dc49a88a3e3622f1724902cd892d6d2242dd9300","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-03-02 18:41:19.000000000","message":"Patch Set 8: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/2010cb7a1fb04d62a52023f1c3960b28 : SUCCESS in 4m 48s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/578dba9058a744c59d1e1518aaf0b9e5 : SUCCESS in 6m 11s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/85e32fccb46741cd9c42fb3f5fc93723 : SUCCESS in 4m 59s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/18d9a4f926884a88bf145ef936599d1f : SUCCESS in 6m 10s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/b062758c9d504642a733a8032f4e44ed : SUCCESS in 1h 27m 52s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/c26c70cc90bd4cf182dc1ba71cd91f72 : SUCCESS in 5m 52s","accounts_in_message":[],"_revision_number":8},{"id":"020512c695ec42cca2700937d3007f5021e95b15","tag":"autogenerated:gerrit:merged","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-03-02 18:41:20.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":8},{"id":"eb12607dacaa3f6d31982cc56695efa42548475f","tag":"autogenerated:zuul:promote","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2021-03-02 18:42:53.000000000","message":"Patch Set 8:\n\nBuild succeeded (promote pipeline).\n\n- promote-openstack-tox-docs https://zuul.opendev.org/t/openstack/build/4a843e26a93d4d79b05ced7fa7d2a336 : SUCCESS in 1m 04s\n- promote-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/11d1a10eb886416690177f4f54dd2161 : SUCCESS in 1m 07s","accounts_in_message":[],"_revision_number":8},{"id":"8425f9d40ec55f0d637d0a41401140118be4099f","author":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"date":"2021-04-11 07:37:20.000000000","message":"Patch Set 8:\n\nHello,\n\nI\u0027ve provided this patch on February 11th, it was merged on March 2nd, but can\u0027t see this in stable/wallaby branch, does it mean it will be not released in wallaby ?\n\nCould I please ask why ? \nI think there was quite big window to include this patch in wallaby and tag as release.\nI will definitely patch library in debian package, just wanted to know what is the reason - to better process patches next time in upstream.\n\nThank you,\nMichal Arbet (kevko)","accounts_in_message":[],"_revision_number":8}],"current_revision_number":8,"current_revision":"788d3c4969e3446778496f3a9055f654602ae2c1","revisions":{"0616af1b18483cfdb13a36e31e663d3f1f6659a6":{"kind":"REWORK","_number":1,"created":"2021-02-03 16:22:16.000000000","uploader":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"ref":"refs/changes/39/773939/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/1"}}},"commit":{"parents":[{"commit":"3659bdad861c1648dc53a1f66c968e82b498eb9c","subject":"Merge \"[goal] Migrate testing to ubuntu focal\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/3659bdad861c1648dc53a1f66c968e82b498eb9c"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 15:26:45.000000000","tz":60},"committer":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 16:22:14.000000000","tz":60},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand removing option, so it will be used by default.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections:\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix ?\n\n  - with memcache_use_advanced_pool\u003dFalse\n    OR\n  - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0616af1b18483cfdb13a36e31e663d3f1f6659a6"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0616af1b18483cfdb13a36e31e663d3f1f6659a6"}]},"branch":"refs/heads/master"},"a16b2a1997a4e8e68a45e1f065f5d7daceed81e7":{"kind":"NO_CODE_CHANGE","_number":2,"created":"2021-02-03 16:24:19.000000000","uploader":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"ref":"refs/changes/39/773939/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/2"}}},"commit":{"parents":[{"commit":"3659bdad861c1648dc53a1f66c968e82b498eb9c","subject":"Merge \"[goal] Migrate testing to ubuntu focal\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/3659bdad861c1648dc53a1f66c968e82b498eb9c"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 15:26:45.000000000","tz":60},"committer":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 16:23:29.000000000","tz":60},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand removing option, so it will be used by default.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections:\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix and test it ?\n\nRepeat above:\n - with memcache_use_advanced_pool\u003dFalse\n   OR\n - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/a16b2a1997a4e8e68a45e1f065f5d7daceed81e7"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/a16b2a1997a4e8e68a45e1f065f5d7daceed81e7"}]},"branch":"refs/heads/master"},"39426655064aa85eef158275f64b024f9aa01aca":{"kind":"TRIVIAL_REBASE","_number":3,"created":"2021-02-03 19:56:43.000000000","uploader":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"ref":"refs/changes/39/773939/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/3"}}},"commit":{"parents":[{"commit":"0a2309dae39dd0d99bd2f422d565ddcc91c28b16","subject":"Updating lower-constraints job as non voting","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0a2309dae39dd0d99bd2f422d565ddcc91c28b16"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 15:26:45.000000000","tz":60},"committer":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 19:56:26.000000000","tz":60},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand removing option, so it will be used by default.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections:\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix and test it ?\n\nRepeat above:\n - with memcache_use_advanced_pool\u003dFalse\n   OR\n - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/39426655064aa85eef158275f64b024f9aa01aca"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/39426655064aa85eef158275f64b024f9aa01aca"}]},"branch":"refs/heads/master"},"e33a9ccda8f5b011e65b78884fed3b08c31b9270":{"kind":"NO_CODE_CHANGE","_number":4,"created":"2021-02-03 21:46:16.000000000","uploader":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"ref":"refs/changes/39/773939/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/4"}}},"commit":{"parents":[{"commit":"0a2309dae39dd0d99bd2f422d565ddcc91c28b16","subject":"Updating lower-constraints job as non voting","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0a2309dae39dd0d99bd2f422d565ddcc91c28b16"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 15:26:45.000000000","tz":60},"committer":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 21:46:03.000000000","tz":60},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand removing option, so it will be used by default.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections (which will grow up):\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix and test it ?\n\nRepeat above, to fix:\n - with memcache_use_advanced_pool\u003dTrue\n   OR\n - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/e33a9ccda8f5b011e65b78884fed3b08c31b9270"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/e33a9ccda8f5b011e65b78884fed3b08c31b9270"}]},"branch":"refs/heads/master"},"9df3ff3e0f630762609b900212c0bd8a25daf8ae":{"kind":"REWORK","_number":5,"created":"2021-02-04 06:04:23.000000000","uploader":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"ref":"refs/changes/39/773939/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/5"}}},"commit":{"parents":[{"commit":"0a2309dae39dd0d99bd2f422d565ddcc91c28b16","subject":"Updating lower-constraints job as non voting","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0a2309dae39dd0d99bd2f422d565ddcc91c28b16"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-03 15:26:45.000000000","tz":60},"committer":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-04 06:03:03.000000000","tz":60},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand removing option, so it will be used by default.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections (which will grow up):\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix and test it ?\n\nRepeat above, to fix:\n - with memcache_use_advanced_pool\u003dTrue\n   OR\n - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/9df3ff3e0f630762609b900212c0bd8a25daf8ae"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/9df3ff3e0f630762609b900212c0bd8a25daf8ae"}]},"branch":"refs/heads/master"},"af4b2206bb9b7b949f7ba193c1c8bfaf6de7944a":{"kind":"REWORK","_number":6,"created":"2021-02-10 13:50:47.000000000","uploader":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"ref":"refs/changes/39/773939/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/6"}}},"commit":{"parents":[{"commit":"0a2309dae39dd0d99bd2f422d565ddcc91c28b16","subject":"Updating lower-constraints job as non voting","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0a2309dae39dd0d99bd2f422d565ddcc91c28b16"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-10 13:47:17.000000000","tz":60},"committer":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-10 13:47:17.000000000","tz":60},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand adding deprecation warning about eventlet-unsafe variant\nof keystonemiddleware\u0027s memcache pool.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections (which will grow up):\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix and test it ?\n\nRepeat above, to fix:\n - with memcache_use_advanced_pool\u003dTrue\n   OR\n - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\n\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/af4b2206bb9b7b949f7ba193c1c8bfaf6de7944a"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/af4b2206bb9b7b949f7ba193c1c8bfaf6de7944a"}]},"branch":"refs/heads/master"},"7b0d79569b3376e33fcb2a07e715b14a373c120d":{"kind":"REWORK","_number":7,"created":"2021-02-11 13:34:09.000000000","uploader":{"_account_id":27339,"name":"Michal Arbet","email":"michal.arbet@ultimum.io","username":"michalarbet"},"ref":"refs/changes/39/773939/7","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/7","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/7"}}},"commit":{"parents":[{"commit":"0a2309dae39dd0d99bd2f422d565ddcc91c28b16","subject":"Updating lower-constraints job as non voting","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0a2309dae39dd0d99bd2f422d565ddcc91c28b16"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-10 13:47:17.000000000","tz":60},"committer":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-11 13:33:40.000000000","tz":60},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand adding deprecation warning about eventlet-unsafe variant\nof keystonemiddleware\u0027s memcache pool.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections (which will grow up):\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix and test it ?\n\nRepeat above, to fix:\n - with memcache_use_advanced_pool\u003dTrue\n   OR\n - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\nCloses-Bug: #1888394\n\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/7b0d79569b3376e33fcb2a07e715b14a373c120d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/7b0d79569b3376e33fcb2a07e715b14a373c120d"}]},"branch":"refs/heads/master"},"788d3c4969e3446778496f3a9055f654602ae2c1":{"kind":"REWORK","_number":8,"created":"2021-02-11 14:36:33.000000000","uploader":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"ref":"refs/changes/39/773939/8","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/keystonemiddleware","ref":"refs/changes/39/773939/8","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/keystonemiddleware refs/changes/39/773939/8"}}},"commit":{"parents":[{"commit":"0a2309dae39dd0d99bd2f422d565ddcc91c28b16","subject":"Updating lower-constraints job as non voting","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/0a2309dae39dd0d99bd2f422d565ddcc91c28b16"}]}],"author":{"name":"Michal Arbet","email":"michal.arbet@ultimum.io","date":"2021-02-10 13:47:17.000000000","tz":60},"committer":{"name":"Stephen Finucane","email":"stephenfin@redhat.com","date":"2021-02-11 14:36:25.000000000","tz":0},"subject":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool","message":"Switch to eventlet-safe oslo.cache\u0027s MemcacheClientPool\n\nIn past days there were discussions about various issues\nwith memcached connections [1][2][3].\n\nAfter investigation it looks like common root cause for above\nproblems is keystonemiddleware. More precisely said the way\nhow keystonemiddleware is caching tokens.\n\nCurrently it\u0027s using some home-made CachePool with direct\nusage of memcached library, moreover it looks like its\napproach is not eventlet-safe.\nDiscussion can be mainly found in [4].\n\nFortunately keystonemiddleware can use \"advanced cache pool\",\nwhich is oslo.cache\u0027s implementation and was added long time ago [5],\nbut it is turned on only if memcache_use_advanced_pool\u003dTrue.\n\nThis patch is switching to more elaborated oslo.cache CachePool\nand adding deprecation warning about eventlet-unsafe variant\nof keystonemiddleware\u0027s memcache pool.\n\nHow to reproduce ?\n\nwith memcache_use_advanced_pool\u003dFalse\n\n1. Build clean ENV of openstack\n2. Deploy core projects (keystone,glance,nova,placement...)\n3. Run while true; do COMMAND FOR SERVICE; done\n   - several bashes, in parallel (5-7)\n\nCOMMAND FOR SERVICE:\n- openstack network list\n- openstack volume list\n- openstack server list\n- openstack image list\n\n4. Check memcached connections (which will grow up):\n    - ss | grep 11211 | wc -l   every second\n\nHow to fix and test it ?\n\nRepeat above, to fix:\n - with memcache_use_advanced_pool\u003dTrue\n   OR\n - apply this patch\n\nCompare measurements in graph.\n\n[1] https://bugs.launchpad.net/keystonemiddleware/+bug/1892852\n[2] https://bugs.launchpad.net/oslo.cache/+bug/1888394\n[3] https://bugs.launchpad.net/keystonemiddleware/+bug/1883659\n\n[4] https://review.opendev.org/c/openstack/oslo.cache/+/742193\n\n[5] https://review.opendev.org/c/openstack/keystonemiddleware/+/268664\n\nCloses-Bug: #1883659\nCloses-Bug: #1892852\nCloses-Bug: #1888394\n\nChange-Id: I0e96334b65a0bf369ebf1d88651d13feb8d2ecac\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/788d3c4969e3446778496f3a9055f654602ae2c1"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/keystonemiddleware/commit/788d3c4969e3446778496f3a9055f654602ae2c1"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"status":"CLOSED","labels":[{"label":"Verified","status":"OK","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"OK","applied_by":{"_account_id":5046,"name":"Lance Bragstad","email":"lbragstad@redhat.com","username":"ldbragst"}},{"label":"Workflow","status":"OK","applied_by":{"_account_id":8482,"name":"Colleen Murphy","email":"colleen@gazlene.net","username":"krinkle"}}]}],"submit_requirements":[{"name":"Verified","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"label:Verified\u003dCustom-Rule","fulfilled":true,"status":"PASS","passing_atoms":["label:Verified\u003dCustom-Rule"],"failing_atoms":[],"atom_explanations":{}}},{"name":"Workflow","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"label:Workflow\u003dCustom-Rule","fulfilled":true,"status":"PASS","passing_atoms":["label:Workflow\u003dCustom-Rule"],"failing_atoms":[],"atom_explanations":{}}},{"name":"Code-Review","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"label:Code-Review\u003dCustom-Rule","fulfilled":true,"status":"PASS","passing_atoms":["label:Code-Review\u003dCustom-Rule"],"failing_atoms":[],"atom_explanations":{}}}]}