)]}'
{"id":"openstack%2Foslo.service~693237","triplet_id":"openstack%2Foslo.service~master~Iffcedff0a78b05804d17eae9eb933558dfe405a2","project":"openstack/oslo.service","branch":"master","topic":"looping-call-warn-threshold","hashtags":[],"change_id":"Iffcedff0a78b05804d17eae9eb933558dfe405a2","subject":"WIP: Add looping_call_delay_warn_threshold config option","status":"ABANDONED","created":"2019-11-06 19:32:34.000000000","updated":"2019-11-16 14:33:21.000000000","total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"3dac233649437c40c52506b267ab3996a660a0da","_number":693237,"virtual_id_number":693237,"owner":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"actions":{},"labels":{"Verified":{"disliked":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},{"tag":"autogenerated:zuul:check","value":-1,"date":"2019-11-06 20:40:44.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":-1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"rejected":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"all":[{"value":-1,"date":"2019-11-07 00:45:58.000000000","permitted_voting_range":{"min":-1,"max":0},"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true},"Backport-Candidate":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do Not Backport","-1":"Not A Backport Candidate"," 0":"Backport Review Needed","+1":"Proposed Backport","+2":"Should Backport"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2019-11-06 20:40:44.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"43954a5f2eb89feb3c55eb9cf0eef7437d37644a","author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"date":"2019-11-06 19:32:34.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"0afcdca4b5bfa46395512d51c92ddbfd801a5c34","author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"date":"2019-11-06 19:36:05.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"9f932ebaff294bd4ec2d26b62f602abba4919477","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2019-11-06 20:40:44.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttp://docs.openstack.org/infra/manual/developers.html#automated-testing\n\n\n- tempest-full https://zuul.opendev.org/t/openstack/build/ea0971f766484b07a631ae19a5aa7c9a : FAILURE in 1h 02m 57s\n- tempest-full-py3 https://zuul.opendev.org/t/openstack/build/d75550b054e845caa137d88cc76180bd : FAILURE in 57m 03s\n- openstack-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/a0e72bcaff914397a68a23edf1e41ad9 : SUCCESS in 4m 47s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/c69d4048eb0b42ecb5f379df54bb0508 : SUCCESS in 3m 51s\n- openstack-tox-py27 https://zuul.opendev.org/t/openstack/build/18794ba016f14d119015d5fa9de9dc6d : SUCCESS in 5m 31s\n- openstack-tox-py36 https://zuul.opendev.org/t/openstack/build/2eddc0d3da5c4f8894429749d486128b : SUCCESS in 4m 48s\n- openstack-tox-py37 https://zuul.opendev.org/t/openstack/build/1f3fca67bf2c48b09f8430d85e22aeee : SUCCESS in 4m 35s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/519052b8a680466e856d31384ba12483 : SUCCESS in 3m 53s","accounts_in_message":[],"_revision_number":2},{"id":"227370663e1e772b5338e9e288440e4667caa06c","author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"date":"2019-11-07 00:45:58.000000000","message":"Patch Set 2: Workflow-1\n\nneutron agent blows up with this since the options aren\u0027t registered","accounts_in_message":[],"_revision_number":2},{"id":"9188a4c609189861c3bc0a6a00e45fe50fb7c40f","author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"date":"2019-11-16 14:33:21.000000000","message":"Abandoned\n\nNot worth it.","accounts_in_message":[],"_revision_number":2}],"current_revision_number":2,"current_revision":"a2379f8ccc3558ac0e94a4f9f260ca0cc1edd674","revisions":{"d02918acdfa9debbd0f5dcff252ce1c12ad2e63c":{"kind":"REWORK","_number":1,"created":"2019-11-06 19:32:34.000000000","uploader":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"ref":"refs/changes/37/693237/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.service","ref":"refs/changes/37/693237/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/1"}}},"commit":{"parents":[{"commit":"a19c77a3ea6ff0ad8a378c6d6868195c67c017df","subject":"Merge \"tox: Keeping going with docs\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.service/commit/a19c77a3ea6ff0ad8a378c6d6868195c67c017df"}]}],"author":{"name":"Matt Riedemann","email":"mriedem.os@gmail.com","date":"2019-11-06 19:23:37.000000000","tz":-300},"committer":{"name":"Matt Riedemann","email":"mriedem.os@gmail.com","date":"2019-11-06 19:31:48.000000000","tz":-300},"subject":"WIP: Add looping_call_delay_warn_threshold config option","message":"WIP: Add looping_call_delay_warn_threshold config option\n\nToday any looping call that overruns its interval generates a warning.\nSometimes these are just a couple of seconds but we sometimes see\nnearly a minute in the gate [1]:\n\n  2019-11-05 23:12:57.389957 | ubuntu-bionic | 2019-11-05 23:11:23,400\n  WARNING [oslo.service.loopingcall] Function \u0027nova.servicegroup.drivers.\n  db.DbDriver._report_state\u0027 run outlasted interval by 59.77 sec\n\nIn that warning above, the looping call runs in a threadgroup by default\nevery 10 seconds. Having it take nearly a minute is a pretty good indication\nthat the node is overloaded.\n\nTrying to query on that kind of warning in logstash though isn\u0027t very\nhelpful because it also picks up all of the 1-2 second overrun warnings.\n\nThis change introduces a new option, looping_call_delay_warn_threshold,\nwhich by default is backward compatible with the existing looping call\nbehavior of logging a warning if there is any delay, but can be configured\nto (1) not log anything for delays or (2) set a threshold such that if\nthe delay is greater than or equal to that threshold, a warning is logged\n(like today) but if the delay is below the threshold then it\u0027s only logged\nat INFO.\n\nPractically speaking, in the gate we could set the threshold in devstack\nto 10 seconds and then anything going over that is logged at WARNING\nindicating a load problem on the node and we can query and track that\nusing logstash which also helps indicate which node providers are hitting\nthis issue.\n\nQuestions to answer before going further:\n\n- do we care about this? is it useful? obviously a real production\n  deployment should have sensors and alarms set up when node load\n  goes too high.\n- the global conf is naughty, is there a better way?\n- the option isn\u0027t specific to periodic tasks really, so while this\n  is in the [DEFAULT] group it shows up under periodic_tasks in the\n  docs, is that OK?\n- obviously tests and a release note would be needed\n\n[1] https://bugs.launchpad.net/nova/+bug/1844929/comments/16\n\nChange-Id: Iffcedff0a78b05804d17eae9eb933558dfe405a2\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.service/commit/d02918acdfa9debbd0f5dcff252ce1c12ad2e63c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.service/commit/d02918acdfa9debbd0f5dcff252ce1c12ad2e63c"}]},"branch":"refs/heads/master"},"a2379f8ccc3558ac0e94a4f9f260ca0cc1edd674":{"kind":"REWORK","_number":2,"created":"2019-11-06 19:36:05.000000000","uploader":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"ref":"refs/changes/37/693237/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/oslo.service","ref":"refs/changes/37/693237/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/oslo.service refs/changes/37/693237/2"}}},"commit":{"parents":[{"commit":"a19c77a3ea6ff0ad8a378c6d6868195c67c017df","subject":"Merge \"tox: Keeping going with docs\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.service/commit/a19c77a3ea6ff0ad8a378c6d6868195c67c017df"}]}],"author":{"name":"Matt Riedemann","email":"mriedem.os@gmail.com","date":"2019-11-06 19:23:37.000000000","tz":-300},"committer":{"name":"Matt Riedemann","email":"mriedem.os@gmail.com","date":"2019-11-06 19:35:59.000000000","tz":-300},"subject":"WIP: Add looping_call_delay_warn_threshold config option","message":"WIP: Add looping_call_delay_warn_threshold config option\n\nToday any looping call that overruns its interval generates a warning.\nSometimes these are just a couple of seconds but we sometimes see\nnearly a minute in the gate [1]:\n\n  2019-11-05 23:12:57.389957 | ubuntu-bionic | 2019-11-05 23:11:23,400\n  WARNING [oslo.service.loopingcall] Function \u0027nova.servicegroup.drivers.\n  db.DbDriver._report_state\u0027 run outlasted interval by 59.77 sec\n\nIn that warning above, the looping call runs in a threadgroup by default\nevery 10 seconds. Having it take nearly a minute is a pretty good indication\nthat the node is overloaded.\n\nTrying to query on that kind of warning in logstash though isn\u0027t very\nhelpful because it also picks up all of the 1-2 second overrun warnings.\n\nThis change introduces a new option, looping_call_delay_warn_threshold,\nwhich by default is backward compatible with the existing looping call\nbehavior of logging a warning if there is any delay, but can be configured\nto (1) not log anything for delays or (2) set a threshold such that if\nthe delay is greater than or equal to that threshold, a warning is logged\n(like today) but if the delay is below the threshold then it\u0027s only logged\nat INFO.\n\nPractically speaking, in the gate we could set the threshold in devstack\nto 10 seconds and then anything going over that is logged at WARNING\nindicating a load problem on the node and we can query and track that\nusing logstash which also helps indicate which node providers are hitting\nthis issue.\n\nQuestions to answer before going further:\n\n- do we care about this? is it useful? obviously a real production\n  deployment should have sensors and alarms set up when node load\n  goes too high.\n- the global conf is naughty, is there a better way?\n- the option isn\u0027t specific to periodic tasks really, so while this\n  is in the [DEFAULT] group it shows up under periodic_tasks in the\n  docs, is that OK?\n- obviously tests and a release note would be needed\n\n[1] https://bugs.launchpad.net/nova/+bug/1844929/comments/16\n\nChange-Id: Iffcedff0a78b05804d17eae9eb933558dfe405a2\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.service/commit/a2379f8ccc3558ac0e94a4f9f260ca0cc1edd674"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/oslo.service/commit/a2379f8ccc3558ac0e94a4f9f260ca0cc1edd674"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}