)]}'
{"id":"openstack%2Fswift~879425","triplet_id":"openstack%2Fswift~master~I32e70ee2d73b950849e293929841b1692139626d","project":"openstack/swift","branch":"master","topic":"timeout_emit_sysloghandler","attention_set":{"7233":{"account":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"last_update":"2023-04-04 10:00:00.000000000","reason":"A robot voted negatively on a label"}},"removed_from_attention_set":{},"hashtags":[],"change_id":"I32e70ee2d73b950849e293929841b1692139626d","subject":"DNM: Add an emit_timeout to proxy logging","status":"NEW","created":"2023-04-04 07:17:02.000000000","updated":"2023-04-04 14:49:15.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":6,"unresolved_comment_count":5,"has_review_started":true,"meta_rev_id":"98a66a8eb1646b4f7134a0eec96786dd8d28396a","_number":879425,"virtual_id_number":879425,"owner":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"actions":{},"labels":{"Verified":{"disliked":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:zuul:check","value":-1,"date":"2023-04-04 10:00:00.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":-1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"CC":[{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"}],"REVIEWER":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2023-04-04 07:36:31.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2023-04-04 10:00:00.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2023-04-04 14:49:15.000000000","updated_by":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"reviewer":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"state":"CC"}],"messages":[{"id":"41dfd6da91931955b9435174f7d70846b63ca613","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2023-04-04 07:17:02.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"e9c94636b18328d71aa61a7e6dff11969b81307e","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-04-04 07:36:31.000000000","message":"Patch Set 1:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/e76c6c505cf849db888bd5cc79c16ee7\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/0fddc556371649d7bf12c034ed765c07 : FAILURE in 18m 15s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/4deee7f81bc84c678d62fbfa0b8debcf : FAILURE in 15m 48s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/974ff99c13004cfcaaca84088c05759f : FAILURE in 3m 59s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/1e0d6cb775c84be0af2616bb1cc9742e : FAILURE in 3m 16s","accounts_in_message":[],"_revision_number":1},{"id":"1c3dce2ec4155d308155442f060fb8cdf74a0161","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2023-04-04 07:50:59.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"bcfc5c035347e57e946fe0c07ada011d3e664f32","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-04-04 08:11:22.000000000","message":"Patch Set 2:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/35e6cbef0f3b43299d0652924f6f7299\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/9a0190d5a3894996a8860f9e6f93d8c7 : FAILURE in 15m 04s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/e9e91afa47644d939800db7287bc9da0 : FAILURE in 15m 43s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/f36e0d44c1644a92860659ed16ed2dac : SUCCESS in 18m 45s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/9d2306e6490f4216a4b86d7199e02937 : SUCCESS in 18m 33s","accounts_in_message":[],"_revision_number":2},{"id":"8874a8a449a1f3ab457dbc623af9b9398ce7e9f1","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-04-04 10:00:00.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\nhttps://zuul.opendev.org/t/openstack/buildset/611ba26651bb447994c7479f99cf77e6\n\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/5cb38bfd78fa4468af4e9980b20c5889 : SUCCESS in 8m 57s\n- grenade https://zuul.opendev.org/t/openstack/build/cc72bfa26bb14ff0996b6a0ab3176bf7 : SUCCESS in 57m 54s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/1125974690214ec1a123fb9c9386979a : SUCCESS in 53m 08s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/d04ed509648249d4908824ae8eca04ae : SUCCESS in 53m 09s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/d5d6b0e1daed464da7a6c0f35ee7f53e : SUCCESS in 18m 58s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/c495f542f3604003b7122bbcb6c1f261 : SUCCESS in 17m 30s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/9f3d9f3900d441f0b30a208c6f4c60d1 : SUCCESS in 54m 07s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/e80ed566dcc646088b09c3993479d578 : SUCCESS in 8m 58s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/6b5277cb663a4c6a9b0fde7db42be868 : SUCCESS in 5m 31s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/09c388fda1fb410397ae7b1c024c209c : FAILURE in 42m 50s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/647e81ae19f44dea8907b8b64225e215 : FAILURE in 19m 50s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/a1113bf78fed46339d33fceb337a210b : FAILURE in 14m 59s\n- swift-tox-py310 https://zuul.opendev.org/t/openstack/build/55135950c13b4a3290e0d3da197270ae : FAILURE in 13m 44s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/4bfc511139934319a998a30b7a8981f4 : SUCCESS in 36m 53s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/b0450ec591ef4d31bcceaa1c2598ca3e : SUCCESS in 17m 06s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/cb3a4edcf9db485ba818b54fa7d749db : SUCCESS in 18m 10s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/3d4389156d0c406da554c39d30ce0a9e : SUCCESS in 12m 46s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/627eb7d053404f97a7a5f0535d7c6f77 : SUCCESS in 15m 07s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/03150d299c1649e2a05190d1686aaff9 : SUCCESS in 15m 47s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/0b350b37a75c47ea8cd5d8f69ceac792 : SUCCESS in 6m 48s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/581074d723ea4804a4182ca9c45caacc : SUCCESS in 17m 34s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/32068307162d4b928ab4176261f89d73 : SUCCESS in 7m 39s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/3ee91032db6b4b739120e4df14607380 : TIMED_OUT in 2h 00m 51s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/56b6f5c7e26b480cac0fa43fd05b12a9 : SUCCESS in 1h 13m 59s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/54a4cb3f99f241159bc0e33b7193eba7 : SUCCESS in 1h 18m 51s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/a4142e9c2709451098bf133ccd5e49b8 : SUCCESS in 48m 21s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/7756a94f86a64e55966ab56432957015 : FAILURE in 18m 32s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/a42db9128bdc4624a89677e100e462d5 : SUCCESS in 4m 46s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/110c1e7c03a344538d890c26cca5e0bf : SUCCESS in 23m 06s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/18fc5cdc86784059a65caa0cf61783da : SUCCESS in 47m 49s","accounts_in_message":[],"_revision_number":2},{"id":"98a66a8eb1646b4f7134a0eec96786dd8d28396a","author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"date":"2023-04-04 14:49:15.000000000","message":"Patch Set 2:\n\n(6 comments)","accounts_in_message":[],"_revision_number":2}],"current_revision_number":2,"current_revision":"c910f2696428ad95572b039a5eb680b012cdaad4","revisions":{"5075d14057edcf8ab89bddccbdde609e2fb40437":{"kind":"REWORK","_number":1,"created":"2023-04-04 07:17:02.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/25/879425/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/25/879425/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/25/879425/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/25/879425/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/25/879425/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/25/879425/1"}}},"commit":{"parents":[{"commit":"91d62b1879bffc8d5667b49c0207bd5a0f182379","subject":"Merge \"tests: Skip s3api functional tests when no s3api user configured\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/91d62b1879bffc8d5667b49c0207bd5a0f182379"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-04-04 06:08:56.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-04-04 07:16:36.000000000","tz":600},"subject":"DNM: Add an emit_timeout to proxy logging","message":"DNM: Add an emit_timeout to proxy logging\n\nWe\u0027ve noticed that sometimes a single proxy workers accept-queue gets\nfilled up. When we look at the process it\u0027s sitting there polling on the\n/dev/log unix socket.\n\nWhen we use xray to look at the running process, all green threads are\nstuck on the SyslogAdaptor emit method were it attempts to send a\nmessage through the domain socket, from SyslogAdapter.emit:\n\n  ...\n  if self.unixsocket:\n      try:\n          self.socket.send(msg)\n      except OSError:\n          self.socket.close()\n          self._connect_unixsocket(self.address)\n          self.socket.send(msg)\n\nIf we use gdb to close the socket, then The threads get unstuck. I\nassume this is because an OSError is thrown and the socket is\nreconnected (see the except clause above).\n\nStill haven\u0027t found the underlying cause, it could be 873744, which\nmesses with the sockets underlying settimeout during socket use, so\nmaybe there is a race?\nIf it is the cause then the eventlet fix the precipitated this landed\nso 873744 may not be required anymore.\n\nThis patch wraps the socket.send in a timeout that can be configured in\nproxy_logging as emit_timeout. When the value is \u003e0, a timeout is used\nwhich will raise an OSError causing the socket cleanup and reconnect.\n\nHopefully this will work as a workaround patch until we can find the\nunderlying cause.\n\nChange-Id: I32e70ee2d73b950849e293929841b1692139626d\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5075d14057edcf8ab89bddccbdde609e2fb40437"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5075d14057edcf8ab89bddccbdde609e2fb40437"}]},"branch":"refs/heads/master"},"c910f2696428ad95572b039a5eb680b012cdaad4":{"kind":"REWORK","_number":2,"created":"2023-04-04 07:50:59.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/25/879425/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/25/879425/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/25/879425/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/25/879425/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/25/879425/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/25/879425/2"}}},"commit":{"parents":[{"commit":"91d62b1879bffc8d5667b49c0207bd5a0f182379","subject":"Merge \"tests: Skip s3api functional tests when no s3api user configured\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/91d62b1879bffc8d5667b49c0207bd5a0f182379"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-04-04 06:08:56.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-04-04 07:50:47.000000000","tz":600},"subject":"DNM: Add an emit_timeout to proxy logging","message":"DNM: Add an emit_timeout to proxy logging\n\nWe\u0027ve noticed that sometimes a single proxy workers accept-queue gets\nfilled up. When we look at the process it\u0027s sitting there polling on the\n/dev/log unix socket.\n\nWhen we use xray to look at the running process, all green threads are\nstuck on the SyslogAdaptor emit method were it attempts to send a\nmessage through the domain socket, from SyslogAdapter.emit:\n\n  ...\n  if self.unixsocket:\n      try:\n          self.socket.send(msg)\n      except OSError:\n          self.socket.close()\n          self._connect_unixsocket(self.address)\n          self.socket.send(msg)\n\nIf we use gdb to close the socket, then The threads get unstuck. I\nassume this is because an OSError is thrown and the socket is\nreconnected (see the except clause above).\n\nStill haven\u0027t found the underlying cause, it could be 873744, which\nmesses with the sockets underlying settimeout during socket use, so\nmaybe there is a race?\nIf it is the cause then the eventlet fix the precipitated this landed\nso 873744 may not be required anymore.\n\nThis patch wraps the socket.send in a timeout that can be configured in\nproxy_logging as emit_timeout. When the value is \u003e0, a timeout is used\nwhich will raise an OSError causing the socket cleanup and reconnect.\n\nHopefully this will work as a workaround patch until we can find the\nunderlying cause.\n\nChange-Id: I32e70ee2d73b950849e293929841b1692139626d\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/c910f2696428ad95572b039a5eb680b012cdaad4"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/c910f2696428ad95572b039a5eb680b012cdaad4"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY"},{"label":"Workflow","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}