)]}'
{"id":"opendev%2Fpuppet-log_processor~759492","triplet_id":"opendev%2Fpuppet-log_processor~master~I6d32036566834da75f3a73f2d086475ef3431165","project":"opendev/puppet-log_processor","branch":"master","topic":"stream-log-files","hashtags":[],"change_id":"I6d32036566834da75f3a73f2d086475ef3431165","subject":"Stream log files instead of loading full files into memory","status":"MERGED","created":"2020-10-23 17:31:23.000000000","updated":"2020-11-09 18:50:22.000000000","submitted":"2020-11-09 18:50:22.000000000","submitter":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"total_comment_count":4,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"759492-1604947822630-40a8ff94","meta_rev_id":"277b4384ddf6720cc7170ec93bb50f430da56644","_number":759492,"virtual_id_number":759492,"owner":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"actions":{},"labels":{"Verified":{"approved":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},{"value":0,"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},{"tag":"autogenerated:zuul:gate","value":2,"date":"2020-11-09 18:50:22.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":0,"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","default_value":0,"optional":true},"Code-Review":{"approved":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"all":[{"value":0,"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},{"value":0,"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":2,"date":"2020-11-09 17:50:21.000000000","_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"approved":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"all":[{"value":0,"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},{"value":0,"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},{"value":0,"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},{"value":1,"date":"2020-11-09 17:50:21.000000000","_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-11-09 17:25:40.000000000","updated_by":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"reviewer":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"state":"REVIEWER"},{"updated":"2020-11-09 17:50:21.000000000","updated_by":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"reviewer":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"state":"REVIEWER"},{"updated":"2020-11-09 18:50:22.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"2f7f514327b60e008957b448f5bf3a679ff5b7d7","author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"date":"2020-10-23 17:31:23.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"d2d1c087f4109a390aba02bb91d6ac0e446576a8","author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"date":"2020-10-23 17:35:09.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"be849550304c1d7083c1fd2eb624abde0ae2dfd7","author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"date":"2020-10-23 17:46:03.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"a07a778ca8adb9643add04f736a8dc83c0432cc9","author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"date":"2020-10-23 17:56:40.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"a4133ac51f2926d87d805ec2320ca338700937a9","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-23 17:56:45.000000000","message":"Patch Set 1: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- legacy-puppet-lint https://zuul.opendev.org/t/openstack/build/bf31e6704f494a9ea38736f69c7abbf1 : FAILURE in 8m 01s\n- system-config-puppet-apply-4-ubuntu-xenial https://zuul.opendev.org/t/openstack/build/9e831b495bcb489c92c28075c22050d6 : SUCCESS in 17m 10s\n- system-config-puppet-beaker-rspec-puppet-4-infra https://zuul.opendev.org/t/openstack/build/4d020fdb8b0e40a1804971c5e4eb0d85 : SUCCESS in 17m 11s","accounts_in_message":[],"_revision_number":1},{"id":"06d8bcf8feeed9e6c46a9db7ed40aac05bd317f0","author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"date":"2020-10-23 18:13:35.000000000","message":"Uploaded patch set 2: Patch Set 1 was rebased.","accounts_in_message":[],"_revision_number":2},{"id":"a2dd5eb0e9a6ccd6a6a6af8811dc1c0132272005","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-23 18:39:00.000000000","message":"Patch Set 2: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- legacy-puppet-lint https://zuul.opendev.org/t/openstack/build/baae290d7b6c41cf8842d0385950d05a : SUCCESS in 7m 04s\n- system-config-puppet-apply-4-ubuntu-xenial https://zuul.opendev.org/t/openstack/build/4ec4ccacfc0242e380ef733669de7246 : SUCCESS in 14m 56s\n- system-config-puppet-beaker-rspec-puppet-4-infra https://zuul.opendev.org/t/openstack/build/30739ff704984b948083fbb3e5ba16b6 : FAILURE in 14m 29s","accounts_in_message":[],"_revision_number":2},{"id":"12eeb5b4d0739d400851d254aa544f1b6dcb1afd","author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"date":"2020-10-23 18:52:10.000000000","message":"Patch Set 2:\n\nrecheck since the beaker-rspec failure looks like a connectivity problem reaching rubygems.org","accounts_in_message":[],"_revision_number":2},{"id":"958116b5c2ab03e117701b4e003c564a028499d2","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-10-23 19:24:20.000000000","message":"Patch Set 2: Verified+1\n\nBuild succeeded (check pipeline).\n\n- legacy-puppet-lint https://zuul.opendev.org/t/openstack/build/9d9b333bb4c148eba58ea2fb8c927695 : SUCCESS in 7m 49s\n- system-config-puppet-apply-4-ubuntu-xenial https://zuul.opendev.org/t/openstack/build/6918f0152f4e4480a3f4dbd1e899909c : SUCCESS in 22m 30s\n- system-config-puppet-beaker-rspec-puppet-4-infra https://zuul.opendev.org/t/openstack/build/0ccd9f368c834a0da79fd202fd50f703 : SUCCESS in 22m 50s","accounts_in_message":[],"_revision_number":2},{"id":"de727f098cb9f3b8721d714d77df0b9859c27110","author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"date":"2020-10-23 20:44:57.000000000","message":"Patch Set 2: Code-Review+2","accounts_in_message":[],"_revision_number":2},{"id":"3a2816da829e6d634dd5cb79dc19f48936a65f95","author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"date":"2020-10-24 07:16:57.000000000","message":"Patch Set 2:\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"e6e197e32f22269b0fd0bc29b415a8f4af2f86b8","author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"date":"2020-11-09 16:35:20.000000000","message":"Patch Set 2: Code-Review-1\n\nPython requests is not installed on these worker nodes so we will need to do a pip install in https://opendev.org/opendev/puppet-log_processor/src/branch/master/manifests/worker.pp\n\nI can help you with that, just taking notes here as I work through getting this deployed on a server for testing. Maybe hold off with any updates until I\u0027ve got it running for further feedback.","accounts_in_message":[],"_revision_number":2},{"id":"978a09abb67702b23db56365dda001c1562aca27","author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"date":"2020-11-09 17:24:11.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"4706dd2c1b197db507a43847e8c8efcaae006bb3","author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"date":"2020-11-09 17:25:40.000000000","message":"Patch Set 3: Code-Review+2","accounts_in_message":[],"_revision_number":3},{"id":"22f825d7ae2e3aa6af5946827002343d6ec94541","author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"date":"2020-11-09 17:36:12.000000000","message":"Patch Set 3: Code-Review+2 Workflow+1","accounts_in_message":[],"_revision_number":3},{"id":"0c877efd3b6865fff503ac78583a3b0b1927da57","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-09 17:45:57.000000000","message":"Patch Set 3: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- legacy-puppet-lint https://zuul.opendev.org/t/openstack/build/c5ae929170444e3890e5223b04655266 : SUCCESS in 8m 48s\n- system-config-puppet-apply-4-ubuntu-xenial https://zuul.opendev.org/t/openstack/build/f707878a4c5b401e891b01ae80681da9 : FAILURE in 13m 39s\n- system-config-puppet-beaker-rspec-puppet-4-infra https://zuul.opendev.org/t/openstack/build/c08e7f4c603b4d9ea9cf5d8163a09589 : SUCCESS in 12m 21s","accounts_in_message":[],"_revision_number":3},{"id":"eb1a865356957e319ebcb48cf698cfae02dc01b7","author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"date":"2020-11-09 17:49:13.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"3183325d8d2e8a2a0cfa0e57d7294c0b336d7605","author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"date":"2020-11-09 17:50:21.000000000","message":"Patch Set 4: Code-Review+2 Workflow+1","accounts_in_message":[],"_revision_number":4},{"id":"764d9f2779e3438e512f341cca90811ba4a63736","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-09 18:26:03.000000000","message":"Patch Set 4: Verified+1\n\nBuild succeeded (check pipeline).\n\n- legacy-puppet-lint https://zuul.opendev.org/t/openstack/build/7a85704acc1d4e86bed5c5128733083f : SUCCESS in 6m 22s\n- system-config-puppet-apply-4-ubuntu-xenial https://zuul.opendev.org/t/openstack/build/9e8e59aed7504a4589967c1349f28f76 : SUCCESS in 16m 46s\n- system-config-puppet-beaker-rspec-puppet-4-infra https://zuul.opendev.org/t/openstack/build/f6d9c8b3f2a9480cac86db5c2f9eaa39 : SUCCESS in 17m 12s","accounts_in_message":[],"_revision_number":4},{"id":"1219f59164cf6f562b5090cf3b96fc898f0d104c","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-09 18:26:25.000000000","message":"Patch Set 4: -Verified\n\nStarting gate jobs.","accounts_in_message":[],"_revision_number":4},{"id":"5deba368e8dbae13172e9bde5bc51c5e8b2f65be","tag":"autogenerated:zuul:gate","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-09 18:50:22.000000000","message":"Patch Set 4: Verified+2\n\nBuild succeeded (gate pipeline).\n\n- legacy-puppet-lint https://zuul.opendev.org/t/openstack/build/01bce0a3457f47e386e137f9ee82ac6c : SUCCESS in 6m 32s\n- system-config-puppet-apply-4-ubuntu-xenial https://zuul.opendev.org/t/openstack/build/2878ae6e68b34bb19219614c1a975959 : SUCCESS in 14m 04s\n- system-config-puppet-beaker-rspec-puppet-4-infra https://zuul.opendev.org/t/openstack/build/366592f4e5474448a797d1d1604eef36 : SUCCESS in 13m 28s","accounts_in_message":[],"_revision_number":4},{"id":"7c140806d0edfd315ba73d5aadcb9716448a3178","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2020-11-09 18:50:22.000000000","message":"Change has been successfully merged by Zuul","accounts_in_message":[],"_revision_number":4}],"current_revision_number":4,"current_revision":"89bfe00dda0b9761bd79b0aa1ac2092940f0f11d","revisions":{"06d3cb4c80863939248fd111b4e63fa0dadd65bc":{"kind":"REWORK","_number":1,"created":"2020-10-23 17:31:23.000000000","uploader":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"ref":"refs/changes/92/759492/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/opendev/puppet-log_processor","ref":"refs/changes/92/759492/1","commands":{"Checkout":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/1"}}},"commit":{"parents":[{"commit":"09b0ed74f77ebff022d6292f4a9544a0346e39de","subject":"Merge \"Handle case where content encoding isn\u0027t set\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/09b0ed74f77ebff022d6292f4a9544a0346e39de"}]}],"author":{"name":"melanie witt","email":"melwittt@gmail.com","date":"2020-10-22 17:18:32.000000000","tz":0},"committer":{"name":"melanie witt","email":"melwittt@gmail.com","date":"2020-10-23 17:23:41.000000000","tz":0},"subject":"Stream log files instead of loading full files into memory","message":"Stream log files instead of loading full files into memory\n\nFor awhile now lately, we have been seeing Elastic Search indexing\nquickly fall behind as some log files generated in the gate have become\nlarger. Currently, we download a full log file into memory and then\nemit it line-by-line to be received by a logstash listener. When log\nfiles are large (example: 40M) logstash gets bogged down processing\nthem.\n\nInstead of downloading full files into memory, we can stream the files\nand emit their lines on-the-fly to try to alleviate load on the log\nprocessor.\n\nThis:\n\n  * Replaces use of urllib2.urlopen with requests with stream\u003dTrue\n  * Removes manual decoding of gzip and deflate compression\n    formats as these are decoded automatically by requests.iter_lines\n  * Removes unrelated unused imports\n  * Removes an unused arg \u0027retry\u0027 from the log retrieval method\n\nChange-Id: I6d32036566834da75f3a73f2d086475ef3431165\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/06d3cb4c80863939248fd111b4e63fa0dadd65bc"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/06d3cb4c80863939248fd111b4e63fa0dadd65bc"}]},"branch":"refs/heads/master"},"29deb29a26860f434da61ea79a7db42ea5aa2a8b":{"kind":"TRIVIAL_REBASE","_number":2,"created":"2020-10-23 18:13:35.000000000","uploader":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"ref":"refs/changes/92/759492/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/opendev/puppet-log_processor","ref":"refs/changes/92/759492/2","commands":{"Checkout":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/2"}}},"commit":{"parents":[{"commit":"a9da3284365f48ddcf4f53ee1381db0afda387cf","subject":"Fix puppet linter complaints about :: prefixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/a9da3284365f48ddcf4f53ee1381db0afda387cf"}]}],"author":{"name":"melanie witt","email":"melwittt@gmail.com","date":"2020-10-22 17:18:32.000000000","tz":0},"committer":{"name":"Clark Boylan","email":"clark.boylan@gmail.com","date":"2020-10-23 18:13:27.000000000","tz":-420},"subject":"Stream log files instead of loading full files into memory","message":"Stream log files instead of loading full files into memory\n\nFor awhile now lately, we have been seeing Elastic Search indexing\nquickly fall behind as some log files generated in the gate have become\nlarger. Currently, we download a full log file into memory and then\nemit it line-by-line to be received by a logstash listener. When log\nfiles are large (example: 40M) logstash gets bogged down processing\nthem.\n\nInstead of downloading full files into memory, we can stream the files\nand emit their lines on-the-fly to try to alleviate load on the log\nprocessor.\n\nThis:\n\n  * Replaces use of urllib2.urlopen with requests with stream\u003dTrue\n  * Removes manual decoding of gzip and deflate compression\n    formats as these are decoded automatically by requests.iter_lines\n  * Removes unrelated unused imports\n  * Removes an unused arg \u0027retry\u0027 from the log retrieval method\n\nChange-Id: I6d32036566834da75f3a73f2d086475ef3431165\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/29deb29a26860f434da61ea79a7db42ea5aa2a8b"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/29deb29a26860f434da61ea79a7db42ea5aa2a8b"}]},"branch":"refs/heads/master"},"ab55e331a15e2f2528c097fd3e8d0d1d3373ac85":{"kind":"REWORK","_number":3,"created":"2020-11-09 17:24:11.000000000","uploader":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"ref":"refs/changes/92/759492/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/opendev/puppet-log_processor","ref":"refs/changes/92/759492/3","commands":{"Checkout":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/3"}}},"commit":{"parents":[{"commit":"a9da3284365f48ddcf4f53ee1381db0afda387cf","subject":"Fix puppet linter complaints about :: prefixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/a9da3284365f48ddcf4f53ee1381db0afda387cf"}]}],"author":{"name":"melanie witt","email":"melwittt@gmail.com","date":"2020-10-22 17:18:32.000000000","tz":0},"committer":{"name":"Clark Boylan","email":"clark.boylan@gmail.com","date":"2020-11-09 17:24:02.000000000","tz":-480},"subject":"Stream log files instead of loading full files into memory","message":"Stream log files instead of loading full files into memory\n\nFor awhile now lately, we have been seeing Elastic Search indexing\nquickly fall behind as some log files generated in the gate have become\nlarger. Currently, we download a full log file into memory and then\nemit it line-by-line to be received by a logstash listener. When log\nfiles are large (example: 40M) logstash gets bogged down processing\nthem.\n\nInstead of downloading full files into memory, we can stream the files\nand emit their lines on-the-fly to try to alleviate load on the log\nprocessor.\n\nThis:\n\n  * Replaces use of urllib2.urlopen with requests with stream\u003dTrue\n  * Removes manual decoding of gzip and deflate compression\n    formats as these are decoded automatically by requests.iter_lines\n  * Removes unrelated unused imports\n  * Removes an unused arg \u0027retry\u0027 from the log retrieval method\n\nChange-Id: I6d32036566834da75f3a73f2d086475ef3431165\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/ab55e331a15e2f2528c097fd3e8d0d1d3373ac85"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/ab55e331a15e2f2528c097fd3e8d0d1d3373ac85"}]},"branch":"refs/heads/master"},"89bfe00dda0b9761bd79b0aa1ac2092940f0f11d":{"kind":"REWORK","_number":4,"created":"2020-11-09 17:49:13.000000000","uploader":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"ref":"refs/changes/92/759492/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/opendev/puppet-log_processor","ref":"refs/changes/92/759492/4","commands":{"Checkout":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/opendev/puppet-log_processor refs/changes/92/759492/4"}}},"commit":{"parents":[{"commit":"a9da3284365f48ddcf4f53ee1381db0afda387cf","subject":"Fix puppet linter complaints about :: prefixes","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/a9da3284365f48ddcf4f53ee1381db0afda387cf"}]}],"author":{"name":"melanie witt","email":"melwittt@gmail.com","date":"2020-10-22 17:18:32.000000000","tz":0},"committer":{"name":"Clark Boylan","email":"clark.boylan@gmail.com","date":"2020-11-09 17:49:07.000000000","tz":-480},"subject":"Stream log files instead of loading full files into memory","message":"Stream log files instead of loading full files into memory\n\nFor awhile now lately, we have been seeing Elastic Search indexing\nquickly fall behind as some log files generated in the gate have become\nlarger. Currently, we download a full log file into memory and then\nemit it line-by-line to be received by a logstash listener. When log\nfiles are large (example: 40M) logstash gets bogged down processing\nthem.\n\nInstead of downloading full files into memory, we can stream the files\nand emit their lines on-the-fly to try to alleviate load on the log\nprocessor.\n\nThis:\n\n  * Replaces use of urllib2.urlopen with requests with stream\u003dTrue\n  * Removes manual decoding of gzip and deflate compression\n    formats as these are decoded automatically by requests.iter_lines\n  * Removes unrelated unused imports\n  * Removes an unused arg \u0027retry\u0027 from the log retrieval method\n\nChange-Id: I6d32036566834da75f3a73f2d086475ef3431165\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/89bfe00dda0b9761bd79b0aa1ac2092940f0f11d"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/opendev/puppet-log_processor/commit/89bfe00dda0b9761bd79b0aa1ac2092940f0f11d"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}