)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":24162,"name":"Sorin Sbârnea","display_name":"zbr","email":"ssbarnea@redhat.com","username":"ssbarnea","status":"do not feed the troll 🥕"},"change_message_id":"681b721596759121a0dbe26f65017cf353d7141c","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When collecting logs of many nodes and storing them on"},{"line_number":10,"context_line":"the primary nodepool node or slave it makes sense"},{"line_number":11,"context_line":"to compress the files, tar and send. Pretty standard"},{"line_number":12,"context_line":"logic there.   In upstream the default remains not"},{"line_number":13,"context_line":"to compress prior to tar or prior to publish. There"},{"line_number":14,"context_line":"is a sunk time cost in compressing and uncompressing."},{"line_number":15,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"3fa7e38b_51f7cad2","line":12,"range":{"start_line":11,"start_character":37,"end_line":12,"end_character":12},"updated":"2020-02-18 21:35:31.000000000","message":"Interesting affirmation, can it be backed with a reference? I think that I almost always seen tar|gzip approach and almost never seen gzip before.","commit_id":"e2bccc1bb12ddebc0b34edf48f5b56ea5533dc5f"},{"author":{"_account_id":24162,"name":"Sorin Sbârnea","display_name":"zbr","email":"ssbarnea@redhat.com","username":"ssbarnea","status":"do not feed the troll 🥕"},"change_message_id":"681b721596759121a0dbe26f65017cf353d7141c","unresolved":false,"context_lines":[{"line_number":10,"context_line":"the primary nodepool node or slave it makes sense"},{"line_number":11,"context_line":"to compress the files, tar and send. Pretty standard"},{"line_number":12,"context_line":"logic there.   In upstream the default remains not"},{"line_number":13,"context_line":"to compress prior to tar or prior to publish. There"},{"line_number":14,"context_line":"is a sunk time cost in compressing and uncompressing."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Note: this removes the optional logic to compress"},{"line_number":17,"context_line":"based on file size as it make more sense to compress"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"3fa7e38b_d1241a62","line":14,"range":{"start_line":13,"start_character":45,"end_line":14,"end_character":52},"updated":"2020-02-18 21:35:31.000000000","message":"That is very true when not using gzip --fast.","commit_id":"e2bccc1bb12ddebc0b34edf48f5b56ea5533dc5f"},{"author":{"_account_id":24162,"name":"Sorin Sbârnea","display_name":"zbr","email":"ssbarnea@redhat.com","username":"ssbarnea","status":"do not feed the troll 🥕"},"change_message_id":"681b721596759121a0dbe26f65017cf353d7141c","unresolved":false,"context_lines":[{"line_number":13,"context_line":"to compress prior to tar or prior to publish. There"},{"line_number":14,"context_line":"is a sunk time cost in compressing and uncompressing."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Note: this removes the optional logic to compress"},{"line_number":17,"context_line":"based on file size as it make more sense to compress"},{"line_number":18,"context_line":"all the files prior to the tar and xfer to the"},{"line_number":19,"context_line":"primary node / slave for faster results."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Signed off by Wes Hayutin."},{"line_number":22,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"3fa7e38b_71de863e","line":19,"range":{"start_line":16,"start_character":0,"end_line":19,"end_character":40},"updated":"2020-02-18 21:35:31.000000000","message":"That change is now removing an official OpenStack recommandation related to logs: https://docs.openstack.org/infra/system-config/third_party.html\n\nWhy not setting the treshold to 0 to keep the feature and obtain the same outcome?","commit_id":"e2bccc1bb12ddebc0b34edf48f5b56ea5533dc5f"}],"tasks/collect.yml":[{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"fc8a2bd7989d8da12fb66937def9927966f97412","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  when: artcl_gzip | bool"},{"line_number":83,"context_line":"  shell: gzip -r ./{{ inventory_hostname }}"},{"line_number":84,"context_line":"  args:"},{"line_number":85,"context_line":"    chdir: /tmp"},{"line_number":86,"context_line":"    warn: false"},{"line_number":87,"context_line":"  changed_when: true"},{"line_number":88,"context_line":"  tags:"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3fa7e38b_5e7ffc67","line":85,"updated":"2020-02-17 08:20:11.000000000","message":"afaics this seems equivalent to what was happening before...\n\nhere you no longer do the rsync, instead directly do everything in /tmp and then move it to collect_dir\n\n[EDIT]: well that isn\u0027t entirely true you are changing the behaviour. You no longer consider file size instead zip all the things. Perhaps worth calling that out in the commit message.","commit_id":"96f53e26f3ab7f2fafcf905d1766edcd156fa8e6"},{"author":{"_account_id":24162,"name":"Sorin Sbârnea","display_name":"zbr","email":"ssbarnea@redhat.com","username":"ssbarnea","status":"do not feed the troll 🥕"},"change_message_id":"5cc13085180da3d7676c6d12ef8c128c63e4c884","unresolved":false,"context_lines":[{"line_number":88,"context_line":"  tags:"},{"line_number":89,"context_line":"    - skip_ansible_lint"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"- name: Create tar archive of logs for faster copying"},{"line_number":92,"context_line":"  shell:"},{"line_number":93,"context_line":"    cmd: tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }};"},{"line_number":94,"context_line":"    chdir: /tmp"},{"line_number":95,"context_line":"  changed_when: true"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"- name: Fetch log archive (tar)"},{"line_number":98,"context_line":"  fetch:"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3fa7e38b_8570ae11","line":95,"range":{"start_line":91,"start_character":0,"end_line":95,"end_character":20},"updated":"2020-02-18 10:29:11.000000000","message":"This risks running out of disk on remote machine, the only approach that I would find safe would be to use a `tar|gzip|ssh|ungzip` pipe, so we stream the data and never store it locally.","commit_id":"96f53e26f3ab7f2fafcf905d1766edcd156fa8e6"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"91a20fa235beaac71258269093981eb2f37529da","unresolved":false,"context_lines":[{"line_number":88,"context_line":"  tags:"},{"line_number":89,"context_line":"    - skip_ansible_lint"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"- name: Create tar archive of logs for faster copying"},{"line_number":92,"context_line":"  shell:"},{"line_number":93,"context_line":"    cmd: tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }};"},{"line_number":94,"context_line":"    chdir: /tmp"},{"line_number":95,"context_line":"  changed_when: true"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"- name: Fetch log archive (tar)"},{"line_number":98,"context_line":"  fetch:"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3fa7e38b_cb5a371e","line":95,"range":{"start_line":91,"start_character":0,"end_line":95,"end_character":20},"in_reply_to":"3fa7e38b_8570ae11","updated":"2020-02-18 11:53:57.000000000","message":"If there is so big risk of possible running out of disk, why the role copies all the logs to /tmp? their size is doubled .. if we can afford store the logs twice, i don\u0027t see a problem if we create another copy using tar as it will have only ~10% of the original size.","commit_id":"96f53e26f3ab7f2fafcf905d1766edcd156fa8e6"},{"author":{"_account_id":10969,"name":"Shnaidman Sagi (Sergey)","display_name":"Shnaidman Sagi","email":"sshnaidm@redhat.com","username":"sergsh"},"change_message_id":"8aca299884964e3917fe0998db6012794c216a00","unresolved":false,"context_lines":[{"line_number":88,"context_line":"  tags:"},{"line_number":89,"context_line":"    - skip_ansible_lint"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"- name: Create tar archive of logs for faster copying"},{"line_number":92,"context_line":"  shell:"},{"line_number":93,"context_line":"    cmd: tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }};"},{"line_number":94,"context_line":"    chdir: /tmp"},{"line_number":95,"context_line":"  changed_when: true"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"- name: Fetch log archive (tar)"},{"line_number":98,"context_line":"  fetch:"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3fa7e38b_cbd1d7fb","line":95,"range":{"start_line":91,"start_character":0,"end_line":95,"end_character":20},"in_reply_to":"3fa7e38b_8570ae11","updated":"2020-02-18 12:05:01.000000000","message":"We never hit the situation with out of space on remote machine, during years of running it. This risk could be ignored.","commit_id":"96f53e26f3ab7f2fafcf905d1766edcd156fa8e6"},{"author":{"_account_id":24162,"name":"Sorin Sbârnea","display_name":"zbr","email":"ssbarnea@redhat.com","username":"ssbarnea","status":"do not feed the troll 🥕"},"change_message_id":"3245db0c9ab026398e211f3214c65c9ce493679a","unresolved":false,"context_lines":[{"line_number":88,"context_line":"  tags:"},{"line_number":89,"context_line":"    - skip_ansible_lint"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"- name: Create tar archive of logs for faster copying"},{"line_number":92,"context_line":"  shell:"},{"line_number":93,"context_line":"    cmd: tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }};"},{"line_number":94,"context_line":"    chdir: /tmp"},{"line_number":95,"context_line":"  changed_when: true"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"- name: Fetch log archive (tar)"},{"line_number":98,"context_line":"  fetch:"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3fa7e38b_6baea381","line":95,"range":{"start_line":91,"start_character":0,"end_line":95,"end_character":20},"in_reply_to":"3fa7e38b_cb5a371e","updated":"2020-02-18 12:02:02.000000000","message":"I do find /tmp use as a bug which may need to be addressed too, but clearly not a valid excuse to double the disk requirements. If I remember well logs are copied with CoW or hardlinking to tmp, which should not increase disk space. Create a tar archive, does clearly need more space.\n\n\nI started a POC for streaming the logs at https://review.opendev.org/#/c/708362/ and I am waiting for the CI results, local testing was quite positive. Also it is mandatory to use --fast with gzip or we will risk slowing down the process due to gzip taking too much CPU. Still the streaming approach works only for ssh connections and may need some tunning.","commit_id":"96f53e26f3ab7f2fafcf905d1766edcd156fa8e6"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"cf087c0f93fed0ac0ab34e77e02b48041447e252","unresolved":false,"context_lines":[{"line_number":88,"context_line":"  tags:"},{"line_number":89,"context_line":"    - skip_ansible_lint"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"- name: Create tar archive of logs for faster copying"},{"line_number":92,"context_line":"  shell:"},{"line_number":93,"context_line":"    cmd: tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }};"},{"line_number":94,"context_line":"    chdir: /tmp"},{"line_number":95,"context_line":"  changed_when: true"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"- name: Fetch log archive (tar)"},{"line_number":98,"context_line":"  fetch:"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3fa7e38b_5dcfea25","line":95,"range":{"start_line":91,"start_character":0,"end_line":95,"end_character":20},"in_reply_to":"3fa7e38b_cbd1d7fb","updated":"2020-02-18 13:05:06.000000000","message":"what is usual node capacity and how much space is used by average logs? I bet there is much more space available than the size of archive which is again, approx 10% of the original size.\n\nEven if not ignored, put aside for sure. This discussions are pointlessly blocking the downstream initiative to reuse the role. So much to the big priority of reusing the tool.","commit_id":"96f53e26f3ab7f2fafcf905d1766edcd156fa8e6"},{"author":{"_account_id":3153,"name":"Emilien Macchi","email":"emilien@redhat.com","username":"emilienm"},"change_message_id":"9f990ca59e12a8635a36f2384b71a677d6f0bb0e","unresolved":false,"context_lines":[{"line_number":110,"context_line":"    warn: false"},{"line_number":111,"context_line":"  delegate_to: localhost"},{"line_number":112,"context_line":"  changed_when: true"},{"line_number":113,"context_line":"  "},{"line_number":114,"context_line":"- name: Archive big collected files"},{"line_number":115,"context_line":"  when: artcl_gzip | bool"},{"line_number":116,"context_line":"  run_once: true"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"3fa7e38b_a805776a","line":113,"updated":"2020-02-18 18:49:33.000000000","message":"whitespace","commit_id":"8658d32358ee5667c58a7344852ea47f5b7aa680"}]}