)]}'
{".zuul.yaml":[{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"8d869e59ec273aeae8dc90eb7412704cf7f34572","unresolved":true,"context_lines":[{"line_number":13,"context_line":"    run: ansible/playbooks/check-services-sender.yml"},{"line_number":14,"context_line":"    nodeset:"},{"line_number":15,"context_line":"      nodes:"},{"line_number":16,"context_line":"        - name: centos-8-stream"},{"line_number":17,"context_line":"          label: centos-8-stream"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"- project:"}],"source_content_type":"text/x-yaml","patch_set":26,"id":"1c271b66_213a9531","line":16,"range":{"start_line":16,"start_character":16,"end_line":16,"end_character":22},"updated":"2022-03-08 10:29:27.000000000","message":"maybe time to try out 9-stream? e.g. in tripleo we only have 9 for master now.","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":false,"context_lines":[{"line_number":13,"context_line":"    run: ansible/playbooks/check-services-sender.yml"},{"line_number":14,"context_line":"    nodeset:"},{"line_number":15,"context_line":"      nodes:"},{"line_number":16,"context_line":"        - name: centos-8-stream"},{"line_number":17,"context_line":"          label: centos-8-stream"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"- project:"}],"source_content_type":"text/x-yaml","patch_set":26,"id":"13c60317_f7d3a9b1","line":16,"range":{"start_line":16,"start_character":16,"end_line":16,"end_character":22},"in_reply_to":"1c271b66_213a9531","updated":"2022-03-08 14:08:18.000000000","message":"The logscraper node is with centos 8 stream, so I prefer to leave this nodeset.","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"7f15e971f7ff7d3571dd2b35606bdd079c6854ae","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"097a7872_36cfb8aa","updated":"2022-02-24 14:56:32.000000000","message":"How are we going to tell if the service performs fast enough? Should it features throughput metrics? For example, what is the backlog size, how many build received vs how many build uploaded.","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"f0759cc09ee688980ca8f3d1237cd38c95a69b0a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"18044786_a9923019","updated":"2022-02-25 08:23:02.000000000","message":"The logscraper changes has been moved to other PS: https://review.opendev.org/c/openstack/ci-log-processing/+/830941","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"342f3655d817c51a8b684517c7962532e85f773b","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"3c456c52_b34d9ec3","in_reply_to":"097a7872_36cfb8aa","updated":"2022-03-08 13:07:24.000000000","message":"Is this resolved?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"45fdd37d_3a40b110","in_reply_to":"3c456c52_b34d9ec3","updated":"2022-03-08 14:08:18.000000000","message":"You mean an information that \"found in directory 100 builds to send into Opensearch\" and on the end: \"all builds have been send\" ?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"b8161a90892c131a0ec87ec752e96d0f034a4b92","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"1e33da6e_8513d298","in_reply_to":"45fdd37d_3a40b110","updated":"2022-03-10 14:55:02.000000000","message":"Done","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"e698b1a6fd42681e021307e99cc1ad6014e383d9","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"b2fcf126_eda3430f","in_reply_to":"45fdd37d_3a40b110","updated":"2022-03-10 14:59:43.000000000","message":"I mean, these services could use counters to track the performance. For example, builds_queue_size, processed_builds, upload_speed, ... I think it\u0027s important and it may better to implement the metrics now instead of adding them latter because that may affect the current API. For example, downloading the file with multiprocessing may prevent sharing a global download_speed counter, and perhaps we should make the function returns the download speed ?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"55ca6d1a39e4ede8dfb62614c479a99bc8e522e6","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":9,"id":"2fcfbbf6_51fed00d","in_reply_to":"b2fcf126_eda3430f","updated":"2022-03-10 15:09:36.000000000","message":"good idea. Can I propose in other PS ?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"133533aefa660aa0ba7d07f35bac170f743ce054","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":19,"id":"13c82b3c_e71e32f1","updated":"2022-03-01 15:18:07.000000000","message":"recheck","commit_id":"527448537575796804a97a5532538b9887f43df6"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"7e51f01372ff21d41721fbd754baaec09d3b0507","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":25,"id":"704fc209_9cdc4046","updated":"2022-03-04 15:34:50.000000000","message":"recheck","commit_id":"3594a962fb7d524cdfc6395e50bbb5d34ed1828b"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"7bf095dc7fbc5d9ed73e986b0c27c08af9052467","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":25,"id":"a81c3570_6020dd19","updated":"2022-03-04 16:08:52.000000000","message":"recheck","commit_id":"3594a962fb7d524cdfc6395e50bbb5d34ed1828b"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"8d869e59ec273aeae8dc90eb7412704cf7f34572","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"087a1db0_db574e38","updated":"2022-03-08 10:29:27.000000000","message":"first pass\n\nthis is a lot of work and from first pass seems very well done (e.g. wrt the ansible itself) \n\nthis is a massive change ... I wonder if it is possible to split it\n\notherwise it is very hard for reviewers to approach and very easy to miss things.","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"}],"doc/source/logsender.rst":[{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":2,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"The logscraper tool is parsing log files that are available"},{"line_number":5,"context_line":"in the directory, attach important that are provided in `buildlog` and"},{"line_number":6,"context_line":"`inventory.yaml` files and send it directly to the Opensearch service."},{"line_number":7,"context_line":""},{"line_number":8,"context_line":"Available arguments for logsender are available on typing:"}],"source_content_type":"text/x-rst","patch_set":26,"id":"7d281315_939959fa","line":5,"updated":"2022-03-08 11:43:23.000000000","message":"important data","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":false,"context_lines":[{"line_number":2,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"The logscraper tool is parsing log files that are available"},{"line_number":5,"context_line":"in the directory, attach important that are provided in `buildlog` and"},{"line_number":6,"context_line":"`inventory.yaml` files and send it directly to the Opensearch service."},{"line_number":7,"context_line":""},{"line_number":8,"context_line":"Available arguments for logsender are available on typing:"}],"source_content_type":"text/x-rst","patch_set":26,"id":"4fbf9fec_35187efd","line":5,"in_reply_to":"7d281315_939959fa","updated":"2022-03-08 14:08:18.000000000","message":"Done","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":5,"context_line":"in the directory, attach important that are provided in `buildlog` and"},{"line_number":6,"context_line":"`inventory.yaml` files and send it directly to the Opensearch service."},{"line_number":7,"context_line":""},{"line_number":8,"context_line":"Available arguments for logsender are available on typing:"},{"line_number":9,"context_line":""},{"line_number":10,"context_line":".. code-block::"},{"line_number":11,"context_line":""}],"source_content_type":"text/x-rst","patch_set":26,"id":"2a06744f_9c9a2a3e","line":8,"updated":"2022-03-08 11:43:23.000000000","message":"Available arguments for logsender are:","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":false,"context_lines":[{"line_number":5,"context_line":"in the directory, attach important that are provided in `buildlog` and"},{"line_number":6,"context_line":"`inventory.yaml` files and send it directly to the Opensearch service."},{"line_number":7,"context_line":""},{"line_number":8,"context_line":"Available arguments for logsender are available on typing:"},{"line_number":9,"context_line":""},{"line_number":10,"context_line":".. code-block::"},{"line_number":11,"context_line":""}],"source_content_type":"text/x-rst","patch_set":26,"id":"3cae3599_3732d5d8","line":8,"in_reply_to":"2a06744f_9c9a2a3e","updated":"2022-03-08 14:08:18.000000000","message":"Done","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":20,"context_line":"     --host HOST           Opensearch host"},{"line_number":21,"context_line":"     --port PORT           Opensearch port"},{"line_number":22,"context_line":"     --username USERNAME   Opensearch username"},{"line_number":23,"context_line":"     --password PASSWORD   Opensearch user password"},{"line_number":24,"context_line":"     --index-prefix INDEX_PREFIX"},{"line_number":25,"context_line":"                           Prefix for the index. Defaults to logstash-"},{"line_number":26,"context_line":"     --index INDEX         Opensearch index. Defaults to: \u003cindex-prefix\u003e-YYYY-DD"}],"source_content_type":"text/x-rst","patch_set":26,"id":"7f96b7de_9fba6464","line":23,"updated":"2022-03-08 11:43:23.000000000","message":"password should not be on the command line. Use env var.","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"b8161a90892c131a0ec87ec752e96d0f034a4b92","unresolved":false,"context_lines":[{"line_number":20,"context_line":"     --host HOST           Opensearch host"},{"line_number":21,"context_line":"     --port PORT           Opensearch port"},{"line_number":22,"context_line":"     --username USERNAME   Opensearch username"},{"line_number":23,"context_line":"     --password PASSWORD   Opensearch user password"},{"line_number":24,"context_line":"     --index-prefix INDEX_PREFIX"},{"line_number":25,"context_line":"                           Prefix for the index. Defaults to logstash-"},{"line_number":26,"context_line":"     --index INDEX         Opensearch index. Defaults to: \u003cindex-prefix\u003e-YYYY-DD"}],"source_content_type":"text/x-rst","patch_set":26,"id":"586dd3e7_60c724cc","line":23,"in_reply_to":"0412128f_552c3cee","updated":"2022-03-10 14:55:02.000000000","message":"Done","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":true,"context_lines":[{"line_number":20,"context_line":"     --host HOST           Opensearch host"},{"line_number":21,"context_line":"     --port PORT           Opensearch port"},{"line_number":22,"context_line":"     --username USERNAME   Opensearch username"},{"line_number":23,"context_line":"     --password PASSWORD   Opensearch user password"},{"line_number":24,"context_line":"     --index-prefix INDEX_PREFIX"},{"line_number":25,"context_line":"                           Prefix for the index. Defaults to logstash-"},{"line_number":26,"context_line":"     --index INDEX         Opensearch index. Defaults to: \u003cindex-prefix\u003e-YYYY-DD"}],"source_content_type":"text/x-rst","patch_set":26,"id":"0412128f_552c3cee","line":23,"in_reply_to":"7f96b7de_9fba6464","updated":"2022-03-08 14:08:18.000000000","message":"but it is optional. You can disable security plugin in Opensearch, then the elasticsearch client will not work","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"}],"logscraper/logscraper.py":[{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":422,"context_line":"    logging.debug(\"Started fetching %s\" % url)"},{"line_number":423,"context_line":"    filename \u003d url.split(\"/\")[-1]"},{"line_number":424,"context_line":"    try:"},{"line_number":425,"context_line":"        response \u003d requests.get(url, verify\u003dinsecure)"},{"line_number":426,"context_line":"        if response.status_code \u003d\u003d 200:"},{"line_number":427,"context_line":"            if directory:"},{"line_number":428,"context_line":"                with open(\"%s/%s\" % (directory, filename), \u0027w\u0027) as f:"}],"source_content_type":"text/x-python","patch_set":9,"id":"f348ee9d_4b71457d","line":425,"updated":"2022-02-24 14:42:42.000000000","message":"It seems like this is loading the whole response, it is better to use the stream download to ensure constant memory usage with https://docs.python-requests.org/en/latest/api/#requests.Response.iter_content","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"f0759cc09ee688980ca8f3d1237cd38c95a69b0a","unresolved":false,"context_lines":[{"line_number":422,"context_line":"    logging.debug(\"Started fetching %s\" % url)"},{"line_number":423,"context_line":"    filename \u003d url.split(\"/\")[-1]"},{"line_number":424,"context_line":"    try:"},{"line_number":425,"context_line":"        response \u003d requests.get(url, verify\u003dinsecure)"},{"line_number":426,"context_line":"        if response.status_code \u003d\u003d 200:"},{"line_number":427,"context_line":"            if directory:"},{"line_number":428,"context_line":"                with open(\"%s/%s\" % (directory, filename), \u0027w\u0027) as f:"}],"source_content_type":"text/x-python","patch_set":9,"id":"d3749407_4f62ef3a","line":425,"in_reply_to":"f348ee9d_4b71457d","updated":"2022-02-25 08:23:02.000000000","message":"Done","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":441,"context_line":"                      \"uuid: %s\" % job_result[\"uuid\"])"},{"line_number":442,"context_line":"        return"},{"line_number":443,"context_line":""},{"line_number":444,"context_line":"    build_log_urls \u003d [job_result[\"log_url\"] + s for s in file_to_check]"},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"    results \u003d []"},{"line_number":447,"context_line":"    pool \u003d ThreadPoolExecutor(max_workers\u003dargs.workers)"}],"source_content_type":"text/x-python","patch_set":9,"id":"ea3c20c9_6a88f596","line":444,"updated":"2022-02-24 14:42:42.000000000","message":"It might be better to use urlparse.urljoin to avoid issue when building new urls.","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"f0759cc09ee688980ca8f3d1237cd38c95a69b0a","unresolved":false,"context_lines":[{"line_number":441,"context_line":"                      \"uuid: %s\" % job_result[\"uuid\"])"},{"line_number":442,"context_line":"        return"},{"line_number":443,"context_line":""},{"line_number":444,"context_line":"    build_log_urls \u003d [job_result[\"log_url\"] + s for s in file_to_check]"},{"line_number":445,"context_line":""},{"line_number":446,"context_line":"    results \u003d []"},{"line_number":447,"context_line":"    pool \u003d ThreadPoolExecutor(max_workers\u003dargs.workers)"}],"source_content_type":"text/x-python","patch_set":9,"id":"923ff86f_52698727","line":444,"in_reply_to":"ea3c20c9_6a88f596","updated":"2022-02-25 08:23:02.000000000","message":"Done","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":447,"context_line":"    pool \u003d ThreadPoolExecutor(max_workers\u003dargs.workers)"},{"line_number":448,"context_line":"    for page in pool.map(download_file, build_log_urls,"},{"line_number":449,"context_line":"                         itertools.repeat(directory),"},{"line_number":450,"context_line":"                         itertools.repeat(insecure)):"},{"line_number":451,"context_line":"        if page:"},{"line_number":452,"context_line":"            results.append(page)"},{"line_number":453,"context_line":"    return results"}],"source_content_type":"text/x-python","patch_set":9,"id":"bc415cad_96461479","line":450,"updated":"2022-02-24 14:42:42.000000000","message":"Since insecure is a static argument, perhaps we should use the environment to set its value, e.g. run the tool with TLS_NO_VERIFY\u003d1 for insecure mode and do `os.environ.get(\"TLS_NO_VERIFY\")` to check the value.","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"1ff1b536d7b82ab59f829252e703c2aa8eb7837d","unresolved":false,"context_lines":[{"line_number":447,"context_line":"    pool \u003d ThreadPoolExecutor(max_workers\u003dargs.workers)"},{"line_number":448,"context_line":"    for page in pool.map(download_file, build_log_urls,"},{"line_number":449,"context_line":"                         itertools.repeat(directory),"},{"line_number":450,"context_line":"                         itertools.repeat(insecure)):"},{"line_number":451,"context_line":"        if page:"},{"line_number":452,"context_line":"            results.append(page)"},{"line_number":453,"context_line":"    return results"}],"source_content_type":"text/x-python","patch_set":9,"id":"8a8eb716_16b9d911","line":450,"in_reply_to":"3fae59ac_0249db62","updated":"2022-03-07 13:43:01.000000000","message":"Done","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"f0759cc09ee688980ca8f3d1237cd38c95a69b0a","unresolved":true,"context_lines":[{"line_number":447,"context_line":"    pool \u003d ThreadPoolExecutor(max_workers\u003dargs.workers)"},{"line_number":448,"context_line":"    for page in pool.map(download_file, build_log_urls,"},{"line_number":449,"context_line":"                         itertools.repeat(directory),"},{"line_number":450,"context_line":"                         itertools.repeat(insecure)):"},{"line_number":451,"context_line":"        if page:"},{"line_number":452,"context_line":"            results.append(page)"},{"line_number":453,"context_line":"    return results"}],"source_content_type":"text/x-python","patch_set":9,"id":"3fae59ac_0249db62","line":450,"in_reply_to":"bc415cad_96461479","updated":"2022-02-25 08:23:02.000000000","message":"I would prefer that the user is consciously use the parameter --insecure, but if it will help for deployment, we can use your solution. Right now I prefer to keep as it is.ok?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"}],"logscraper/logsender.py":[{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":169,"context_line":"    fields[\"build_queue\"] \u003d build_details[\"pipeline\"]"},{"line_number":170,"context_line":"    fields[\"build_ref\"] \u003d buildinfo.get(\"ref\")"},{"line_number":171,"context_line":"    fields[\"build_branch\"] \u003d buildinfo.get(\"branch\")"},{"line_number":172,"context_line":"    # FIXME Can not find proper information"},{"line_number":173,"context_line":"    fields[\"build_zuul_url\"] \u003d buildinfo.get(\"build_args\").get(\"zuul_api_url\")"},{"line_number":174,"context_line":"    fields[\"build_change\"] \u003d buildinfo.get(\"change\")"},{"line_number":175,"context_line":"    fields[\"build_patchset\"] \u003d buildinfo.get(\"patchset\")"}],"source_content_type":"text/x-python","patch_set":9,"id":"c661e585_21dd46fa","line":172,"updated":"2022-02-24 14:42:42.000000000","message":"what does this mean?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"784aa29a6956e03e219c53e15f995493f17e6510","unresolved":false,"context_lines":[{"line_number":169,"context_line":"    fields[\"build_queue\"] \u003d build_details[\"pipeline\"]"},{"line_number":170,"context_line":"    fields[\"build_ref\"] \u003d buildinfo.get(\"ref\")"},{"line_number":171,"context_line":"    fields[\"build_branch\"] \u003d buildinfo.get(\"branch\")"},{"line_number":172,"context_line":"    # FIXME Can not find proper information"},{"line_number":173,"context_line":"    fields[\"build_zuul_url\"] \u003d buildinfo.get(\"build_args\").get(\"zuul_api_url\")"},{"line_number":174,"context_line":"    fields[\"build_change\"] \u003d buildinfo.get(\"change\")"},{"line_number":175,"context_line":"    fields[\"build_patchset\"] \u003d buildinfo.get(\"patchset\")"}],"source_content_type":"text/x-python","patch_set":9,"id":"6d1a5629_75cf4011","line":172,"in_reply_to":"c661e585_21dd46fa","updated":"2022-02-25 19:46:48.000000000","message":"That information probably it is deprecated.\nI was comparing information from https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-client.py#L124 and logscraper,so maybe that record can be removed now.","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":197,"context_line":"                                                thread_count\u003dworkers,"},{"line_number":198,"context_line":"                                                chunk_size\u003dchunk_size):"},{"line_number":199,"context_line":"            if not success:"},{"line_number":200,"context_line":"                return"},{"line_number":201,"context_line":"    except Exception as e:"},{"line_number":202,"context_line":"        logging.critical(\"Exception occured on pushing data to \""},{"line_number":203,"context_line":"                         \"Elasticsearch %s\" % e)"}],"source_content_type":"text/x-python","patch_set":9,"id":"8fdead63_fe933c1f","line":200,"updated":"2022-02-24 14:42:42.000000000","message":"should we log an error?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"784aa29a6956e03e219c53e15f995493f17e6510","unresolved":false,"context_lines":[{"line_number":197,"context_line":"                                                thread_count\u003dworkers,"},{"line_number":198,"context_line":"                                                chunk_size\u003dchunk_size):"},{"line_number":199,"context_line":"            if not success:"},{"line_number":200,"context_line":"                return"},{"line_number":201,"context_line":"    except Exception as e:"},{"line_number":202,"context_line":"        logging.critical(\"Exception occured on pushing data to \""},{"line_number":203,"context_line":"                         \"Elasticsearch %s\" % e)"}],"source_content_type":"text/x-python","patch_set":9,"id":"732758bd_5bc95532","line":200,"in_reply_to":"8fdead63_fe933c1f","updated":"2022-02-25 19:46:48.000000000","message":"Done","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":213,"context_line":"    for line in file_content:"},{"line_number":214,"context_line":"        fields \u003d copy.deepcopy(es_fields)"},{"line_number":215,"context_line":"        try:"},{"line_number":216,"context_line":"            timestamp_search \u003d re.search(r\u0027[-0-9]{10}\\s+[0-9.:]{12}\u0027, line)"},{"line_number":217,"context_line":"            timestamp \u003d (timestamp_search.group() if timestamp_search else"},{"line_number":218,"context_line":"                         \"2022-01-01 00:00:00.000\")"},{"line_number":219,"context_line":"            fields[\"@timestamp\"] \u003d datetime.datetime.fromisoformat("}],"source_content_type":"text/x-python","patch_set":9,"id":"d93fe800_d66ccd50","line":216,"updated":"2022-02-24 14:42:42.000000000","message":"Perhaps it would be better to have a dedicated helper with test, e.g. `def get_timestamp(line: str) -\u003e datetime`.\n\nShould this support iso8601 format?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"784aa29a6956e03e219c53e15f995493f17e6510","unresolved":false,"context_lines":[{"line_number":213,"context_line":"    for line in file_content:"},{"line_number":214,"context_line":"        fields \u003d copy.deepcopy(es_fields)"},{"line_number":215,"context_line":"        try:"},{"line_number":216,"context_line":"            timestamp_search \u003d re.search(r\u0027[-0-9]{10}\\s+[0-9.:]{12}\u0027, line)"},{"line_number":217,"context_line":"            timestamp \u003d (timestamp_search.group() if timestamp_search else"},{"line_number":218,"context_line":"                         \"2022-01-01 00:00:00.000\")"},{"line_number":219,"context_line":"            fields[\"@timestamp\"] \u003d datetime.datetime.fromisoformat("}],"source_content_type":"text/x-python","patch_set":9,"id":"68bbe7a0_aa071b06","line":216,"in_reply_to":"d93fe800_d66ccd50","updated":"2022-02-25 19:46:48.000000000","message":"Done","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":215,"context_line":"        try:"},{"line_number":216,"context_line":"            timestamp_search \u003d re.search(r\u0027[-0-9]{10}\\s+[0-9.:]{12}\u0027, line)"},{"line_number":217,"context_line":"            timestamp \u003d (timestamp_search.group() if timestamp_search else"},{"line_number":218,"context_line":"                         \"2022-01-01 00:00:00.000\")"},{"line_number":219,"context_line":"            fields[\"@timestamp\"] \u003d datetime.datetime.fromisoformat("},{"line_number":220,"context_line":"                timestamp).strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\")"},{"line_number":221,"context_line":"        except Exception as e:"}],"source_content_type":"text/x-python","patch_set":9,"id":"7f59deec_de8348ac","line":218,"updated":"2022-02-24 14:42:42.000000000","message":"When the timestamp can\u0027t be found, should the value be null or utcnow() instead?\n\nI think that is going to be an issue with multiline log, e.g. Traceback, where only the first line has the timestamp. In that case, it might be better to re-use the last discovered timestamp.","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"784aa29a6956e03e219c53e15f995493f17e6510","unresolved":true,"context_lines":[{"line_number":215,"context_line":"        try:"},{"line_number":216,"context_line":"            timestamp_search \u003d re.search(r\u0027[-0-9]{10}\\s+[0-9.:]{12}\u0027, line)"},{"line_number":217,"context_line":"            timestamp \u003d (timestamp_search.group() if timestamp_search else"},{"line_number":218,"context_line":"                         \"2022-01-01 00:00:00.000\")"},{"line_number":219,"context_line":"            fields[\"@timestamp\"] \u003d datetime.datetime.fromisoformat("},{"line_number":220,"context_line":"                timestamp).strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\")"},{"line_number":221,"context_line":"        except Exception as e:"}],"source_content_type":"text/x-python","patch_set":9,"id":"925fe6a6_fe55f5da","line":218,"in_reply_to":"7f59deec_de8348ac","updated":"2022-02-25 19:46:48.000000000","message":"imho it should be any date. On the beginning I was thinking to 01 01 1970, but then the error will be not cached in Elasticsearch when someone will check the logs.\nMaybe utcnow is a good way for now.","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"1ff1b536d7b82ab59f829252e703c2aa8eb7837d","unresolved":false,"context_lines":[{"line_number":215,"context_line":"        try:"},{"line_number":216,"context_line":"            timestamp_search \u003d re.search(r\u0027[-0-9]{10}\\s+[0-9.:]{12}\u0027, line)"},{"line_number":217,"context_line":"            timestamp \u003d (timestamp_search.group() if timestamp_search else"},{"line_number":218,"context_line":"                         \"2022-01-01 00:00:00.000\")"},{"line_number":219,"context_line":"            fields[\"@timestamp\"] \u003d datetime.datetime.fromisoformat("},{"line_number":220,"context_line":"                timestamp).strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\")"},{"line_number":221,"context_line":"        except Exception as e:"}],"source_content_type":"text/x-python","patch_set":9,"id":"2033fa0d_567ef1b5","line":218,"in_reply_to":"925fe6a6_fe55f5da","updated":"2022-03-07 13:43:01.000000000","message":"Done","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"a5851b600cc1b058767abd6201e24e006c84ca1d","unresolved":true,"context_lines":[{"line_number":283,"context_line":""},{"line_number":284,"context_line":"    if not index:"},{"line_number":285,"context_line":"        index \u003d args.index_prefix + \\"},{"line_number":286,"context_line":"            datetime.datetime.today().strftime(\u0027%Y.%m.%d\u0027)"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"    es_client \u003d get_es_client(args)"},{"line_number":289,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"6b03a216_5b553579","line":286,"updated":"2022-02-24 14:42:42.000000000","message":"that is odd, should the value be `%Y-%m-%d`?\n\nWhat about logs created yesterday, is it ok to push them to the next day index?","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"784aa29a6956e03e219c53e15f995493f17e6510","unresolved":false,"context_lines":[{"line_number":283,"context_line":""},{"line_number":284,"context_line":"    if not index:"},{"line_number":285,"context_line":"        index \u003d args.index_prefix + \\"},{"line_number":286,"context_line":"            datetime.datetime.today().strftime(\u0027%Y.%m.%d\u0027)"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"    es_client \u003d get_es_client(args)"},{"line_number":289,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"f4498cab_ee310675","line":286,"in_reply_to":"6b03a216_5b553579","updated":"2022-02-25 19:46:48.000000000","message":"yeah, because in ES you base on index pattern, that will cover that, so you will see in the Dashboard service all the data.\nIssue is only when you are doing API request to get the data, because you will have a habit to check index from same day as the logs comes.\n\nFor index how it should look like - both are ok.\nMaybe %Y-%m-%d will be more safe, but current index have such pattern (on opendev and sf.io)","commit_id":"6eaa3e85ee4250bae47f1a3baf8336ac5a25271a"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":13,"context_line":"# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the"},{"line_number":14,"context_line":"# License for the specific language governing permissions and limitations"},{"line_number":15,"context_line":"# under the License."},{"line_number":16,"context_line":"\"\"\""},{"line_number":17,"context_line":"The goal is to get content from build uuid directory and send to Opensearch"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"[ CLI ] -\u003e [ Log directory ] -\u003e [ Zuul inventory ] -\u003e [ Send logs to ES ]"}],"source_content_type":"text/x-python","patch_set":26,"id":"35f64b57_a8ace866","line":16,"updated":"2022-03-08 11:43:23.000000000","message":"Would have been nice to use Typing + mypy. It helps the reviewer to understand the code and prevent type mistakes then runtime errors ...","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":false,"context_lines":[{"line_number":13,"context_line":"# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the"},{"line_number":14,"context_line":"# License for the specific language governing permissions and limitations"},{"line_number":15,"context_line":"# under the License."},{"line_number":16,"context_line":"\"\"\""},{"line_number":17,"context_line":"The goal is to get content from build uuid directory and send to Opensearch"},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"[ CLI ] -\u003e [ Log directory ] -\u003e [ Zuul inventory ] -\u003e [ Send logs to ES ]"}],"source_content_type":"text/x-python","patch_set":26,"id":"7a6b70c3_83003304","line":16,"in_reply_to":"35f64b57_a8ace866","updated":"2022-03-08 14:08:18.000000000","message":"I\u0027m using mypy, also with other tools, but to add Typing IMHO it should be done in another PS.","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":58,"context_line":"    parser.add_argument(\"--username\","},{"line_number":59,"context_line":"                        help\u003d\"Opensearch username\","},{"line_number":60,"context_line":"                        default\u003d\u0027logstash\u0027)"},{"line_number":61,"context_line":"    parser.add_argument(\"--password\", help\u003d\"Opensearch user password\")"},{"line_number":62,"context_line":"    parser.add_argument(\"--index-prefix\", help\u003d\"Prefix for the index. \""},{"line_number":63,"context_line":"                        \"Defaults to logstash-\","},{"line_number":64,"context_line":"                        default\u003d\u0027logstash-\u0027)"}],"source_content_type":"text/x-python","patch_set":26,"id":"55d2251d_6d62afbb","line":61,"updated":"2022-03-08 11:43:23.000000000","message":"X","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":64,"context_line":"                        default\u003d\u0027logstash-\u0027)"},{"line_number":65,"context_line":"    parser.add_argument(\"--index\","},{"line_number":66,"context_line":"                        help\u003d\"Opensearch index. Defaults to: \""},{"line_number":67,"context_line":"                        \"\u003cindex-prefix\u003e-YYYY-DD\")"},{"line_number":68,"context_line":"    parser.add_argument(\"--doc-type\", help\u003d\"Doc type information that will be\""},{"line_number":69,"context_line":"                        \"send to the Opensearch service\","},{"line_number":70,"context_line":"                        default\u003d\"_doc\")"}],"source_content_type":"text/x-python","patch_set":26,"id":"ce32ee3b_fcb83d7c","line":67,"updated":"2022-03-08 11:43:23.000000000","message":"Keep in mind that by default EL allows only 1000 shards.\nMaybe you should expose an option for max_number_of_shards https://www.elastic.co/guide/en/elasticsearch/reference/master/index-modules.html#_static_index_settings","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":true,"context_lines":[{"line_number":64,"context_line":"                        default\u003d\u0027logstash-\u0027)"},{"line_number":65,"context_line":"    parser.add_argument(\"--index\","},{"line_number":66,"context_line":"                        help\u003d\"Opensearch index. Defaults to: \""},{"line_number":67,"context_line":"                        \"\u003cindex-prefix\u003e-YYYY-DD\")"},{"line_number":68,"context_line":"    parser.add_argument(\"--doc-type\", help\u003d\"Doc type information that will be\""},{"line_number":69,"context_line":"                        \"send to the Opensearch service\","},{"line_number":70,"context_line":"                        default\u003d\"_doc\")"}],"source_content_type":"text/x-python","patch_set":26,"id":"27f6149c_11ea43ca","line":67,"in_reply_to":"ce32ee3b_fcb83d7c","updated":"2022-03-08 14:08:18.000000000","message":"the old indexes will be removed after a while. Is this required as a basic parameter?","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":185,"context_line":"    return fields"},{"line_number":186,"context_line":""},{"line_number":187,"context_line":""},{"line_number":188,"context_line":"def send_bulk(es_client, request, workers, ignore_es_status, chunk_size):"},{"line_number":189,"context_line":"    \"\"\"Send bulk request to Opensearch\"\"\""},{"line_number":190,"context_line":"    try:"},{"line_number":191,"context_line":"        if ignore_es_status:"}],"source_content_type":"text/x-python","patch_set":26,"id":"4fe11f7c_3198434a","line":188,"updated":"2022-03-08 11:43:23.000000000","message":"Here \u0027request\u0027 should should be an iterator.","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"df3390e8116302edae1430889892e0a24f0af123","unresolved":true,"context_lines":[{"line_number":235,"context_line":"    \"\"\"Send document to the Opensearch\"\"\""},{"line_number":236,"context_line":"    request \u003d []"},{"line_number":237,"context_line":"    logging.info(\"Working on %s\" % build_file)"},{"line_number":238,"context_line":"    file_content \u003d read_text_file(build_file)"},{"line_number":239,"context_line":"    for line in file_content:"},{"line_number":240,"context_line":"        fields \u003d copy.deepcopy(es_fields)"},{"line_number":241,"context_line":"        fields[\"@timestamp\"] \u003d get_timestamp(line)"}],"source_content_type":"text/x-python","patch_set":26,"id":"bf774da0_5b76cf4b","line":238,"updated":"2022-03-08 11:43:23.000000000","message":"If you get an iterator here then you are safer in term of memory consumption. What if a file is multiple GB ?","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"},{"author":{"_account_id":20676,"name":"daniel.pawlik","display_name":"Daniel Pawlik","email":"dpawlik@redhat.com","username":"daniel.pawlik"},"change_message_id":"52907a47cbf17d2b1e91d97d690ff7240a33c5b3","unresolved":false,"context_lines":[{"line_number":235,"context_line":"    \"\"\"Send document to the Opensearch\"\"\""},{"line_number":236,"context_line":"    request \u003d []"},{"line_number":237,"context_line":"    logging.info(\"Working on %s\" % build_file)"},{"line_number":238,"context_line":"    file_content \u003d read_text_file(build_file)"},{"line_number":239,"context_line":"    for line in file_content:"},{"line_number":240,"context_line":"        fields \u003d copy.deepcopy(es_fields)"},{"line_number":241,"context_line":"        fields[\"@timestamp\"] \u003d get_timestamp(line)"}],"source_content_type":"text/x-python","patch_set":26,"id":"22503a74_7108ba3b","line":238,"in_reply_to":"bf774da0_5b76cf4b","updated":"2022-03-08 14:08:18.000000000","message":"It has been changed in other PS","commit_id":"d41bb6a8c0ed50853d5e89f0c9b2afa701565639"}]}