)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"814841e73806a17319035c6fc878977975e9da6d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"95462b3b_ac619321","updated":"2025-12-18 17:48:33.000000000","message":"The idea is that all changes should be made from the deployment host, never locally. If a node in the deployment dies, you can then trivially re-deploy it. Even better, all the config on the deployment host is under version control (for example using Kayobe). If you have local changes, they are likely to get lost. \nThere is another problem, than on a multi-node deployment, all Grafana instances must be configured identically, otherwise you can get strange behaviour with load balancing (not all Grafana config goes into the DB).","commit_id":"d68d385ecb3d259d4e094860198ee1ae7eb17201"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"35052bf79cd9dfa069838cb9fcfc67d4db10c84b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"9b87ffcd_402ea773","updated":"2025-12-19 16:17:36.000000000","message":"Nice job on re-working this Piotr. I think it\u0027s almost there.","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"8791619cb226f5f8a69446481e20ddb0b6e545b8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":11,"id":"c45f66d5_3a067acf","updated":"2026-03-25 08:45:09.000000000","message":"recheck - POST_FAILURE","commit_id":"755d3eb9bda1dbd9ed870f2db2cfda254cbcd9dc"}],"ansible/roles/grafana/tasks/config.yml":[{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"35052bf79cd9dfa069838cb9fcfc67d4db10c84b","unresolved":true,"context_lines":[{"line_number":98,"context_line":"  vars:"},{"line_number":99,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""},{"line_number":100,"context_line":"  copy:"},{"line_number":101,"context_line":"    src: \"{{ node_custom_config }}/grafana/dashboards/\""},{"line_number":102,"context_line":"    dest: \"{{ node_config_directory }}/grafana/dashboards/\""},{"line_number":103,"context_line":"    mode: \"0660\""},{"line_number":104,"context_line":"  register: copy_result"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"0fa38189_b0cbd9b4","line":101,"updated":"2025-12-19 16:17:36.000000000","message":"Please amend so that sub-folders are also copied","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"57c7a6358d814cc158b2e13da7fe74d702225322","unresolved":false,"context_lines":[{"line_number":98,"context_line":"  vars:"},{"line_number":99,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""},{"line_number":100,"context_line":"  copy:"},{"line_number":101,"context_line":"    src: \"{{ node_custom_config }}/grafana/dashboards/\""},{"line_number":102,"context_line":"    dest: \"{{ node_config_directory }}/grafana/dashboards/\""},{"line_number":103,"context_line":"    mode: \"0660\""},{"line_number":104,"context_line":"  register: copy_result"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"1fd4f728_5bc85e0f","line":101,"in_reply_to":"0fa38189_b0cbd9b4","updated":"2025-12-22 11:01:19.000000000","message":"Acknowledged","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"35052bf79cd9dfa069838cb9fcfc67d4db10c84b","unresolved":true,"context_lines":[{"line_number":103,"context_line":"    mode: \"0660\""},{"line_number":104,"context_line":"  register: copy_result"},{"line_number":105,"context_line":"  when:"},{"line_number":106,"context_line":"    - grafana_custom_dashboards_folder.stat.exists"},{"line_number":107,"context_line":"    - grafana_custom_dashboards_folder.stat.isdir"},{"line_number":108,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":109,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":3,"id":"29a3b9af_d4bfbac7","line":106,"updated":"2025-12-19 16:17:36.000000000","message":"We have lost the `not ansible_check_mode` condition here","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"57c7a6358d814cc158b2e13da7fe74d702225322","unresolved":false,"context_lines":[{"line_number":103,"context_line":"    mode: \"0660\""},{"line_number":104,"context_line":"  register: copy_result"},{"line_number":105,"context_line":"  when:"},{"line_number":106,"context_line":"    - grafana_custom_dashboards_folder.stat.exists"},{"line_number":107,"context_line":"    - grafana_custom_dashboards_folder.stat.isdir"},{"line_number":108,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":109,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":3,"id":"763ede6b_6215e531","line":106,"in_reply_to":"29a3b9af_d4bfbac7","updated":"2025-12-22 11:01:19.000000000","message":"Acknowledged","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"6c470f420adee63dbf4e3b8568e1eb5dd4281ff8","unresolved":true,"context_lines":[{"line_number":115,"context_line":"    file_type: file"},{"line_number":116,"context_line":"  register: remote_dashboards"},{"line_number":117,"context_line":"  when:"},{"line_number":118,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"- name: Remove obsolete dashboards"},{"line_number":121,"context_line":"  become: true"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"2c7badf5_d66bedd3","line":118,"updated":"2025-12-19 16:50:48.000000000","message":"This needs to be added to the task:\n\n```\nvars:\n    service: \"{{ grafana_services[\u0027grafana\u0027] }}\"\n```","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"57c7a6358d814cc158b2e13da7fe74d702225322","unresolved":false,"context_lines":[{"line_number":115,"context_line":"    file_type: file"},{"line_number":116,"context_line":"  register: remote_dashboards"},{"line_number":117,"context_line":"  when:"},{"line_number":118,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"- name: Remove obsolete dashboards"},{"line_number":121,"context_line":"  become: true"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"a92f3c8c_04b0a851","line":118,"in_reply_to":"2c7badf5_d66bedd3","updated":"2025-12-22 11:01:19.000000000","message":"Acknowledged","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"6c470f420adee63dbf4e3b8568e1eb5dd4281ff8","unresolved":true,"context_lines":[{"line_number":124,"context_line":"    state: absent"},{"line_number":125,"context_line":"  loop: \"{{ remote_dashboards.files }}\""},{"line_number":126,"context_line":"  when:"},{"line_number":127,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":128,"context_line":"    - item.path | basename not in lookup(\u0027fileglob\u0027, node_custom_config + \u0027/grafana/dashboards/*.json\u0027, wantlist\u003dTrue) | map(\u0027basename\u0027) | list"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"7a720867_51cb89d2","line":127,"updated":"2025-12-19 16:50:48.000000000","message":"ditto","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"57c7a6358d814cc158b2e13da7fe74d702225322","unresolved":false,"context_lines":[{"line_number":124,"context_line":"    state: absent"},{"line_number":125,"context_line":"  loop: \"{{ remote_dashboards.files }}\""},{"line_number":126,"context_line":"  when:"},{"line_number":127,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":128,"context_line":"    - item.path | basename not in lookup(\u0027fileglob\u0027, node_custom_config + \u0027/grafana/dashboards/*.json\u0027, wantlist\u003dTrue) | map(\u0027basename\u0027) | list"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"8b111a72_b794f4d4","line":127,"in_reply_to":"7a720867_51cb89d2","updated":"2025-12-22 11:01:19.000000000","message":"Acknowledged","commit_id":"b027db968664f342e51592de7225ecb65bf105c2"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"0f1470bbdcaf2d257dd1d00e99a177024d147efb","unresolved":true,"context_lines":[{"line_number":117,"context_line":"  register: copy_result"},{"line_number":118,"context_line":"  when:"},{"line_number":119,"context_line":"    - grafana_custom_dashboards_folder.stat.exists"},{"line_number":120,"context_line":"    - grafana_custom_dashboards_folder.stat.isdir"},{"line_number":121,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":122,"context_line":"    - not ansible_check_mode"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":6,"id":"636c80a6_6e4c1a88","line":120,"updated":"2026-01-09 16:40:27.000000000","message":"Do we need to ensure that all the sub-directories are copied first?","commit_id":"86c7d89862875697d7c65c2eb5186b41d98749f9"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"5f7959b313a4f0cd191e2deb9b1a37f2eb8144d8","unresolved":false,"context_lines":[{"line_number":117,"context_line":"  register: copy_result"},{"line_number":118,"context_line":"  when:"},{"line_number":119,"context_line":"    - grafana_custom_dashboards_folder.stat.exists"},{"line_number":120,"context_line":"    - grafana_custom_dashboards_folder.stat.isdir"},{"line_number":121,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":122,"context_line":"    - not ansible_check_mode"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":6,"id":"8cfe3712_641fa723","line":120,"in_reply_to":"2087659f_741fa7fd","updated":"2026-01-12 11:46:44.000000000","message":"Thanks","commit_id":"86c7d89862875697d7c65c2eb5186b41d98749f9"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"87f6c73ff1e01be690a286e47953bb52cc569344","unresolved":true,"context_lines":[{"line_number":117,"context_line":"  register: copy_result"},{"line_number":118,"context_line":"  when:"},{"line_number":119,"context_line":"    - grafana_custom_dashboards_folder.stat.exists"},{"line_number":120,"context_line":"    - grafana_custom_dashboards_folder.stat.isdir"},{"line_number":121,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":122,"context_line":"    - not ansible_check_mode"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":6,"id":"2087659f_741fa7fd","line":120,"in_reply_to":"636c80a6_6e4c1a88","updated":"2026-01-09 17:01:47.000000000","message":"This check is a safeguard for the code. Without it, if someone accidentally created a file named dashboards instead of a directory on the control host (localhost), the copy modules would trigger an error when trying to perform directory-level operations on a file. It essentially \u0027insures\u0027 the playbook against structural errors in the configuration path","commit_id":"86c7d89862875697d7c65c2eb5186b41d98749f9"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"5f7959b313a4f0cd191e2deb9b1a37f2eb8144d8","unresolved":true,"context_lines":[{"line_number":110,"context_line":"  vars:"},{"line_number":111,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""},{"line_number":112,"context_line":"  copy:"},{"line_number":113,"context_line":"    src: \"{{ node_custom_config }}/grafana/dashboards/\""},{"line_number":114,"context_line":"    dest: \"{{ node_config_directory }}/grafana/dashboards/\""},{"line_number":115,"context_line":"    mode: \"0660\""},{"line_number":116,"context_line":"    directory_mode: \"0770\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"56912e9c_92cfe5ae","line":113,"updated":"2026-01-12 11:46:44.000000000","message":"The original lines 111 and 112 will \u0027synchronise\u0027 `src` with `dest`. I.e. Ansible will remove files not in `src` from `dest`.  The change made here stops it from doing that.\n\n\nThere is a subtle hint in the module doc:\n\n```\nIf path is a directory, it is copied recursively. In this case, if path ends with /, only inside contents of that directory are copied to destination. Otherwise, if it does not end with /, the directory itself with all contents is copied. This behavior is similar to the rsync command line tool.\n```","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"6f214f2f2bda368aa1b2fd9ce84ef7759e0b837b","unresolved":false,"context_lines":[{"line_number":110,"context_line":"  vars:"},{"line_number":111,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""},{"line_number":112,"context_line":"  copy:"},{"line_number":113,"context_line":"    src: \"{{ node_custom_config }}/grafana/dashboards/\""},{"line_number":114,"context_line":"    dest: \"{{ node_config_directory }}/grafana/dashboards/\""},{"line_number":115,"context_line":"    mode: \"0660\""},{"line_number":116,"context_line":"    directory_mode: \"0770\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"dc210bd0_11491024","line":113,"in_reply_to":"56912e9c_92cfe5ae","updated":"2026-03-12 15:04:49.000000000","message":"Acknowledged","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"5f7959b313a4f0cd191e2deb9b1a37f2eb8144d8","unresolved":true,"context_lines":[{"line_number":134,"context_line":"  when:"},{"line_number":135,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Remove obsolete dashboards"},{"line_number":138,"context_line":"  become: true"},{"line_number":139,"context_line":"  vars:"},{"line_number":140,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"4bc907b0_30e7f235","line":137,"updated":"2026-01-12 11:46:44.000000000","message":"This task (and the one above) are not required if you revert lines 111 and 112.\n\nI think the crux of this issue, is that the stale files still exist in the container, even if they don\u0027t outside. Run `docker exec -it grafana bash` and go check `/var/lib/grafana/dashboards` to see that stale files are not removed there.\n\nThe easiest solution is to use a Docker volume to expose the dashboards to the container, rather than copying them in when it starts.\n\nYou could reuse the existing Docker volume for the Grafana config. The only issue I can see with that, is that people will need to update their `provisioning.yml` files to point to the new location. A symlink could make that easier. *Or* you could just make a second volume.","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"802a13d4f5375bcac77efd679c60d43079d992f4","unresolved":true,"context_lines":[{"line_number":134,"context_line":"  when:"},{"line_number":135,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Remove obsolete dashboards"},{"line_number":138,"context_line":"  become: true"},{"line_number":139,"context_line":"  vars:"},{"line_number":140,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"318423ea_000560e1","line":137,"in_reply_to":"081a344f_c7383bfc","updated":"2026-03-25 14:11:16.000000000","message":"Hi @Piotr - I still think we need to remove orphaned dashboards copied to the internal filesystem. Perhaps I missed something? Steps to reproduce:\n\n1. Configure grafana with custom dashboards\n2. Remove all the custom dashboards on the deployment host\n3. Reconfigure Grafana. Verify all custom dashboards are removed on the target nodes (up to here we are ok)\n4. On a target node, run docker exec -it grafana bash and go check /var/lib/grafana/dashboards - I think you will see the old dashboards?","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"177e14068f2a07e03519e8a546f4794650781817","unresolved":true,"context_lines":[{"line_number":134,"context_line":"  when:"},{"line_number":135,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Remove obsolete dashboards"},{"line_number":138,"context_line":"  become: true"},{"line_number":139,"context_line":"  vars:"},{"line_number":140,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"9219e853_bc68740f","line":137,"in_reply_to":"318423ea_000560e1","updated":"2026-03-25 15:46:18.000000000","message":"https://paste.opendev.org/show/beB5pTIuaq80TVpPAvq1/\n\nit works fine for me","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"6f214f2f2bda368aa1b2fd9ce84ef7759e0b837b","unresolved":true,"context_lines":[{"line_number":134,"context_line":"  when:"},{"line_number":135,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Remove obsolete dashboards"},{"line_number":138,"context_line":"  become: true"},{"line_number":139,"context_line":"  vars:"},{"line_number":140,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"081a344f_c7383bfc","line":137,"in_reply_to":"4bc907b0_30e7f235","updated":"2026-03-12 15:04:49.000000000","message":"@doug@stackhpc.com Can you see if the current solution is acceptable?","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"},{"author":{"_account_id":17669,"name":"Doug Szumski","email":"doug@stackhpc.com","username":"DougSzumski"},"change_message_id":"cd7c9cfbdd936698a4c61c7967d77872c2329166","unresolved":true,"context_lines":[{"line_number":134,"context_line":"  when:"},{"line_number":135,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Remove obsolete dashboards"},{"line_number":138,"context_line":"  become: true"},{"line_number":139,"context_line":"  vars:"},{"line_number":140,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"ecf2ca62_000ef274","line":137,"in_reply_to":"9219e853_bc68740f","updated":"2026-03-25 17:10:11.000000000","message":"Thanks, yeah, that looks good. I\u0027ll take a closer look tomorrow.","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"},{"author":{"_account_id":37306,"name":"Piotr Milewski","display_name":"Piotr Milewski","email":"vurmil@gmail.com","username":"vurmil"},"change_message_id":"a87912871b0971c110e2e5981ab1cd57208badfa","unresolved":true,"context_lines":[{"line_number":134,"context_line":"  when:"},{"line_number":135,"context_line":"    - service | service_enabled_and_mapped_to_host"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"- name: Remove obsolete dashboards"},{"line_number":138,"context_line":"  become: true"},{"line_number":139,"context_line":"  vars:"},{"line_number":140,"context_line":"    service: \"{{ grafana_services[\u0027grafana\u0027] }}\""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"d889e00c_a5f4ed2d","line":137,"in_reply_to":"ecf2ca62_000ef274","updated":"2026-03-31 08:36:53.000000000","message":"@doug@stackhpc.com can you see?","commit_id":"1031a2ef3e0fc5e548a79d5b4d5d85bee83522bb"}]}