)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":22636,"name":"Cliff Parsons","email":"cliffhparsons@aol.com","username":"cliffparsons"},"change_message_id":"00bda0d0b4f17204384cec6c9b8c0eb2660f0a66","unresolved":true,"context_lines":[{"line_number":9,"context_line":"This change adds  cleanup mechanism to archive by following steps:"},{"line_number":10,"context_line":"1) add archive_cleanup.sh under /tmp directory"},{"line_number":11,"context_line":"2) through the start.sh this script will be triggered"},{"line_number":12,"context_line":"3) It runs every min, checking utilization of archive dir"},{"line_number":13,"context_line":"4) If it is above threshold it deletes half of old files"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Change-Id: I918284b0aa5a698a6028b9807fcbf6559ef0ff45"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"59757132_eea2755a","line":12,"range":{"start_line":12,"start_character":11,"end_line":12,"end_character":20},"updated":"2021-01-11 19:36:03.000000000","message":"every hour","commit_id":"da8b5c9bda5ce5fda5ec9d7e2737ba8afb96cac9"},{"author":{"_account_id":28778,"name":"Dan Crank","email":"danielcrank@microsoft.com","username":"dancrank"},"change_message_id":"e1b41253b4d36bbb653222ed377913ad24bc5fc5","unresolved":true,"context_lines":[{"line_number":7,"context_line":"postgres archive cleanup script"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"This change adds  cleanup mechanism to archive by following steps:"},{"line_number":10,"context_line":"1) add archive_cleanup.sh under /tmp directory"},{"line_number":11,"context_line":"2) through the start.sh this script will be triggered"},{"line_number":12,"context_line":"3) It runs every hour, checking utilization of archive dir"},{"line_number":13,"context_line":"4) If it is above threshold it deletes half of old files"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"0154a508_e5c47a32","line":10,"range":{"start_line":10,"start_character":32,"end_line":10,"end_character":36},"updated":"2021-01-11 20:54:24.000000000","message":"Is there a reason the script is placed in /tmp? It seems like it might get accidentally removed from there.","commit_id":"09381ce8e9df365025a242e412e95c710ba97578"},{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"efc9f0b8fe879958a94b0b8f90c20a4603122214","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":12,"id":"35493618_e399edf8","updated":"2021-01-11 23:10:36.000000000","message":"Is there an issue that this fixes?","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"},{"author":{"_account_id":31537,"name":"Apurva Gokani","email":"apurvagokani@gmail.com","username":"ag759t"},"change_message_id":"ff8484af18b431869246006807ec015eaffa5ecc","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":12,"id":"efe17ae2_fe06d4f5","in_reply_to":"35493618_e399edf8","updated":"2021-01-12 16:59:26.000000000","message":"There is ongoing issue where archiving directory is getting full on multiple sites because archiving does not have any deletion mechanism. Archiving was introduced to solve issue of postgres xlog directory filling up with WAL(write ahead log) files but those WAL logs also contain MAAS node status which fills up archiving in long run (~month). This change is essential to avoid that issue.","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"}],"postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl":[{"author":{"_account_id":22636,"name":"Cliff Parsons","email":"cliffhparsons@aol.com","username":"cliffparsons"},"change_message_id":"19888893210c368983178bdc478fe130ba5eef02","unresolved":true,"context_lines":[{"line_number":45,"context_line":"        then"},{"line_number":46,"context_line":"                        clean_up"},{"line_number":47,"context_line":"        fi"},{"line_number":48,"context_line":"        sleep 60"},{"line_number":49,"context_line":"done"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":""}],"source_content_type":"text/x-smarty","patch_set":8,"id":"da14ea49_af18fa55","line":48,"range":{"start_line":48,"start_character":8,"end_line":48,"end_character":16},"updated":"2021-01-11 19:20:02.000000000","message":"this seems excessive to run this every minute until the server terminates.","commit_id":"b6b1319562ff9b4e876377505329421883432966"},{"author":{"_account_id":22636,"name":"Cliff Parsons","email":"cliffhparsons@aol.com","username":"cliffparsons"},"change_message_id":"f6a383ca03c8ab4a7a1ef3814a9deb07e225142c","unresolved":true,"context_lines":[{"line_number":43,"context_line":"        UTILIZATION\u003d$(df -h $ARCHIVE_DIR | awk \u0027 NR\u003d\u003d2 {print $5} \u0027 | awk \u0027{ print substr( $0, 1, length($0)-1 ) }\u0027)"},{"line_number":44,"context_line":"        if [[ $UTILIZATION -gt ${ARCHIVE_LIMIT} ]];"},{"line_number":45,"context_line":"        then"},{"line_number":46,"context_line":"                        clean_up"},{"line_number":47,"context_line":"        fi"},{"line_number":48,"context_line":"        sleep 3600"},{"line_number":49,"context_line":"done"}],"source_content_type":"text/x-smarty","patch_set":10,"id":"b450f5e8_91028db8","line":46,"range":{"start_line":46,"start_character":24,"end_line":46,"end_character":32},"updated":"2021-01-11 19:34:46.000000000","message":"nit: indented too much","commit_id":"da8b5c9bda5ce5fda5ec9d7e2737ba8afb96cac9"},{"author":{"_account_id":28778,"name":"Dan Crank","email":"danielcrank@microsoft.com","username":"dancrank"},"change_message_id":"e1b41253b4d36bbb653222ed377913ad24bc5fc5","unresolved":true,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"# Constants"},{"line_number":18,"context_line":"# Absolute Location of postgres configuration"},{"line_number":19,"context_line":"CONF_FILE\u003d\"/tmp/postgresql.conf\""},{"line_number":20,"context_line":"# Absolute Location of archive mount"},{"line_number":21,"context_line":"ARCHIVE_DIR\u003d\"/var/lib/archive\""},{"line_number":22,"context_line":"# ARCHIVE_LIMIT env variable is Threshold of archiving supposed to be kept in percentage"}],"source_content_type":"text/x-smarty","patch_set":11,"id":"8f6b8290_18ad4c24","line":19,"range":{"start_line":19,"start_character":0,"end_line":19,"end_character":9},"updated":"2021-01-11 20:54:24.000000000","message":"nit: this is not used","commit_id":"09381ce8e9df365025a242e412e95c710ba97578"},{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"efc9f0b8fe879958a94b0b8f90c20a4603122214","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":12,"id":"ae5989cd_174fc705","updated":"2021-01-11 23:10:36.000000000","message":"Most of the bash we use is 2 space delimited, can we have this file follow the same format as well.","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"},{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"efc9f0b8fe879958a94b0b8f90c20a4603122214","unresolved":true,"context_lines":[{"line_number":15,"context_line":"set +ex"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"# Absolute Location of archive mount"},{"line_number":18,"context_line":"ARCHIVE_DIR\u003d\"/var/lib/archive\""},{"line_number":19,"context_line":"# ARCHIVE_LIMIT env variable is Threshold of archiving supposed to be kept in percentage"},{"line_number":20,"context_line":"clean_up () {"},{"line_number":21,"context_line":"        echo \"Cleanup required as Utilization is above threshold\""}],"source_content_type":"text/x-smarty","patch_set":12,"id":"c64e812f_6aed7b6c","line":18,"range":{"start_line":18,"start_character":0,"end_line":18,"end_character":30},"updated":"2021-01-11 23:10:36.000000000","message":"We reference this in values.yaml, why not use that similar to ARCHIVE_LIMIT here.","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"},{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"efc9f0b8fe879958a94b0b8f90c20a4603122214","unresolved":true,"context_lines":[{"line_number":19,"context_line":"# ARCHIVE_LIMIT env variable is Threshold of archiving supposed to be kept in percentage"},{"line_number":20,"context_line":"clean_up () {"},{"line_number":21,"context_line":"        echo \"Cleanup required as Utilization is above threshold\""},{"line_number":22,"context_line":"        # Get file count and delete half of the archive while maintaing the order of the files"},{"line_number":23,"context_line":"        FILE_COUNT\u003d$(ls -1 ${ARCHIVE_DIR} | sort | wc -l)"},{"line_number":24,"context_line":"        COUNT\u003d0"},{"line_number":25,"context_line":"        echo $((FILE_COUNT/2))"}],"source_content_type":"text/x-smarty","patch_set":12,"id":"48798fbd_07a1bdb1","line":22,"range":{"start_line":22,"start_character":62,"end_line":22,"end_character":72},"updated":"2021-01-11 23:10:36.000000000","message":"nit: \"maintaining\"","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"},{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"efc9f0b8fe879958a94b0b8f90c20a4603122214","unresolved":true,"context_lines":[{"line_number":19,"context_line":"# ARCHIVE_LIMIT env variable is Threshold of archiving supposed to be kept in percentage"},{"line_number":20,"context_line":"clean_up () {"},{"line_number":21,"context_line":"        echo \"Cleanup required as Utilization is above threshold\""},{"line_number":22,"context_line":"        # Get file count and delete half of the archive while maintaing the order of the files"},{"line_number":23,"context_line":"        FILE_COUNT\u003d$(ls -1 ${ARCHIVE_DIR} | sort | wc -l)"},{"line_number":24,"context_line":"        COUNT\u003d0"},{"line_number":25,"context_line":"        echo $((FILE_COUNT/2))"}],"source_content_type":"text/x-smarty","patch_set":12,"id":"ac6bd041_17513eef","line":22,"range":{"start_line":22,"start_character":29,"end_line":22,"end_character":40},"updated":"2021-01-11 23:10:36.000000000","message":"why half?","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"},{"author":{"_account_id":31537,"name":"Apurva Gokani","email":"apurvagokani@gmail.com","username":"ag759t"},"change_message_id":"ff8484af18b431869246006807ec015eaffa5ecc","unresolved":true,"context_lines":[{"line_number":19,"context_line":"# ARCHIVE_LIMIT env variable is Threshold of archiving supposed to be kept in percentage"},{"line_number":20,"context_line":"clean_up () {"},{"line_number":21,"context_line":"        echo \"Cleanup required as Utilization is above threshold\""},{"line_number":22,"context_line":"        # Get file count and delete half of the archive while maintaing the order of the files"},{"line_number":23,"context_line":"        FILE_COUNT\u003d$(ls -1 ${ARCHIVE_DIR} | sort | wc -l)"},{"line_number":24,"context_line":"        COUNT\u003d0"},{"line_number":25,"context_line":"        echo $((FILE_COUNT/2))"}],"source_content_type":"text/x-smarty","patch_set":12,"id":"af0a6198_e6e6d951","line":22,"range":{"start_line":22,"start_character":29,"end_line":22,"end_character":40},"in_reply_to":"ac6bd041_17513eef","updated":"2021-01-12 16:59:26.000000000","message":"We want to safeguard against memory issue but still the archiving helps in PITR (point in time recovery). Postgres has mechanism which will unpack these zipped WAL files and use them to automatically recover in case of crash. Script is adjusted to keep 30-60 % default archiving which helps in PITR without risk of running into memory issues.","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"}],"postgresql/templates/bin/_start.sh.tpl":[{"author":{"_account_id":8863,"name":"Andrii Ostapenko","email":"anost1986@gmail.com","username":"aostapenko"},"change_message_id":"44dbb8d36c20a80a62a12a31cb6ca5fdf0e17134","unresolved":true,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"set -x"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"bash /tmp/archive_cleanup.sh \u0026"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"exec /docker-entrypoint.sh postgres -c config_file\u003d/tmp/postgresql.conf"}],"source_content_type":"text/x-smarty","patch_set":13,"id":"ec46fd6c_67be7747","line":38,"range":{"start_line":38,"start_character":0,"end_line":38,"end_character":30},"updated":"2021-01-13 18:59:18.000000000","message":"why background here? it\u0027s not supposed to take long and since we get no time benefit - lets keep it simpler","commit_id":"a1b2e56bb0c381fd88c9c57f5c2e85707cd794b1"},{"author":{"_account_id":31537,"name":"Apurva Gokani","email":"apurvagokani@gmail.com","username":"ag759t"},"change_message_id":"0245203b6bd14a45c338eabee489e1d34d6cd7ef","unresolved":true,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"set -x"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"bash /tmp/archive_cleanup.sh \u0026"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"exec /docker-entrypoint.sh postgres -c config_file\u003d/tmp/postgresql.conf"}],"source_content_type":"text/x-smarty","patch_set":13,"id":"12753b9d_774bf40e","line":38,"range":{"start_line":38,"start_character":0,"end_line":38,"end_character":30},"in_reply_to":"ec46fd6c_67be7747","updated":"2021-01-13 19:27:49.000000000","message":"It is done in background since this check is ran continuously every hour for the lifetime of pod. If we just execute the cleanup script without \u0026, bash script execution would not move to next line which starts up postgres DB.","commit_id":"a1b2e56bb0c381fd88c9c57f5c2e85707cd794b1"}],"postgresql/values.yaml":[{"author":{"_account_id":22636,"name":"Cliff Parsons","email":"cliffhparsons@aol.com","username":"cliffparsons"},"change_message_id":"19888893210c368983178bdc478fe130ba5eef02","unresolved":true,"context_lines":[{"line_number":181,"context_line":"    class_path: volume.beta.kubernetes.io/storage-class"},{"line_number":182,"context_line":"  archive_pvc:"},{"line_number":183,"context_line":"    size: 5Gi"},{"line_number":184,"context_line":"    memory_limit: 60"},{"line_number":185,"context_line":"    class_name: general"},{"line_number":186,"context_line":"    class_path: volume.beta.kubernetes.io/storage-class"},{"line_number":187,"context_line":"  host:"}],"source_content_type":"text/x-yaml","patch_set":8,"id":"2580762c_365214ce","line":184,"range":{"start_line":184,"start_character":4,"end_line":184,"end_character":10},"updated":"2021-01-11 19:20:02.000000000","message":"I think \"archive_limit\" is a better name for this, since you map this to an env variable named \"ARCHIVE_LIMIT\"","commit_id":"b6b1319562ff9b4e876377505329421883432966"},{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"efc9f0b8fe879958a94b0b8f90c20a4603122214","unresolved":true,"context_lines":[{"line_number":181,"context_line":"    class_path: volume.beta.kubernetes.io/storage-class"},{"line_number":182,"context_line":"  archive_pvc:"},{"line_number":183,"context_line":"    size: 5Gi"},{"line_number":184,"context_line":"    archive_limit: 60"},{"line_number":185,"context_line":"    class_name: general"},{"line_number":186,"context_line":"    class_path: volume.beta.kubernetes.io/storage-class"},{"line_number":187,"context_line":"  host:"}],"source_content_type":"text/x-yaml","patch_set":12,"id":"20b94c08_adfdf258","line":184,"range":{"start_line":184,"start_character":4,"end_line":184,"end_character":21},"updated":"2021-01-11 23:10:36.000000000","message":"We use mount_path for archive below in the cleanup script, let\u0027s move this value down there as well.","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"},{"author":{"_account_id":21420,"name":"Gage Hugo","email":"gagehugo@gmail.com","username":"ghugo"},"change_message_id":"efc9f0b8fe879958a94b0b8f90c20a4603122214","unresolved":true,"context_lines":[{"line_number":189,"context_line":"  mount:"},{"line_number":190,"context_line":"    path: /var/lib/postgresql"},{"line_number":191,"context_line":"    subpath: ."},{"line_number":192,"context_line":"  archive:"},{"line_number":193,"context_line":"    mount_path: /var/lib/archive"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"labels:"},{"line_number":196,"context_line":"  server:"}],"source_content_type":"text/x-yaml","patch_set":12,"id":"604537e7_5d18664c","line":193,"range":{"start_line":192,"start_character":0,"end_line":193,"end_character":32},"updated":"2021-01-11 23:10:36.000000000","message":"Here","commit_id":"6c75d8860ce04e909b5efe68e6c5334c16354933"}]}