)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":8898,"name":"Chris Wedgwood","email":"cw@f00f.org","username":"anticw"},"change_message_id":"ff6dacbdf7407af3b030abe1a256792e04b8746a","unresolved":false,"context_lines":[{"line_number":7,"context_line":"[CEPH] Setup a cronjob to run OSD defrags for FileStore"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Create a cron and associated script to run monthly OSD defrags."},{"line_number":10,"context_line":"When the script runs it will switch the OSD disk to the CFQ I/O"},{"line_number":11,"context_line":"scheduler to ensure that this is a non-blocking operation for ceph."},{"line_number":12,"context_line":"While this cron job will run monthly, it will only execute on OSDs"},{"line_number":13,"context_line":"that are HDD based with Filestore."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"1f769fc5_e2dbb87a","line":11,"range":{"start_line":10,"start_character":0,"end_line":11,"end_character":67},"updated":"2018-12-23 01:15:16.000000000","message":"why does the choice of scheduler matter here? it will block otherwise?","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":29268,"name":"Matthew Heler","email":"matthew.heler@hotmail.com","username":"mheler"},"change_message_id":"50fd5ffb8383f9c4859f5ed14627cca4d5e92fe8","unresolved":false,"context_lines":[{"line_number":7,"context_line":"[CEPH] Setup a cronjob to run OSD defrags for FileStore"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Create a cron and associated script to run monthly OSD defrags."},{"line_number":10,"context_line":"When the script runs it will switch the OSD disk to the CFQ I/O"},{"line_number":11,"context_line":"scheduler to ensure that this is a non-blocking operation for ceph."},{"line_number":12,"context_line":"While this cron job will run monthly, it will only execute on OSDs"},{"line_number":13,"context_line":"that are HDD based with Filestore."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"1f769fc5_0247d41f","line":11,"range":{"start_line":10,"start_character":0,"end_line":11,"end_character":67},"in_reply_to":"1f769fc5_e2dbb87a","updated":"2018-12-23 02:00:30.000000000","message":"ionice is calling xfs_fsr to run a defragmentation. ionice priority classes only work with the CFQ disk scheduler. Basically this script only uses idle IO to run a defrag, this way there is no IO contention with Ceph. Without this the latency in Ceph spikes up pretty high.","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"}],"ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl":[{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"36e1ce7905c0025a2a9133a3cb4db72f2e4371a3","unresolved":false,"context_lines":[{"line_number":26,"context_line":"  \u0027--output\u003djsonpath\u003d{range .items[*]}{.metadata.name}{\"\\n\"}{end}\u0027)"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"  for POD in ${PODS}; do"},{"line_number":29,"context_line":"    kubectl exec -t ${POD} --namespace\u003d${NAMESPACE} -- \\"},{"line_number":30,"context_line":"    sh -c -e \"/tmp/utils-defragOSDs.sh defrag\""},{"line_number":31,"context_line":"  done"},{"line_number":32,"context_line":"fi"}],"source_content_type":"text/x-smarty","patch_set":7,"id":"1f769fc5_66ed5cee","line":29,"updated":"2018-12-27 16:46:19.000000000","message":"I don\u0027t like the idea of `kubectl exec`ing into running pods to do maintenance, although it makes for a simple script below.  Have any alternatives been explored?","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":29268,"name":"Matthew Heler","email":"matthew.heler@hotmail.com","username":"mheler"},"change_message_id":"ce50a8c56b0c675683bbf272b1aceb203ec7614f","unresolved":false,"context_lines":[{"line_number":26,"context_line":"  \u0027--output\u003djsonpath\u003d{range .items[*]}{.metadata.name}{\"\\n\"}{end}\u0027)"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"  for POD in ${PODS}; do"},{"line_number":29,"context_line":"    kubectl exec -t ${POD} --namespace\u003d${NAMESPACE} -- \\"},{"line_number":30,"context_line":"    sh -c -e \"/tmp/utils-defragOSDs.sh defrag\""},{"line_number":31,"context_line":"  done"},{"line_number":32,"context_line":"fi"}],"source_content_type":"text/x-smarty","patch_set":7,"id":"1f769fc5_863e7827","line":29,"in_reply_to":"1f769fc5_66ed5cee","updated":"2018-12-27 17:17:30.000000000","message":"We are already doing this with several other solutions for Ceph. There is no way around this.","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"2b2f1f8a7c57ffa6c47c4b739c12f5f81916afba","unresolved":false,"context_lines":[{"line_number":26,"context_line":"  \u0027--output\u003djsonpath\u003d{range .items[*]}{.metadata.name}{\"\\n\"}{end}\u0027)"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"  for POD in ${PODS}; do"},{"line_number":29,"context_line":"    kubectl exec -t ${POD} --namespace\u003d${NAMESPACE} -- \\"},{"line_number":30,"context_line":"    sh -c -e \"/tmp/utils-defragOSDs.sh defrag\""},{"line_number":31,"context_line":"  done"},{"line_number":32,"context_line":"fi"}],"source_content_type":"text/x-smarty","patch_set":7,"id":"1f769fc5_ca0d4a38","line":29,"in_reply_to":"1f769fc5_863e7827","updated":"2018-12-28 18:59:08.000000000","message":"Maybe so - I\u0027ll marinate on this.","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"36e1ce7905c0025a2a9133a3cb4db72f2e4371a3","unresolved":false,"context_lines":[{"line_number":47,"context_line":"    ionice -c 3 xfs_fsr \"${OSD_DEVICE}\" 2\u003e/dev/null"},{"line_number":48,"context_line":"    # NOTE(supamatt): Switch back to previous IO scheduler"},{"line_number":49,"context_line":"    echo ${ODEV_SCHEDULER} | tee /sys/block/${ODEV}/queue/scheduler || true"},{"line_number":50,"context_line":"  fi"},{"line_number":51,"context_line":"fi"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"exit 0"}],"source_content_type":"text/x-smarty","patch_set":7,"id":"1f769fc5_669b1c72","line":50,"updated":"2018-12-27 16:46:19.000000000","message":"nice work!","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"}],"ceph-osd/templates/daemonset-osd.yaml":[{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"36e1ce7905c0025a2a9133a3cb4db72f2e4371a3","unresolved":false,"context_lines":[{"line_number":246,"context_line":"              mountPath: /tmp/utils-checkDNS.sh"},{"line_number":247,"context_line":"              subPath: utils-checkDNS.sh"},{"line_number":248,"context_line":"              readOnly: true"},{"line_number":249,"context_line":"            - name: ceph-osd-bin"},{"line_number":250,"context_line":"              mountPath: /tmp/utils-defragOSDs.sh"},{"line_number":251,"context_line":"              subPath: utils-defragOSDs.sh"},{"line_number":252,"context_line":"              readOnly: true"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"1f769fc5_461ce069","line":249,"updated":"2018-12-27 16:46:19.000000000","message":"Why are we pulling the defrag script into the daemonset?  It looks like it\u0027s only being called from the cronjob, right?","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":29268,"name":"Matthew Heler","email":"matthew.heler@hotmail.com","username":"mheler"},"change_message_id":"ce50a8c56b0c675683bbf272b1aceb203ec7614f","unresolved":false,"context_lines":[{"line_number":246,"context_line":"              mountPath: /tmp/utils-checkDNS.sh"},{"line_number":247,"context_line":"              subPath: utils-checkDNS.sh"},{"line_number":248,"context_line":"              readOnly: true"},{"line_number":249,"context_line":"            - name: ceph-osd-bin"},{"line_number":250,"context_line":"              mountPath: /tmp/utils-defragOSDs.sh"},{"line_number":251,"context_line":"              subPath: utils-defragOSDs.sh"},{"line_number":252,"context_line":"              readOnly: true"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"1f769fc5_464860cb","line":249,"in_reply_to":"1f769fc5_461ce069","updated":"2018-12-27 17:17:30.000000000","message":"The cronjob will execute the script on the pod, the script still needs to exist on the pod. You\u0027ll want to read the code to understand why that\u0027s the case.","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"2b2f1f8a7c57ffa6c47c4b739c12f5f81916afba","unresolved":false,"context_lines":[{"line_number":246,"context_line":"              mountPath: /tmp/utils-checkDNS.sh"},{"line_number":247,"context_line":"              subPath: utils-checkDNS.sh"},{"line_number":248,"context_line":"              readOnly: true"},{"line_number":249,"context_line":"            - name: ceph-osd-bin"},{"line_number":250,"context_line":"              mountPath: /tmp/utils-defragOSDs.sh"},{"line_number":251,"context_line":"              subPath: utils-defragOSDs.sh"},{"line_number":252,"context_line":"              readOnly: true"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"1f769fc5_aa080e28","line":249,"in_reply_to":"1f769fc5_464860cb","updated":"2018-12-28 18:59:08.000000000","message":"Yep, I\u0027d seen that but didn\u0027t put two and two together.  I\u0027m not completely sold on the \"one script, two contexts\" thing, but don\u0027t hate it enough to push back.  Is this a common pattern (not a scripting expert)?  It keeps the defrag-related script consolidated into one place, but in reality it\u0027s two very different scripts run by different actors.","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"}],"ceph-osd/values.yaml":[{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"36e1ce7905c0025a2a9133a3cb4db72f2e4371a3","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  cluster: 192.168.0.0/16"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"jobs:"},{"line_number":85,"context_line":"  ceph_defragOSDs:"},{"line_number":86,"context_line":"    # Execute the 1st of each month"},{"line_number":87,"context_line":"    cron: \"0 0 1 * *\""},{"line_number":88,"context_line":"    history:"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"1f769fc5_cb3a39e6","line":85,"updated":"2018-12-27 16:46:19.000000000","message":"nit: can we lowercase this guy for consistency? Likewise with the cronjob filename itself.","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"2b2f1f8a7c57ffa6c47c4b739c12f5f81916afba","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  cluster: 192.168.0.0/16"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"jobs:"},{"line_number":85,"context_line":"  ceph_defragOSDs:"},{"line_number":86,"context_line":"    # Execute the 1st of each month"},{"line_number":87,"context_line":"    cron: \"0 0 1 * *\""},{"line_number":88,"context_line":"    history:"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"1f769fc5_eafe0604","line":85,"in_reply_to":"1f769fc5_2619a4d6","updated":"2018-12-28 18:59:08.000000000","message":"Names of Kubernetes resources should be lower case by convention:\nhttps://kubernetes.io/docs/concepts/overview/working-with-objects/names/","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":29268,"name":"Matthew Heler","email":"matthew.heler@hotmail.com","username":"mheler"},"change_message_id":"ce50a8c56b0c675683bbf272b1aceb203ec7614f","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  cluster: 192.168.0.0/16"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"jobs:"},{"line_number":85,"context_line":"  ceph_defragOSDs:"},{"line_number":86,"context_line":"    # Execute the 1st of each month"},{"line_number":87,"context_line":"    cron: \"0 0 1 * *\""},{"line_number":88,"context_line":"    history:"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"1f769fc5_2619a4d6","line":85,"in_reply_to":"1f769fc5_cb3a39e6","updated":"2018-12-27 17:17:30.000000000","message":"It\u0027s OSD not osd... but for consistency sakes we match what is being doing with the checkPG\u0027s cronjob in the ceph-mon chart.","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":29268,"name":"Matthew Heler","email":"matthew.heler@hotmail.com","username":"mheler"},"change_message_id":"c1876f59dc69b3de7f8abb533aa43e36b72e77c2","unresolved":false,"context_lines":[{"line_number":82,"context_line":"  cluster: 192.168.0.0/16"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"jobs:"},{"line_number":85,"context_line":"  ceph_defragOSDs:"},{"line_number":86,"context_line":"    # Execute the 1st of each month"},{"line_number":87,"context_line":"    cron: \"0 0 1 * *\""},{"line_number":88,"context_line":"    history:"}],"source_content_type":"text/x-yaml","patch_set":7,"id":"1f769fc5_ca746a9c","line":85,"in_reply_to":"1f769fc5_eafe0604","updated":"2018-12-28 19:20:00.000000000","message":"fixed","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":22477,"name":"Matt McEuen","email":"matt.mceuen@att.com","username":"mattmceuen"},"change_message_id":"36e1ce7905c0025a2a9133a3cb4db72f2e4371a3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"1f769fc5_6b56852c","line":263,"updated":"2018-12-27 16:46:19.000000000","message":"ditto","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"},{"author":{"_account_id":29268,"name":"Matthew Heler","email":"matthew.heler@hotmail.com","username":"mheler"},"change_message_id":"c1876f59dc69b3de7f8abb533aa43e36b72e77c2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"1f769fc5_6a6996bc","line":263,"in_reply_to":"1f769fc5_6b56852c","updated":"2018-12-28 19:20:00.000000000","message":"fixed","commit_id":"13922dd15fea4b8ce35919b95cf72e41546b5cdb"}]}