)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":17589,"name":"Darla Ahlert","email":"da741q@att.com","username":"darla.ahlert"},"change_message_id":"d68b4a48d25d83aa418c5b487ff018cd5a38259e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"7f287b81_f12bfe91","updated":"2017-08-28 20:45:48.000000000","message":"Could you make sure your commit message follows the guidelines in [1]? Really I think you just need a blank line after your first line.\n\n[1]https://docs.openstack.org/infra/manual/developers.html#committing-a-change","commit_id":"b3f52180fcdfdae55646fe0a6680c1f3da2a5bf8"}],"ceph/templates/bin/_ceph-liveness-readiness.sh.tpl":[{"author":{"_account_id":17589,"name":"Darla Ahlert","email":"da741q@att.com","username":"darla.ahlert"},"change_message_id":"d68b4a48d25d83aa418c5b487ff018cd5a38259e","unresolved":false,"context_lines":[{"line_number":11,"context_line":"# default: no sockets, not live"},{"line_number":12,"context_line":"cond\u003d1"},{"line_number":13,"context_line":"for sock in $SOCKDIR/$SBASE.*.$SSUFFIX; do"},{"line_number":14,"context_line":" if [ -S $sock ]; then    "},{"line_number":15,"context_line":"  osdid\u003d`echo $sock | awk -F. \u0027{print $2}\u0027`"},{"line_number":16,"context_line":"  state\u003d`${CEPH} --connect-timeout 1 --admin-daemon $sock status|grep state|sed \u0027s/.*://;s/[^a-z]//g\u0027`"},{"line_number":17,"context_line":"  echo \"OSD $osdid $state\"; "}],"source_content_type":"text/x-smarty","patch_set":5,"id":"7f287b81_31618606","line":14,"range":{"start_line":14,"start_character":22,"end_line":14,"end_character":26},"updated":"2017-08-28 20:45:48.000000000","message":"Remove white space at end of line","commit_id":"b3f52180fcdfdae55646fe0a6680c1f3da2a5bf8"},{"author":{"_account_id":17589,"name":"Darla Ahlert","email":"da741q@att.com","username":"darla.ahlert"},"change_message_id":"d68b4a48d25d83aa418c5b487ff018cd5a38259e","unresolved":false,"context_lines":[{"line_number":14,"context_line":" if [ -S $sock ]; then    "},{"line_number":15,"context_line":"  osdid\u003d`echo $sock | awk -F. \u0027{print $2}\u0027`"},{"line_number":16,"context_line":"  state\u003d`${CEPH} --connect-timeout 1 --admin-daemon $sock status|grep state|sed \u0027s/.*://;s/[^a-z]//g\u0027`"},{"line_number":17,"context_line":"  echo \"OSD $osdid $state\"; "},{"line_number":18,"context_line":"  # this might be a stricter check than we actually want.  what are the"},{"line_number":19,"context_line":"  # other values for the \"state\" field?"},{"line_number":20,"context_line":"  if [ \"x${state}x\" \u003d \u0027xactivex\u0027 ]; then"}],"source_content_type":"text/x-smarty","patch_set":5,"id":"7f287b81_115e8a43","line":17,"range":{"start_line":17,"start_character":27,"end_line":17,"end_character":28},"updated":"2017-08-28 20:45:48.000000000","message":"ditto","commit_id":"b3f52180fcdfdae55646fe0a6680c1f3da2a5bf8"},{"author":{"_account_id":24165,"name":"Renis Makadia","email":"renis.makadia@att.com","username":"renmak"},"change_message_id":"c42f1f3eea508e12f732cbc3413935a03ea60851","unresolved":false,"context_lines":[{"line_number":2,"context_line":"# A liveness check for ceph OSDs: exit 0 iff"},{"line_number":3,"context_line":"# all OSDs on this host are in the \"active\" state"},{"line_number":4,"context_line":"# per their admin sockets."},{"line_number":5,"context_line":"CEPH\u003d${OSD_CEPH_CMD:-/usr/bin/ceph}"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"SOCKDIR\u003d${OSD_SOCKET_DIR:-/var/run/ceph}"},{"line_number":8,"context_line":"SBASE\u003d${OSD_SOCKET_BASE:-ceph-osd}"}],"source_content_type":"text/x-smarty","patch_set":6,"id":"7f287b81_22adaf9a","line":5,"range":{"start_line":5,"start_character":0,"end_line":5,"end_character":35},"updated":"2017-08-29 01:19:50.000000000","message":"Dave, this looks good. Only one change i would suggest is to follow OSH pattern on bash script and add start and stop functions. Please use this example as guide. functionhttps://review.openstack.org/#/c/469981/147/designate/templates/bin/_designate-bind9.sh.tpl","commit_id":"a7f44354e370f92166bc181679ef8afd2e7aff77"}],"ceph/templates/bin/_ceph-mon-liveness-readiness.sh.tpl":[{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"2208fdb30fc7bd793e46c06eed2286f3d26dc5d3","unresolved":false,"context_lines":[{"line_number":6,"context_line":"SBASE\u003d${CEPH_OSD_SOCKET_BASE:-ceph-mon}"},{"line_number":7,"context_line":"SSUFFIX\u003d${CEPH_SOCKET_SUFFIX:-asok}"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"mon_live_state\u003d\"probing electing synchronizing leader peon\""},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"monid\u003d`ps auwwx | grep ceph-mon | grep -v \"$1\" | grep -v grep | sed \u0027s/.*-i\\ *//;s/\\ *-.*//\u0027|awk \u0027{print $1}\u0027`"},{"line_number":12,"context_line":""}],"source_content_type":"text/x-smarty","patch_set":11,"id":"5f2577a7_5dc8e720","line":9,"updated":"2017-09-13 14:38:35.000000000","message":"Dave, given all of these states, what is an example state we would uncover where we would not consider ourselves ready?","commit_id":"d0ccd0a04b9fb84cd2cee4efce100b62c716cd65"},{"author":{"_account_id":26441,"name":"dave kormann","email":"davek@research.att.com","username":"multics"},"change_message_id":"83092814c45f95f46023a5da6198c63a314907dc","unresolved":false,"context_lines":[{"line_number":6,"context_line":"SBASE\u003d${CEPH_OSD_SOCKET_BASE:-ceph-mon}"},{"line_number":7,"context_line":"SSUFFIX\u003d${CEPH_SOCKET_SUFFIX:-asok}"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"mon_live_state\u003d\"probing electing synchronizing leader peon\""},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"monid\u003d`ps auwwx | grep ceph-mon | grep -v \"$1\" | grep -v grep | sed \u0027s/.*-i\\ *//;s/\\ *-.*//\u0027|awk \u0027{print $1}\u0027`"},{"line_number":12,"context_line":""}],"source_content_type":"text/x-smarty","patch_set":11,"id":"5f2577a7_439fe6b3","line":9,"in_reply_to":"5f2577a7_5dc8e720","updated":"2017-09-13 15:31:07.000000000","message":"I wrestled with that a bit myself and I\u0027m not sure.  While the stable states should be leader and peon, I believe a ready monitor could pass through any of these states at any time in its existence.  For example, when monitors move around, they and their peers would certainly at least enter \u0027electing\u0027, and possibly spend time in  \u0027probing\u0027 and \u0027synchronizing\u0027.\n\nThis page suggests as much:\n\nhttp://docs.ceph.com/docs/jewel/rados/troubleshooting/troubleshooting-mon/\n\nTo head off the inevitable followup, I\u0027m not entirely sure what states would indicate that the monitor is NOT ready or live, so in a sense I\u0027m really looking for any valid value here.","commit_id":"d0ccd0a04b9fb84cd2cee4efce100b62c716cd65"}],"ceph/templates/bin/_ceph-osd-liveness-readiness.sh.tpl":[{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"2208fdb30fc7bd793e46c06eed2286f3d26dc5d3","unresolved":false,"context_lines":[{"line_number":20,"context_line":"  if [ \"x${state}x\" \u003d \u0027xactivex\u0027 ]; then"},{"line_number":21,"context_line":"   cond\u003d0"},{"line_number":22,"context_line":"  else"},{"line_number":23,"context_line":"   # one\u0027s not ready, so the whole pod\u0027s not ready."},{"line_number":24,"context_line":"   exit 1"},{"line_number":25,"context_line":"  fi"},{"line_number":26,"context_line":" else"}],"source_content_type":"text/x-smarty","patch_set":11,"id":"5f2577a7_dd0e173a","line":23,"updated":"2017-09-13 14:38:35.000000000","message":"I understand the script and the logic - we would need to be careful of usage scenarios.  In other words, if this is used for liveness, and a pod is housing several OSDs and one is leveraging a physical disk that has failed - it may never be able to come up until the physical issue is resolved.  If this is a liveness check for the entire pod, after several failures it may determine the Pod needs to be terminated/recreated at which point one disk failure ends up impacting many OSDs.","commit_id":"d0ccd0a04b9fb84cd2cee4efce100b62c716cd65"},{"author":{"_account_id":26441,"name":"dave kormann","email":"davek@research.att.com","username":"multics"},"change_message_id":"27bc848e0b60fa43e619376fdd0beb7843043e9a","unresolved":false,"context_lines":[{"line_number":20,"context_line":"  if [ \"x${state}x\" \u003d \u0027xactivex\u0027 ]; then"},{"line_number":21,"context_line":"   cond\u003d0"},{"line_number":22,"context_line":"  else"},{"line_number":23,"context_line":"   # one\u0027s not ready, so the whole pod\u0027s not ready."},{"line_number":24,"context_line":"   exit 1"},{"line_number":25,"context_line":"  fi"},{"line_number":26,"context_line":" else"}],"source_content_type":"text/x-smarty","patch_set":11,"id":"5f2577a7_c85f6e70","line":23,"in_reply_to":"5f2577a7_a84b0229","updated":"2017-09-13 17:06:11.000000000","message":"That confirms my vague intuition about how liveness/readiness works. \n Where I\u0027m not sure about that for monitors is that I think that it might be perfectly normal to see a monitor in one of the other (non leader) states even though it\u0027s quite happily serving requests.  The cases I\u0027m thinking of are when elections get called.  I\u0027m pretty sure these happen both routinely and when, for example, a monitor is added or removed.  While the election is running (which may take a bit of time, paxos not being particularly cheap), I think you\u0027ll at least see things in the \u0027electing\u0027 state, and perhaps one of the others.\n\nThat said, I don\u0027t think there\u0027d be anything wrong with calling those states \u0027ready\u0027 but not \u0027live\u0027.  Rather than use the same script for both, I\u0027ll just split it in two with different states, unless there\u0027s a particular objection to adding more scripts.","commit_id":"d0ccd0a04b9fb84cd2cee4efce100b62c716cd65"},{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"de67292ae919b72efe8b342c857d6b4d5e15aaba","unresolved":false,"context_lines":[{"line_number":20,"context_line":"  if [ \"x${state}x\" \u003d \u0027xactivex\u0027 ]; then"},{"line_number":21,"context_line":"   cond\u003d0"},{"line_number":22,"context_line":"  else"},{"line_number":23,"context_line":"   # one\u0027s not ready, so the whole pod\u0027s not ready."},{"line_number":24,"context_line":"   exit 1"},{"line_number":25,"context_line":"  fi"},{"line_number":26,"context_line":" else"}],"source_content_type":"text/x-smarty","patch_set":11,"id":"5f2577a7_a84b0229","line":23,"in_reply_to":"5f2577a7_c372b6e3","updated":"2017-09-13 16:53:56.000000000","message":"I would describe readiness as \"should I add this pod as a member of a service endpoint to start getting requests\" as well as \"if other things depend on me and will try and find a ready version of me in Kubernetes, I would call myself ready\" and liveness as \"is this pod sick and I should kill it to let it restart\"\n\nWhen we\u0027re talking about OSDs and MONs it gets a bit tricker to define readiness as these pods are part of a larger organism.  As an armchair comment I might consider any MON in a leader or peon role as ready, where as any MON not returning any of the states as failing liveness and in need of being poked. \n\nFor OSDs, all signs point toward one OSD in one pod so I think your current check seems fine","commit_id":"d0ccd0a04b9fb84cd2cee4efce100b62c716cd65"},{"author":{"_account_id":26441,"name":"dave kormann","email":"davek@research.att.com","username":"multics"},"change_message_id":"83092814c45f95f46023a5da6198c63a314907dc","unresolved":false,"context_lines":[{"line_number":20,"context_line":"  if [ \"x${state}x\" \u003d \u0027xactivex\u0027 ]; then"},{"line_number":21,"context_line":"   cond\u003d0"},{"line_number":22,"context_line":"  else"},{"line_number":23,"context_line":"   # one\u0027s not ready, so the whole pod\u0027s not ready."},{"line_number":24,"context_line":"   exit 1"},{"line_number":25,"context_line":"  fi"},{"line_number":26,"context_line":" else"}],"source_content_type":"text/x-smarty","patch_set":11,"id":"5f2577a7_c372b6e3","line":23,"in_reply_to":"5f2577a7_dd0e173a","updated":"2017-09-13 15:31:07.000000000","message":"Two thoughts: first, I\u0027m mostly thinking about the model where each OSD has its own pod, so _hopefully_ a pod getting gunned by this script isn\u0027t catastrophic but the point is well-taken.  Second, that may suggest a need for distinguishing between liveness and readiness.  I haven\u0027t really got a handle on how to distinguish the two for OSDs, but maybe it would allow for better logic here.","commit_id":"d0ccd0a04b9fb84cd2cee4efce100b62c716cd65"}],"ceph/templates/daemonset-mon.yaml":[{"author":{"_account_id":8898,"name":"Chris Wedgwood","email":"cw@f00f.org","username":"anticw"},"change_message_id":"4ba929d85710107f158a97c77aa2815b16ac9474","unresolved":false,"context_lines":[{"line_number":104,"context_line":"            command:"},{"line_number":105,"context_line":"             - /ceph-mon-readiness-readiness.sh"},{"line_number":106,"context_line":"           initialDelaySeconds: 60"},{"line_number":107,"context_line":"           periodSeconds: 15"},{"line_number":108,"context_line":"          volumeMounts:"},{"line_number":109,"context_line":"            - name: ceph-bin"},{"line_number":110,"context_line":"              mountPath: /start_mon.sh"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"ff82abbf_5c82ae01","line":107,"updated":"2017-11-29 17:02:13.000000000","message":"how heavy weight are these scripts?\n\ndo we need to run that often?","commit_id":"9319aacf41fb1418708bbf0e03358c1458cca999"},{"author":{"_account_id":26441,"name":"dave kormann","email":"davek@research.att.com","username":"multics"},"change_message_id":"44639b35c964d88df05f05a1fdc4889a6427f922","unresolved":false,"context_lines":[{"line_number":104,"context_line":"            command:"},{"line_number":105,"context_line":"             - /ceph-mon-readiness-readiness.sh"},{"line_number":106,"context_line":"           initialDelaySeconds: 60"},{"line_number":107,"context_line":"           periodSeconds: 15"},{"line_number":108,"context_line":"          volumeMounts:"},{"line_number":109,"context_line":"            - name: ceph-bin"},{"line_number":110,"context_line":"              mountPath: /start_mon.sh"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"ff82abbf_1c8bc6c2","line":107,"in_reply_to":"ff82abbf_5c82ae01","updated":"2017-11-29 17:17:19.000000000","message":"the scripts are very cheap — they just make a status request against the admin unix domain socket — but I have no objection to running less often; the only effect will be on how quickly we decide things are up and running, and how rapidly we notice they are broken.\n\nFeel free to suggest a frequency.  I don\u0027t really have a good intuition about what the right number is.","commit_id":"9319aacf41fb1418708bbf0e03358c1458cca999"}],"ceph/templates/daemonset-osd.yaml":[{"author":{"_account_id":7769,"name":"Pentheus","display_name":"Alan Meadows","email":"alan.meadows@gmail.com","username":"alanmeadows"},"change_message_id":"2208fdb30fc7bd793e46c06eed2286f3d26dc5d3","unresolved":false,"context_lines":[{"line_number":74,"context_line":"          livenessProbe:"},{"line_number":75,"context_line":"           exec:"},{"line_number":76,"context_line":"            command:"},{"line_number":77,"context_line":"             - /ceph-osd-liveness-readiness.sh"},{"line_number":78,"context_line":"           initialDelaySeconds: 60"},{"line_number":79,"context_line":"           periodSeconds: 15"},{"line_number":80,"context_line":"          readinessProbe:"}],"source_content_type":"text/x-yaml","patch_set":11,"id":"5f2577a7_98365de6","line":77,"updated":"2017-09-13 14:38:35.000000000","message":"See comment regarding liveness usage of this script.","commit_id":"d0ccd0a04b9fb84cd2cee4efce100b62c716cd65"},{"author":{"_account_id":8898,"name":"Chris Wedgwood","email":"cw@f00f.org","username":"anticw"},"change_message_id":"4ba929d85710107f158a97c77aa2815b16ac9474","unresolved":false,"context_lines":[{"line_number":82,"context_line":"            command:"},{"line_number":83,"context_line":"             - /ceph-osd-liveness-readiness.sh"},{"line_number":84,"context_line":"           initialDelaySeconds: 60"},{"line_number":85,"context_line":"           periodSeconds: 5"},{"line_number":86,"context_line":"          volumeMounts:"},{"line_number":87,"context_line":"            - name: devices"},{"line_number":88,"context_line":"              mountPath: /dev"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"ff82abbf_3c7b2a28","line":85,"updated":"2017-11-29 17:02:13.000000000","message":"5s is very frequently ...","commit_id":"9319aacf41fb1418708bbf0e03358c1458cca999"},{"author":{"_account_id":26441,"name":"dave kormann","email":"davek@research.att.com","username":"multics"},"change_message_id":"44639b35c964d88df05f05a1fdc4889a6427f922","unresolved":false,"context_lines":[{"line_number":82,"context_line":"            command:"},{"line_number":83,"context_line":"             - /ceph-osd-liveness-readiness.sh"},{"line_number":84,"context_line":"           initialDelaySeconds: 60"},{"line_number":85,"context_line":"           periodSeconds: 5"},{"line_number":86,"context_line":"          volumeMounts:"},{"line_number":87,"context_line":"            - name: devices"},{"line_number":88,"context_line":"              mountPath: /dev"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"ff82abbf_1cb4267f","line":85,"in_reply_to":"ff82abbf_3c7b2a28","updated":"2017-11-29 17:17:19.000000000","message":"Yep.  See the comments for the other readiness probe — this check is cheap, but I\u0027d happily take suggestions for appropriate values.","commit_id":"9319aacf41fb1418708bbf0e03358c1458cca999"}]}