)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"b3cd1716c186f86e916b7da119723353c6d5bc22","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Document ceph cluster maintenance"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"- Test effect of Node filure and expansion on"},{"line_number":10,"context_line":"Ceph cluster"},{"line_number":11,"context_line":"- Document steps to fix/recover Ceph cluster after"},{"line_number":12,"context_line":"node failure and expansion"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":19,"id":"3f79a3b5_e07116e0","line":9,"range":{"start_line":9,"start_character":22,"end_line":9,"end_character":28},"updated":"2018-08-21 15:16:48.000000000","message":"failure","commit_id":"74cbf83e79ec5978f09bc0c519d7ad482f786678"},{"author":{"_account_id":8898,"name":"Chris Wedgwood","email":"cw@f00f.org","username":"anticw"},"change_message_id":"809b41a584e6933bed57f71321998e7216b3263d","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Document ceph cluster maintenance"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"- Test effect of Node filure and expansion on"},{"line_number":10,"context_line":"Ceph cluster"},{"line_number":11,"context_line":"- Document steps to fix/recover Ceph cluster after"},{"line_number":12,"context_line":"node failure and expansion"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":21,"id":"3f79a3b5_c95e8544","line":9,"range":{"start_line":9,"start_character":22,"end_line":9,"end_character":28},"updated":"2018-10-16 14:28:13.000000000","message":"failure","commit_id":"494cff181bcf1359c9b7472b31a44a5a34076bf8"},{"author":{"_account_id":8898,"name":"Chris Wedgwood","email":"cw@f00f.org","username":"anticw"},"change_message_id":"809b41a584e6933bed57f71321998e7216b3263d","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Document ceph cluster maintenance"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"- Test effect of Node filure and expansion on"},{"line_number":10,"context_line":"Ceph cluster"},{"line_number":11,"context_line":"- Document steps to fix/recover Ceph cluster after"},{"line_number":12,"context_line":"node failure and expansion"},{"line_number":13,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":21,"id":"3f79a3b5_89588d56","line":10,"range":{"start_line":10,"start_character":0,"end_line":10,"end_character":12},"updated":"2018-10-16 14:28:13.000000000","message":"missing indent","commit_id":"494cff181bcf1359c9b7472b31a44a5a34076bf8"},{"author":{"_account_id":8898,"name":"Chris Wedgwood","email":"cw@f00f.org","username":"anticw"},"change_message_id":"809b41a584e6933bed57f71321998e7216b3263d","unresolved":false,"context_lines":[{"line_number":9,"context_line":"- Test effect of Node filure and expansion on"},{"line_number":10,"context_line":"Ceph cluster"},{"line_number":11,"context_line":"- Document steps to fix/recover Ceph cluster after"},{"line_number":12,"context_line":"node failure and expansion"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I5fa952bef4934974d5657bac05c887564b7c471a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":21,"id":"3f79a3b5_a9530939","line":12,"range":{"start_line":12,"start_character":0,"end_line":12,"end_character":26},"updated":"2018-10-16 14:28:13.000000000","message":"indent","commit_id":"494cff181bcf1359c9b7472b31a44a5a34076bf8"}],"doc/source/testing/ceph-node-resiliency.rst":[{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"e663346af25474b9a2ad9cc5a9042f60cb566563","unresolved":false,"context_lines":[{"line_number":2,"context_line":"Ceph - Node Reduction, Expansion and Recovery"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"In this test documentation, we will test effect of node reduction, expansion and"},{"line_number":6,"context_line":"recovery on Ceph cluster."},{"line_number":7,"context_line":""},{"line_number":8,"context_line":"Test Scenario:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5f7c97a3_ff6046a0","line":5,"updated":"2018-08-01 20:53:14.000000000","message":"Also describe that the node in question has the ceph-mon and ceph-osd on it.\n\nIn the test scenario section, describe the two test cases:\n1. a node has a temporary outage.  While the node was out of the kubernetes cluster, the HEALTH_WARN is observed.  Observe the OSDs that were hosted on the departed node has \"out\" status in the \"ceph osd tree\".  Observe the monitor status in the \"ceph quorum_status\" and \"ceph mon_status\".  Once the node re-join the kubernetes cluster, verify the OSDs and mon node joined successfully.\n2. The departed node had to be replaced completely.  What are the openstack-helm and ceph charts that need to run on the replacement node?  Let the mon pod joins the ceph cluster successfully.  Next, delete the old OSDs that were on the departed node from the crush map.  Add the new OSDs on the replacement node.  The new OSD could re-use the OSD numbers you just deleted.  Verify the new OSDs join the ceph cluster and \"ceph osd tree\" shows the new OSDs are associated with the \"replacement\" node.","commit_id":"ed5177c4c899ccb5505eb5fa1b8dcfbc06f3bbaf"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"e663346af25474b9a2ad9cc5a9042f60cb566563","unresolved":false,"context_lines":[{"line_number":4,"context_line":""},{"line_number":5,"context_line":"In this test documentation, we will test effect of node reduction, expansion and"},{"line_number":6,"context_line":"recovery on Ceph cluster."},{"line_number":7,"context_line":""},{"line_number":8,"context_line":"Test Scenario:"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":"From initial healthy Ceph and Openstack deployment, test effect of node lost (reduction),"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5f7c97a3_1f698224","line":7,"updated":"2018-08-01 20:53:14.000000000","message":"The ceph-mon pods are fixed on certain nodes and these pods do not migrate.  The are configured as daemonset.  Might be useful to explain the daemonset and deployment with replica.","commit_id":"ed5177c4c899ccb5505eb5fa1b8dcfbc06f3bbaf"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"e663346af25474b9a2ad9cc5a9042f60cb566563","unresolved":false,"context_lines":[{"line_number":7,"context_line":""},{"line_number":8,"context_line":"Test Scenario:"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":"From initial healthy Ceph and Openstack deployment, test effect of node lost (reduction),"},{"line_number":11,"context_line":"then adding new node for Ceph deployment (expansion) and lastly recover node that"},{"line_number":12,"context_line":"was lost earlier (recovery)"},{"line_number":13,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"5f7c97a3_ff9fe64a","line":10,"updated":"2018-08-01 20:53:14.000000000","message":"Itemize the the test scenarios:\n1. ...\n2. ...","commit_id":"ed5177c4c899ccb5505eb5fa1b8dcfbc06f3bbaf"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"e663346af25474b9a2ad9cc5a9042f60cb566563","unresolved":false,"context_lines":[{"line_number":61,"context_line":""},{"line_number":62,"context_line":".. code-block:: console"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"  ubuntu@mnode1:/opt/openstack-helm$ kubectl exec -n ceph ceph-mon-bfczf -- ceph quorum_status"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"  {\"election_epoch\":8,\"quorum\":[0,1,2],\"quorum_names\":[\"mnode1\",\"mnode2\",\"mnode3\"],"},{"line_number":67,"context_line":"  \"quorum_leader_name\":\"mnode1\",\"monmap\":{\"epoch\":1,\"fsid\":\"c533bb28-f729-487f-b8c2-e50b62ba2a4d\","}],"source_content_type":"text/x-rst","patch_set":3,"id":"5f7c97a3_5fdffa09","line":64,"updated":"2018-08-01 20:53:14.000000000","message":"Can you use the json pretty format \"ceph quorum_status -f json-pretty\"?","commit_id":"ed5177c4c899ccb5505eb5fa1b8dcfbc06f3bbaf"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"e663346af25474b9a2ad9cc5a9042f60cb566563","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":".. code-block:: console"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"  ubuntu@mnode1:~$ kubectl exec -n ceph ceph-mon-bfczf -- ceph quorum_status"},{"line_number":223,"context_line":"  {\"election_epoch\":1792,\"quorum\":[0,1],\"quorum_names\":[\"mnode1\",\"mnode2\"],"},{"line_number":224,"context_line":"  \"quorum_leader_name\":\"mnode1\",\"monmap\":{\"epoch\":12,\"fsid\":\"c533bb28-f729-487f-b8c2-e50b62ba2a4d\","},{"line_number":225,"context_line":"  \"modified\":\"2018-07-27 13:23:53.641051\",\"created\":\"2018-07-27 13:23:26.284077\","}],"source_content_type":"text/x-rst","patch_set":3,"id":"5f7c97a3_9f27d21e","line":222,"updated":"2018-08-01 20:53:14.000000000","message":"add \"-f json-pretty\" to print in json format","commit_id":"ed5177c4c899ccb5505eb5fa1b8dcfbc06f3bbaf"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"e663346af25474b9a2ad9cc5a9042f60cb566563","unresolved":false,"context_lines":[{"line_number":520,"context_line":""},{"line_number":521,"context_line":".. code-block:: console"},{"line_number":522,"context_line":""},{"line_number":523,"context_line":"  ubuntu@mnode1:~$ kubectl exec -n ceph ceph-mon-bfczf -- ceph quorum_status"},{"line_number":524,"context_line":""},{"line_number":525,"context_line":"  {\"election_epoch\":1802,\"quorum\":[0,1,2,3],\"quorum_names\":[\"mnode1\",\"mnode2\",\"mnode3\",\"mnode4\"],"},{"line_number":526,"context_line":"  \"quorum_leader_name\":\"mnode1\",\"monmap\":{\"epoch\":13,\"fsid\":\"c533bb28-f729-487f-b8c2-e50b62ba2a4d\","}],"source_content_type":"text/x-rst","patch_set":3,"id":"5f7c97a3_9f1032ad","line":523,"updated":"2018-08-01 20:53:14.000000000","message":"\"-f json-pretty\"","commit_id":"ed5177c4c899ccb5505eb5fa1b8dcfbc06f3bbaf"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"e663346af25474b9a2ad9cc5a9042f60cb566563","unresolved":false,"context_lines":[{"line_number":636,"context_line":""},{"line_number":637,"context_line":"``Result/Observation:``"},{"line_number":638,"context_line":""},{"line_number":639,"context_line":"- Node successfully got recovered."},{"line_number":640,"context_line":"- Ceph mon and osd recovered on the node"},{"line_number":641,"context_line":"- Ceph status shows HEALTH_OK as previously lost Mon and OSD pods are scheduled."}],"source_content_type":"text/x-rst","patch_set":3,"id":"5f7c97a3_7f099680","line":639,"updated":"2018-08-01 20:53:14.000000000","message":"Should we show the \"ceph status\" output to show the recovery in ceph cluster?","commit_id":"ed5177c4c899ccb5505eb5fa1b8dcfbc06f3bbaf"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"b3cd1716c186f86e916b7da119723353c6d5bc22","unresolved":false,"context_lines":[{"line_number":8,"context_line":"Setup:"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":"- 6 Nodes (VM based) env"},{"line_number":11,"context_line":"- Only 3 nodes will have Ceph and Openstack related labels."},{"line_number":12,"context_line":"- Followed OSH multinode guide steps to setup nodes and install k8 cluster"},{"line_number":13,"context_line":"- Followed OSH multinode guide steps to install Ceph and Openstack charts up to"},{"line_number":14,"context_line":"  Cinder."}],"source_content_type":"text/x-rst","patch_set":19,"id":"3f79a3b5_a07b1ebc","line":11,"range":{"start_line":11,"start_character":34,"end_line":11,"end_character":43},"updated":"2018-08-21 15:16:48.000000000","message":"nit: OpenStack (everywhere in the document)","commit_id":"74cbf83e79ec5978f09bc0c519d7ad482f786678"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"b3cd1716c186f86e916b7da119723353c6d5bc22","unresolved":false,"context_lines":[{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":"- 6 Nodes (VM based) env"},{"line_number":11,"context_line":"- Only 3 nodes will have Ceph and Openstack related labels."},{"line_number":12,"context_line":"- Followed OSH multinode guide steps to setup nodes and install k8 cluster"},{"line_number":13,"context_line":"- Followed OSH multinode guide steps to install Ceph and Openstack charts up to"},{"line_number":14,"context_line":"  Cinder."},{"line_number":15,"context_line":""}],"source_content_type":"text/x-rst","patch_set":19,"id":"3f79a3b5_e05a764e","line":12,"range":{"start_line":12,"start_character":64,"end_line":12,"end_character":66},"updated":"2018-08-21 15:16:48.000000000","message":"noob question here: Why the spelling k8, K8? Wouldn\u0027t it be \"k8s\"?","commit_id":"74cbf83e79ec5978f09bc0c519d7ad482f786678"},{"author":{"_account_id":22713,"name":"Sreejith Punnapuzha","email":"Sreejith.Punnapuzha@outlook.com","username":"sreejith"},"change_message_id":"cd359c3759354c10f2d2d217402de6ab5a59cda4","unresolved":false,"context_lines":[{"line_number":134,"context_line":"              {"},{"line_number":135,"context_line":"                  \"rank\": 1,"},{"line_number":136,"context_line":"                  \"name\": \"mnode2\","},{"line_number":137,"context_line":"                  \"addr\": \"135.91.204.247:6789/0\","},{"line_number":138,"context_line":"                  \"public_addr\": \"135.91.204.247:6789/0\""},{"line_number":139,"context_line":"              },"},{"line_number":140,"context_line":"              {"}],"source_content_type":"text/x-rst","patch_set":19,"id":"3f79a3b5_5dfccf0b","line":137,"range":{"start_line":137,"start_character":27,"end_line":137,"end_character":41},"updated":"2018-08-28 14:35:47.000000000","message":"Please remove all the internal ips from the output.","commit_id":"74cbf83e79ec5978f09bc0c519d7ad482f786678"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"72ce33efa1075a5be8c7c13c4cb901cc61eb3a86","unresolved":false,"context_lines":[{"line_number":1,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":2,"context_line":"Ceph - Node Reduction, Expansion and Ceph Recovery"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"This document captures steps and result from node reduction and expansion as"},{"line_number":6,"context_line":"well as ceph recovery."},{"line_number":7,"context_line":""}],"source_content_type":"text/x-rst","patch_set":21,"id":"3f79a3b5_5539dfd5","line":4,"updated":"2018-08-29 23:50:43.000000000","message":"Can you add the table-of-content for this document with these lines.\n\n.. contents::\n.. sectnum::","commit_id":"494cff181bcf1359c9b7472b31a44a5a34076bf8"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"72ce33efa1075a5be8c7c13c4cb901cc61eb3a86","unresolved":false,"context_lines":[{"line_number":12,"context_line":"- Followed OSH multinode guide steps to setup nodes and install k8 cluster"},{"line_number":13,"context_line":"- Followed OSH multinode guide steps to install Ceph and OpenStack charts up to"},{"line_number":14,"context_line":"  Cinder."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Test Scenarios:"},{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":18,"context_line":"1) Node reduction: Shutdown 1 of 3 nodes to simulate node failure. Capture effect"}],"source_content_type":"text/x-rst","patch_set":21,"id":"3f79a3b5_956d37ce","line":15,"updated":"2018-08-29 23:50:43.000000000","message":"Can you add a comment here explaining that a node has both MON and OSD pods running on it.","commit_id":"494cff181bcf1359c9b7472b31a44a5a34076bf8"},{"author":{"_account_id":28704,"name":"Al Lau","email":"al070e@att.com","username":"al070e"},"change_message_id":"72ce33efa1075a5be8c7c13c4cb901cc61eb3a86","unresolved":false,"context_lines":[{"line_number":45,"context_line":"4) Ceph cluster recovery:"},{"line_number":46,"context_line":"Perform Ceph maintenance to make Ceph cluster HEALTH_OK. Remove lost MON and"},{"line_number":47,"context_line":"OSD from Ceph cluster."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"Step 1: Initial Ceph and OpenStack deployment"},{"line_number":51,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":21,"id":"3f79a3b5_35b30350","line":48,"updated":"2018-08-29 23:50:43.000000000","message":"Run the ceph commands like \"ceph -s \" and \"ceph osd tree\" on the MON POD that is scheduled on the new mnode to see correct output.  Do the same on the OSD POD that is scheduled on the new mnode by running \u0027ceph-disk list\u0027 and \u0027lsblk\u0027.","commit_id":"494cff181bcf1359c9b7472b31a44a5a34076bf8"}]}