)]}'
{"doc/ops-guide/source/ops_maintenance_rabbitmq.rst":[{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"ea53d15b446be18d352abfc2d892808832f524c0","unresolved":false,"context_lines":[{"line_number":1,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":2,"context_line":"RabbitMQ Troubleshooting"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"RabbitMQ is an Advanced Message Queuing Protocol (AMQP) messaging broker"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1aa78d24_bf6863f6","line":2,"range":{"start_line":2,"start_character":9,"end_line":2,"end_character":10},"updated":"2016-07-12 08:30:51.000000000","message":"lowercase t","commit_id":"73127459f92fe75d8ccbf37059804ec91d63ad05"},{"author":{"_account_id":17207,"name":"Atsushi SAKAI","email":"sakaia@jp.fujitsu.com","username":"sakaia"},"change_message_id":"287013a70dc068fe809583136c876440f7be1c82","unresolved":false,"context_lines":[{"line_number":74,"context_line":"If you receive alerts for RabbitMQ, take the following steps to troubleshoot"},{"line_number":75,"context_line":"and resolve the issue:"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"#. Determine which servers the Rabbit MQ alarms are coming from."},{"line_number":78,"context_line":"#. Depending on the environment the servers are in, open up your Dashboard"},{"line_number":79,"context_line":"   and launch an instance."},{"line_number":80,"context_line":"#. If you cannot launch an instance, continue to troubleshoot the issue."}],"source_content_type":"text/x-rst","patch_set":3,"id":"1aa78d24_bbd358d4","line":77,"updated":"2016-07-12 08:09:53.000000000","message":"Rabbit MQ \u003d\u003e RabbitMQ","commit_id":"73127459f92fe75d8ccbf37059804ec91d63ad05"},{"author":{"_account_id":19814,"name":"Travis Truman (automagically)","email":"travis_truman@cable.comcast.com","username":"trumant"},"change_message_id":"e7b21544fc04be1463278fd8b015378b35570649","unresolved":false,"context_lines":[{"line_number":1,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":2,"context_line":"RabbitMQ troubleshooting"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_957531fb","line":1,"updated":"2016-07-13 15:50:49.000000000","message":"Superfluous. This is one installation pattern, but there are others.","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":19814,"name":"Travis Truman (automagically)","email":"travis_truman@cable.comcast.com","username":"trumant"},"change_message_id":"e7b21544fc04be1463278fd8b015378b35570649","unresolved":false,"context_lines":[{"line_number":15,"context_line":"~~~~~~~~~~~~~~~~~~~~~~~~"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"It is quite common for the RabbitMQ service to hang when"},{"line_number":18,"context_line":"restarting or stopping a service. It is highly recommended that"},{"line_number":19,"context_line":"you manually restart RabbitMQ on each controller node."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. Restart the RabbitMQ server on the first controller node:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_f5b63587","line":18,"updated":"2016-07-13 15:50:49.000000000","message":"which service? This is vague. Its unclear if you mean that RabbitMQ hangs when I restart another service.","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":15,"context_line":"~~~~~~~~~~~~~~~~~~~~~~~~"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"It is quite common for the RabbitMQ service to hang when"},{"line_number":18,"context_line":"restarting or stopping a service. It is highly recommended that"},{"line_number":19,"context_line":"you manually restart RabbitMQ on each controller node."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. Restart the RabbitMQ server on the first controller node:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_6cd9c5f4","line":18,"in_reply_to":"1aa78d24_f5b63587","updated":"2016-07-14 02:32:29.000000000","message":"TBC","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10497,"name":"KATO Tomoyuki","email":"kato.tomoyuki@jp.fujitsu.com","username":"to222"},"change_message_id":"9b7e49713b8b97789e1001ee398fc23f63b0cd29","unresolved":false,"context_lines":[{"line_number":18,"context_line":"restarting or stopping a service. It is highly recommended that"},{"line_number":19,"context_line":"you manually restart RabbitMQ on each controller node."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. Restart the RabbitMQ server on the first controller node:"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"   .. code-block:: console"},{"line_number":24,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_ee7968c9","line":21,"range":{"start_line":21,"start_character":24,"end_line":21,"end_character":30},"updated":"2016-07-13 05:38:32.000000000","message":"service?","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":18,"context_line":"restarting or stopping a service. It is highly recommended that"},{"line_number":19,"context_line":"you manually restart RabbitMQ on each controller node."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. Restart the RabbitMQ server on the first controller node:"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"   .. code-block:: console"},{"line_number":24,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_ace3ada3","line":21,"range":{"start_line":21,"start_character":24,"end_line":21,"end_character":30},"in_reply_to":"1aa78d24_d16fed8f","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":22165,"name":"caoyuan","email":"cao.yingjunz@gmail.com","username":"caoyuan"},"change_message_id":"93ee9e7181cabae43e285212325718cf380dc0e6","unresolved":false,"context_lines":[{"line_number":18,"context_line":"restarting or stopping a service. It is highly recommended that"},{"line_number":19,"context_line":"you manually restart RabbitMQ on each controller node."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. Restart the RabbitMQ server on the first controller node:"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"   .. code-block:: console"},{"line_number":24,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_d16fed8f","line":21,"range":{"start_line":21,"start_character":24,"end_line":21,"end_character":30},"in_reply_to":"1aa78d24_ee7968c9","updated":"2016-07-13 06:19:18.000000000","message":"+1","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10497,"name":"KATO Tomoyuki","email":"kato.tomoyuki@jp.fujitsu.com","username":"to222"},"change_message_id":"9b7e49713b8b97789e1001ee398fc23f63b0cd29","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"   .. code-block:: console"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"      service rabbitmq-server stop"},{"line_number":26,"context_line":"      service rabbitmq-server start"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_aeb48081","line":25,"range":{"start_line":25,"start_character":6,"end_line":25,"end_character":13},"updated":"2016-07-13 05:38:32.000000000","message":"# service ...","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"   .. code-block:: console"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"      service rabbitmq-server stop"},{"line_number":26,"context_line":"      service rabbitmq-server start"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_0ca2c14b","line":25,"range":{"start_line":25,"start_character":6,"end_line":25,"end_character":13},"in_reply_to":"1aa78d24_aeb48081","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":17130,"name":"melissaml","email":"ma.lei@99cloud.net","username":"malei"},"change_message_id":"2c5e7270845a691b0e7952cdc9969705424aa1df","unresolved":false,"context_lines":[{"line_number":23,"context_line":"   .. code-block:: console"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"      service rabbitmq-server stop"},{"line_number":26,"context_line":"      service rabbitmq-server start"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"#. If the service refuses to stop, then run the `pkill` command"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_3148f1cb","line":26,"range":{"start_line":26,"start_character":6,"end_line":26,"end_character":13},"updated":"2016-07-13 06:11:15.000000000","message":"# service ...","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":22165,"name":"caoyuan","email":"cao.yingjunz@gmail.com","username":"caoyuan"},"change_message_id":"93ee9e7181cabae43e285212325718cf380dc0e6","unresolved":false,"context_lines":[{"line_number":23,"context_line":"   .. code-block:: console"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"      service rabbitmq-server stop"},{"line_number":26,"context_line":"      service rabbitmq-server start"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"#. If the service refuses to stop, then run the `pkill` command"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_1170553f","line":26,"range":{"start_line":26,"start_character":6,"end_line":26,"end_character":35},"updated":"2016-07-13 06:19:18.000000000","message":"Generally， we use the command \"systemctl restart rabbitmq-server.service\"  to restart the service ，when it faild ，run .......","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10497,"name":"KATO Tomoyuki","email":"kato.tomoyuki@jp.fujitsu.com","username":"to222"},"change_message_id":"9b7e49713b8b97789e1001ee398fc23f63b0cd29","unresolved":false,"context_lines":[{"line_number":26,"context_line":"      service rabbitmq-server start"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"#. If the service refuses to stop, then run the `pkill` command"},{"line_number":30,"context_line":"   to stop the service, then restart the service:"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"   .. code-block:: console"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_0ea4b448","line":29,"range":{"start_line":29,"start_character":48,"end_line":29,"end_character":55},"updated":"2016-07-13 05:38:32.000000000","message":":command:`pkill`","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":26,"context_line":"      service rabbitmq-server start"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"#. If the service refuses to stop, then run the `pkill` command"},{"line_number":30,"context_line":"   to stop the service, then restart the service:"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"   .. code-block:: console"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_ccbef963","line":29,"range":{"start_line":29,"start_character":48,"end_line":29,"end_character":55},"in_reply_to":"1aa78d24_0ea4b448","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10497,"name":"KATO Tomoyuki","email":"kato.tomoyuki@jp.fujitsu.com","username":"to222"},"change_message_id":"9b7e49713b8b97789e1001ee398fc23f63b0cd29","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"   .. code-block:: console"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"      pkill -KILL -u rabbitmq"},{"line_number":35,"context_line":"      service rabbitmq-server start"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_eeb20893","line":34,"range":{"start_line":34,"start_character":6,"end_line":34,"end_character":11},"updated":"2016-07-13 05:38:32.000000000","message":"ditto - overall","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"   .. code-block:: console"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"      pkill -KILL -u rabbitmq"},{"line_number":35,"context_line":"      service rabbitmq-server start"},{"line_number":36,"context_line":""},{"line_number":37,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_ecb9357e","line":34,"range":{"start_line":34,"start_character":6,"end_line":34,"end_character":11},"in_reply_to":"1aa78d24_eeb20893","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":17130,"name":"melissaml","email":"ma.lei@99cloud.net","username":"malei"},"change_message_id":"2c5e7270845a691b0e7952cdc9969705424aa1df","unresolved":false,"context_lines":[{"line_number":43,"context_line":"      rabbitmqctl list_queues"},{"line_number":44,"context_line":"      rabbitmqctl list_queues 2\u003e\u00261 | grep -i error"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"#. If there are errors, go back to the first step and try"},{"line_number":47,"context_line":"   restarting RabbitMQ again. If there are no errors, restart"},{"line_number":48,"context_line":"   RabbitMQ server on the next controller node."},{"line_number":49,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_b158618b","line":46,"range":{"start_line":46,"start_character":12,"end_line":46,"end_character":15},"updated":"2016-07-13 06:11:15.000000000","message":"there are some errors","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":19814,"name":"Travis Truman (automagically)","email":"travis_truman@cable.comcast.com","username":"trumant"},"change_message_id":"e7b21544fc04be1463278fd8b015378b35570649","unresolved":false,"context_lines":[{"line_number":47,"context_line":"   restarting RabbitMQ again. If there are no errors, restart"},{"line_number":48,"context_line":"   RabbitMQ server on the next controller node."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"#. After restartung RabbitMQ on all controller nodes, restart services"},{"line_number":51,"context_line":"   on each controller node. For example:"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"   .. code-block:: console"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_e8093c2c","line":50,"updated":"2016-07-13 15:50:49.000000000","message":"Is this really necessary? I\u0027d expect oslo.messaging to be smart enough to reconnect.","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":17130,"name":"melissaml","email":"ma.lei@99cloud.net","username":"malei"},"change_message_id":"2c5e7270845a691b0e7952cdc9969705424aa1df","unresolved":false,"context_lines":[{"line_number":47,"context_line":"   restarting RabbitMQ again. If there are no errors, restart"},{"line_number":48,"context_line":"   RabbitMQ server on the next controller node."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"#. After restartung RabbitMQ on all controller nodes, restart services"},{"line_number":51,"context_line":"   on each controller node. For example:"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"   .. code-block:: console"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_91a0c543","line":50,"range":{"start_line":50,"start_character":9,"end_line":50,"end_character":19},"updated":"2016-07-13 06:11:15.000000000","message":"restart","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":47,"context_line":"   restarting RabbitMQ again. If there are no errors, restart"},{"line_number":48,"context_line":"   RabbitMQ server on the next controller node."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"#. After restartung RabbitMQ on all controller nodes, restart services"},{"line_number":51,"context_line":"   on each controller node. For example:"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"   .. code-block:: console"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_6c7545cb","line":50,"in_reply_to":"1aa78d24_e8093c2c","updated":"2016-07-14 02:32:29.000000000","message":"TBC","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":22165,"name":"caoyuan","email":"cao.yingjunz@gmail.com","username":"caoyuan"},"change_message_id":"93ee9e7181cabae43e285212325718cf380dc0e6","unresolved":false,"context_lines":[{"line_number":47,"context_line":"   restarting RabbitMQ again. If there are no errors, restart"},{"line_number":48,"context_line":"   RabbitMQ server on the next controller node."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"#. After restartung RabbitMQ on all controller nodes, restart services"},{"line_number":51,"context_line":"   on each controller node. For example:"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"   .. code-block:: console"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_a4a301ff","line":51,"range":{"start_line":50,"start_character":54,"end_line":51,"end_character":26},"updated":"2016-07-13 06:19:18.000000000","message":"The command \"openstack-service restart\" can restart all service on the node","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":47,"context_line":"   restarting RabbitMQ again. If there are no errors, restart"},{"line_number":48,"context_line":"   RabbitMQ server on the next controller node."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"#. After restartung RabbitMQ on all controller nodes, restart services"},{"line_number":51,"context_line":"   on each controller node. For example:"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"   .. code-block:: console"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_4c7209e5","line":51,"range":{"start_line":50,"start_character":54,"end_line":51,"end_character":26},"in_reply_to":"1aa78d24_a4a301ff","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":19814,"name":"Travis Truman (automagically)","email":"travis_truman@cable.comcast.com","username":"trumant"},"change_message_id":"e7b21544fc04be1463278fd8b015378b35570649","unresolved":false,"context_lines":[{"line_number":75,"context_line":"and resolve the issue:"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"#. Determine which servers the RabbitMQ alarms are coming from."},{"line_number":78,"context_line":"#. Depending on the environment the servers are in, open up your Dashboard"},{"line_number":79,"context_line":"   and launch an instance."},{"line_number":80,"context_line":"#. If you cannot launch an instance, continue to troubleshoot the issue."},{"line_number":81,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_e8581cfb","line":78,"updated":"2016-07-13 15:50:49.000000000","message":"Could this be simplified to something like: \"Attempt to boot a nova instance in the effected environment\"","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":75,"context_line":"and resolve the issue:"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"#. Determine which servers the RabbitMQ alarms are coming from."},{"line_number":78,"context_line":"#. Depending on the environment the servers are in, open up your Dashboard"},{"line_number":79,"context_line":"   and launch an instance."},{"line_number":80,"context_line":"#. If you cannot launch an instance, continue to troubleshoot the issue."},{"line_number":81,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_ac044d18","line":78,"in_reply_to":"1aa78d24_e8581cfb","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":19814,"name":"Travis Truman (automagically)","email":"travis_truman@cable.comcast.com","username":"trumant"},"change_message_id":"e7b21544fc04be1463278fd8b015378b35570649","unresolved":false,"context_lines":[{"line_number":81,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"},{"line_number":82,"context_line":"   check the :file:`/var/log/rabbitmq` log files for any reported issues."},{"line_number":83,"context_line":"#. Look for connection issues identified in the OpenStack log files."},{"line_number":84,"context_line":"#. Open Dashboard and launch an instance. If you cannot launch an instance,"},{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_48d0501f","line":84,"updated":"2016-07-13 15:50:49.000000000","message":"Why is this step repeated, the only thing that has changed between this step and the first time is time. I.e. no system changes have been made","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":81,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"},{"line_number":82,"context_line":"   check the :file:`/var/log/rabbitmq` log files for any reported issues."},{"line_number":83,"context_line":"#. Look for connection issues identified in the OpenStack log files."},{"line_number":84,"context_line":"#. Open Dashboard and launch an instance. If you cannot launch an instance,"},{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_5791f299","line":84,"in_reply_to":"1aa78d24_48d0501f","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10497,"name":"KATO Tomoyuki","email":"kato.tomoyuki@jp.fujitsu.com","username":"to222"},"change_message_id":"9b7e49713b8b97789e1001ee398fc23f63b0cd29","unresolved":false,"context_lines":[{"line_number":84,"context_line":"#. Open Dashboard and launch an instance. If you cannot launch an instance,"},{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_ce5bcc52","line":87,"range":{"start_line":87,"start_character":48,"end_line":87,"end_character":55},"updated":"2016-07-13 05:38:32.000000000","message":"I believe we don\u0027t have \"quantum\" at now.","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":84,"context_line":"#. Open Dashboard and launch an instance. If you cannot launch an instance,"},{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_b732d6f0","line":87,"range":{"start_line":87,"start_character":48,"end_line":87,"end_character":55},"in_reply_to":"1aa78d24_52275fc9","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":84,"context_line":"#. Open Dashboard and launch an instance. If you cannot launch an instance,"},{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_ac2d6daa","line":87,"range":{"start_line":87,"start_character":48,"end_line":87,"end_character":55},"in_reply_to":"1aa78d24_ce5bcc52","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":9382,"name":"Shilla Saebi","email":"shilla.saebi@gmail.com","username":"shillasaebi"},"change_message_id":"95673bbd249b3759bc53606f7d1cbf3ae0e46d22","unresolved":false,"context_lines":[{"line_number":84,"context_line":"#. Open Dashboard and launch an instance. If you cannot launch an instance,"},{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_52275fc9","line":87,"range":{"start_line":87,"start_character":48,"end_line":87,"end_character":55},"in_reply_to":"1aa78d24_ce5bcc52","updated":"2016-07-13 15:33:26.000000000","message":"lol yeah quantum should be changed","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":19814,"name":"Travis Truman (automagically)","email":"travis_truman@cable.comcast.com","username":"trumant"},"change_message_id":"e7b21544fc04be1463278fd8b015378b35570649","unresolved":false,"context_lines":[{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"},{"line_number":91,"context_line":"   the node."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_08ae4880","line":88,"updated":"2016-07-13 15:50:49.000000000","message":"Does this suggest that rebooting the host is the correct thing to do? I\u0027m not sure what \"restart\" means here, but I find the suggestion of rebooting a controller host to resolve RabbitMQ alarms a bit heavy-handed","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":85,"context_line":"   proceed to the next step."},{"line_number":86,"context_line":"#. For each controller node in your environment, view the `/etc/init.d`"},{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"},{"line_number":91,"context_line":"   the node."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_17292a38","line":88,"in_reply_to":"1aa78d24_08ae4880","updated":"2016-07-14 02:32:29.000000000","message":"TBC","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10497,"name":"KATO Tomoyuki","email":"kato.tomoyuki@jp.fujitsu.com","username":"to222"},"change_message_id":"9b7e49713b8b97789e1001ee398fc23f63b0cd29","unresolved":false,"context_lines":[{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"},{"line_number":91,"context_line":"   the node."},{"line_number":92,"context_line":"#. Open Dashboard and launch an instance. If the instance launches, the issue"},{"line_number":93,"context_line":"   is resolved."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_6e33b814","line":90,"range":{"start_line":90,"start_character":38,"end_line":90,"end_character":45},"updated":"2016-07-13 05:38:32.000000000","message":"ditto","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":87,"context_line":"   directory and if it contains nova*, cinder*, quantum*, neutron*, or"},{"line_number":88,"context_line":"   glance*, restart the node."},{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"},{"line_number":91,"context_line":"   the node."},{"line_number":92,"context_line":"#. Open Dashboard and launch an instance. If the instance launches, the issue"},{"line_number":93,"context_line":"   is resolved."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_2c105d59","line":90,"range":{"start_line":90,"start_character":38,"end_line":90,"end_character":45},"in_reply_to":"1aa78d24_6e33b814","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":9382,"name":"Shilla Saebi","email":"shilla.saebi@gmail.com","username":"shillasaebi"},"change_message_id":"95673bbd249b3759bc53606f7d1cbf3ae0e46d22","unresolved":false,"context_lines":[{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"},{"line_number":91,"context_line":"   the node."},{"line_number":92,"context_line":"#. Open Dashboard and launch an instance. If the instance launches, the issue"},{"line_number":93,"context_line":"   is resolved."},{"line_number":94,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":95,"context_line":"   files for reported connection issues."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_12103758","line":92,"range":{"start_line":92,"start_character":8,"end_line":92,"end_character":17},"updated":"2016-07-13 15:33:26.000000000","message":"Open the OpenStack Dashboard","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":89,"context_line":"#. For each compute node your environment, view the `/etc/init.d` directory"},{"line_number":90,"context_line":"   and if it contains nova*, cinder*, quantum*, neutron*, or glance*, restart"},{"line_number":91,"context_line":"   the node."},{"line_number":92,"context_line":"#. Open Dashboard and launch an instance. If the instance launches, the issue"},{"line_number":93,"context_line":"   is resolved."},{"line_number":94,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":95,"context_line":"   files for reported connection issues."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_37a38602","line":92,"range":{"start_line":92,"start_character":8,"end_line":92,"end_character":17},"in_reply_to":"1aa78d24_12103758","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":9382,"name":"Shilla Saebi","email":"shilla.saebi@gmail.com","username":"shillasaebi"},"change_message_id":"95673bbd249b3759bc53606f7d1cbf3ae0e46d22","unresolved":false,"context_lines":[{"line_number":93,"context_line":"   is resolved."},{"line_number":94,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":95,"context_line":"   files for reported connection issues."},{"line_number":96,"context_line":"#. Restart the RabbitMQ service on all controller nodes:"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"   .. code-block:: console"},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_92042718","line":96,"range":{"start_line":96,"start_character":35,"end_line":96,"end_character":38},"updated":"2016-07-13 15:33:26.000000000","message":"all of the controller nodes","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"2c3b952e792d914f1a1a4d9b360ef41a158d1956","unresolved":false,"context_lines":[{"line_number":93,"context_line":"   is resolved."},{"line_number":94,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":95,"context_line":"   files for reported connection issues."},{"line_number":96,"context_line":"#. Restart the RabbitMQ service on all controller nodes:"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"   .. code-block:: console"},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"1aa78d24_573c52bf","line":96,"range":{"start_line":96,"start_character":35,"end_line":96,"end_character":38},"in_reply_to":"1aa78d24_92042718","updated":"2016-07-14 02:32:29.000000000","message":"Done","commit_id":"7e00befa7e25806a5487cd288ee5dcf8afddaba0"},{"author":{"_account_id":10897,"name":"Joseph Robinson","email":"josephremail@gmail.com","username":"RobiJoe"},"change_message_id":"6e5a147c19f2ddad76e453f919aec15565d756cf","unresolved":false,"context_lines":[{"line_number":19,"context_line":"      # service rabbitmq-server start"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"#. If the service refuses to stop, then run the :commnand:`pkill` command"},{"line_number":23,"context_line":"   to stop the service, then restart the service:"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"   .. code-block:: console"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1aa78d24_7c6dee98","line":22,"updated":"2016-07-14 05:30:08.000000000","message":"Change to :command: to fix the build error.","commit_id":"425a01a8cc5ae1917cc145c18245eb1b8e394c10"},{"author":{"_account_id":19814,"name":"Travis Truman (automagically)","email":"travis_truman@cable.comcast.com","username":"trumant"},"change_message_id":"dc538dca90b7696b483e38cb59942f696b4f054c","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"   .. code-block:: console"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"      # service rabbitmq-server stop"},{"line_number":19,"context_line":"      # service rabbitmq-server start"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. If the service refuses to stop, then run the :command:`pkill` command"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_51860bdc","line":18,"range":{"start_line":18,"start_character":6,"end_line":18,"end_character":7},"updated":"2016-07-14 12:27:25.000000000","message":"service rabbitmq-server restart should do here","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"   .. code-block:: console"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"      # service rabbitmq-server stop"},{"line_number":19,"context_line":"      # service rabbitmq-server start"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. If the service refuses to stop, then run the :command:`pkill` command"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_3b37ba4e","line":18,"in_reply_to":"1aa78d24_0721e0b4","updated":"2016-07-15 04:35:06.000000000","message":"Done","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"0650b9f8304867049e0dd786d94ed307950f9c56","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"   .. code-block:: console"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"      # service rabbitmq-server stop"},{"line_number":19,"context_line":"      # service rabbitmq-server start"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. If the service refuses to stop, then run the :command:`pkill` command"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_0721e0b4","line":18,"in_reply_to":"1aa78d24_47292df2","updated":"2016-07-14 14:44:51.000000000","message":"Not sure if this note is made elsewhere, but we should warn that the service name may vary across the base level operating system, or vendor who supplies your rabbit. They aren\u0027t all named the same.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"   .. code-block:: console"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"      # service rabbitmq-server stop"},{"line_number":19,"context_line":"      # service rabbitmq-server start"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. If the service refuses to stop, then run the :command:`pkill` command"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_1bdc7e17","line":18,"in_reply_to":"1aa78d24_47292df2","updated":"2016-07-15 04:35:06.000000000","message":"Ok, thanks for your input","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"c7749b402900dce1756496fd7242d9892990ae52","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"   .. code-block:: console"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"      # service rabbitmq-server stop"},{"line_number":19,"context_line":"      # service rabbitmq-server start"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"#. If the service refuses to stop, then run the :command:`pkill` command"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_47292df2","line":18,"in_reply_to":"1aa78d24_51860bdc","updated":"2016-07-14 13:09:02.000000000","message":"There are some pathological cases where a restart will not work, so doing a stop/start can be slightly better in my experience.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"c7749b402900dce1756496fd7242d9892990ae52","unresolved":false,"context_lines":[{"line_number":37,"context_line":"#. If there are errors, go back to the first step and try"},{"line_number":38,"context_line":"   restarting the RabbitMQ service again. If there are no errors, restart"},{"line_number":39,"context_line":"   the RabbitMQ service on the next controller node."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"#. After restarting RabbitMQ on all controller nodes, restart other OpenStack"},{"line_number":42,"context_line":"   services on each controller node:"},{"line_number":43,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_47612d11","line":40,"updated":"2016-07-14 13:09:02.000000000","message":"Another thing you will want to check is that you don\u0027t have a split brain situation, so check the output of\n\nrabbitmqctl cluster_status\n\nand make sure that there are no partitions. See https://www.rabbitmq.com/partitions.html for some more information, but also note that sometimes restarting the services may not be enough, in that case you will have to remove the contents of /var/lib/rabbitmq/mnesia/ between stopping and starting.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":37,"context_line":"#. If there are errors, go back to the first step and try"},{"line_number":38,"context_line":"   restarting the RabbitMQ service again. If there are no errors, restart"},{"line_number":39,"context_line":"   the RabbitMQ service on the next controller node."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"#. After restarting RabbitMQ on all controller nodes, restart other OpenStack"},{"line_number":42,"context_line":"   services on each controller node:"},{"line_number":43,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_b5e73645","line":40,"in_reply_to":"1aa78d24_47612d11","updated":"2016-07-15 04:35:06.000000000","message":"Done","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"0650b9f8304867049e0dd786d94ed307950f9c56","unresolved":false,"context_lines":[{"line_number":39,"context_line":"   the RabbitMQ service on the next controller node."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"#. After restarting RabbitMQ on all controller nodes, restart other OpenStack"},{"line_number":42,"context_line":"   services on each controller node:"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"   .. code-block:: console"},{"line_number":45,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_ea073916","line":42,"updated":"2016-07-14 14:44:51.000000000","message":"This used to be necessary prior to Liberty I believe. In Liberty the rabbit heartbeat functionality in oslo got quite good and services typically recover from a rabbit outage. Restarting services should only be a last measure if they are not picking up new messages from the queues.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":39,"context_line":"   the RabbitMQ service on the next controller node."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"#. After restarting RabbitMQ on all controller nodes, restart other OpenStack"},{"line_number":42,"context_line":"   services on each controller node:"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"   .. code-block:: console"},{"line_number":45,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_75daee18","line":42,"in_reply_to":"1aa78d24_3b9b5174","updated":"2016-07-15 04:35:06.000000000","message":"Done. Let me know if I\u0027ve misinterpreted anything.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":9076,"name":"Kris Lindgren","email":"klindgren@godaddy.com","username":"klindgren"},"change_message_id":"c5eba084b9bb40ec5b6ec1c53c1e0795e7c13780","unresolved":false,"context_lines":[{"line_number":39,"context_line":"   the RabbitMQ service on the next controller node."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"#. After restarting RabbitMQ on all controller nodes, restart other OpenStack"},{"line_number":42,"context_line":"   services on each controller node:"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"   .. code-block:: console"},{"line_number":45,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_3b9b5174","line":42,"in_reply_to":"1aa78d24_ea073916","updated":"2016-07-14 15:50:58.000000000","message":"Also in liberty if you restart neutron-ovs agent and it doesn\u0027t have rabbit/neutron-server access you will black hole all traffic to your vm\u0027s until you fix that datapath.  I would rather tell people to make sure heartbeats are enabled and watch for errors - than tell them to blindly restart services - as some may have a huge unexpected impact on vm\u0027s.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":39,"context_line":"   the RabbitMQ service on the next controller node."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"#. After restarting RabbitMQ on all controller nodes, restart other OpenStack"},{"line_number":42,"context_line":"   services on each controller node:"},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"   .. code-block:: console"},{"line_number":45,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_35d066f7","line":42,"in_reply_to":"1aa78d24_ea073916","updated":"2016-07-15 04:35:06.000000000","message":"Done. Let me know if I\u0027ve misinterpreted anything.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"c7749b402900dce1756496fd7242d9892990ae52","unresolved":false,"context_lines":[{"line_number":43,"context_line":""},{"line_number":44,"context_line":"   .. code-block:: console"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"      # openstack-service restart"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"#. Restart services on all compute nodes."},{"line_number":49,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_e77c99b7","line":46,"updated":"2016-07-14 13:09:02.000000000","message":"On what platform do you have this command? I don\u0027t see it on Ubuntu.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":43,"context_line":""},{"line_number":44,"context_line":"   .. code-block:: console"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"      # openstack-service restart"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"#. Restart services on all compute nodes."},{"line_number":49,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_967685d6","line":46,"in_reply_to":"1aa78d24_e77c99b7","updated":"2016-07-15 04:35:06.000000000","message":"I think this is for Fedora. I\u0027ll keep the commands consistent for Ubuntu.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"c7749b402900dce1756496fd7242d9892990ae52","unresolved":false,"context_lines":[{"line_number":54,"context_line":"and resolve the issue:"},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"#. Determine which servers the RabbitMQ alarms are coming from."},{"line_number":57,"context_line":"#. Attempt to boot a nova instance in the effected environment."},{"line_number":58,"context_line":"#. If you cannot launch an instance, continue to troubleshoot the issue."},{"line_number":59,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"},{"line_number":60,"context_line":"   check the :file:`/var/log/rabbitmq` log files for any reported issues."}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_c75cfd4e","line":57,"updated":"2016-07-14 13:09:02.000000000","message":"s/effected/affected/","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":54,"context_line":"and resolve the issue:"},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"#. Determine which servers the RabbitMQ alarms are coming from."},{"line_number":57,"context_line":"#. Attempt to boot a nova instance in the effected environment."},{"line_number":58,"context_line":"#. If you cannot launch an instance, continue to troubleshoot the issue."},{"line_number":59,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"},{"line_number":60,"context_line":"   check the :file:`/var/log/rabbitmq` log files for any reported issues."}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_d6800d8e","line":57,"in_reply_to":"1aa78d24_c75cfd4e","updated":"2016-07-15 04:35:06.000000000","message":"Done","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"0650b9f8304867049e0dd786d94ed307950f9c56","unresolved":false,"context_lines":[{"line_number":59,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"},{"line_number":60,"context_line":"   check the :file:`/var/log/rabbitmq` log files for any reported issues."},{"line_number":61,"context_line":"#. Look for connection issues identified in the log files."},{"line_number":62,"context_line":"#. For each controller node in your environment, view the ``/etc/init.d``"},{"line_number":63,"context_line":"   directory and if it contains nova*, cinder*, neutron*, or"},{"line_number":64,"context_line":"   glance*, restart the RabbitMQ service."},{"line_number":65,"context_line":"#. For each compute node your environment, view the ``/etc/init.d`` directory"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_aa7251ae","line":62,"updated":"2016-07-14 14:44:51.000000000","message":"Before restarting services, you should check the rabbit queues for a queue that is growing without being consumed. That can indicate that a particular service has lost its connection to rabbit and is no longer processing messages intended for it. That will narrow down which service needs to be restarted.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":59,"context_line":"#. Log in to each of the controller nodes for the affected environment, and"},{"line_number":60,"context_line":"   check the :file:`/var/log/rabbitmq` log files for any reported issues."},{"line_number":61,"context_line":"#. Look for connection issues identified in the log files."},{"line_number":62,"context_line":"#. For each controller node in your environment, view the ``/etc/init.d``"},{"line_number":63,"context_line":"   directory and if it contains nova*, cinder*, neutron*, or"},{"line_number":64,"context_line":"   glance*, restart the RabbitMQ service."},{"line_number":65,"context_line":"#. For each compute node your environment, view the ``/etc/init.d`` directory"}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_98750f99","line":62,"in_reply_to":"1aa78d24_aa7251ae","updated":"2016-07-15 04:35:06.000000000","message":"Thanks, done.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":9076,"name":"Kris Lindgren","email":"klindgren@godaddy.com","username":"klindgren"},"change_message_id":"c5eba084b9bb40ec5b6ec1c53c1e0795e7c13780","unresolved":false,"context_lines":[{"line_number":67,"context_line":"   the RabbitMQ service."},{"line_number":68,"context_line":"#. Open OpenStack Dashboard and launch an instance. If the instance launches,"},{"line_number":69,"context_line":"   the issue is resolved."},{"line_number":70,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":71,"context_line":"   files for reported connection issues."},{"line_number":72,"context_line":"#. Restart the RabbitMQ service on all of the controller nodes:"},{"line_number":73,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_b638f83e","line":70,"updated":"2016-07-14 15:50:58.000000000","message":"Under liberty with rabbit 3.4.x we noticed that the management database would eventually consume the memory allocated to rabbit.  (rabbitmqctl status show memory consumption) By default this is 40% of the host memory.  This is caused by stats collection and processing, under a busy Rabbit this can fall behind and will grow unbounded.  When a single rabbitmq node reaches is memory threshold *ALL* exchange/queue processing is halted until the memory alarm recovers.  Apparently 3.6.x has a large number of bugs in this area that makes the issue worse and they are working on a fix.  Part of the recommendation is to turn off stats collection or to change the sample rate for stats to 30-60 seconds.\n\nI believe people will start to see this more under liberty as in oslo.messaging they change the queues used for RPC from auto-delete to auto close after 10 minutes (default) from when it no longer has a consumer.  So under heavy RPC load - you can really stack up some queue\u0027s/stats.  This is also exacerbated by the fact that only a single node in the cluster handles the storage/processing of stats.  In 3.7 they are talking about having the stats be sharded across cluster members.\n\nhttps://groups.google.com/forum/#!msg/rabbitmq-users/FeBK7iXUcLg/nrQXAoVrAQAJ\n\nYou can change the collect_statistic_interval at run time via:\nabbitmqctl eval \u0027application:set_env(rabbit, collect_statistics_interval, 60000).\u0027\n\nI realize this is might be in the wrong section or a TL;DR but this caused us some serious pain, with the worse case restarting rabbitmq every 2 hours.  So I think if we see common problems that impact rabbitmq it might be nice to specifically call those out to people, as restart rabbitmq may only buy you a few hours to a few weeks with some specific errors/cases.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":17799,"name":"Logan V","email":"logan2211@gmail.com","username":"Logan2211"},"change_message_id":"ab66e154feda5604b4393edd7948a229c63dff82","unresolved":false,"context_lines":[{"line_number":67,"context_line":"   the RabbitMQ service."},{"line_number":68,"context_line":"#. Open OpenStack Dashboard and launch an instance. If the instance launches,"},{"line_number":69,"context_line":"   the issue is resolved."},{"line_number":70,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":71,"context_line":"   files for reported connection issues."},{"line_number":72,"context_line":"#. Restart the RabbitMQ service on all of the controller nodes:"},{"line_number":73,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_ea7294e2","line":70,"in_reply_to":"1aa78d24_b638f83e","updated":"2016-07-14 16:28:50.000000000","message":"One common troubleshooting item with RMQ that will come up in all environments once scaled to a certain size is file descriptor limits for the user running RMQ. The relevant output can be obtained from `rabbitmqctl status`\n\n\"{file_descriptors,\n     [{total_limit,3996},\n      {total_used,135},\n      {sockets_limit,3594},\n      {sockets_used,133}]},\"\n\nIt can be remedied by setting appropriate limits in /etc/security/limits.conf","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":67,"context_line":"   the RabbitMQ service."},{"line_number":68,"context_line":"#. Open OpenStack Dashboard and launch an instance. If the instance launches,"},{"line_number":69,"context_line":"   the issue is resolved."},{"line_number":70,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":71,"context_line":"   files for reported connection issues."},{"line_number":72,"context_line":"#. Restart the RabbitMQ service on all of the controller nodes:"},{"line_number":73,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_d8279761","line":70,"in_reply_to":"1aa78d24_b638f83e","updated":"2016-07-15 04:35:06.000000000","message":"Thanks, this is really helpful information. I\u0027ll add it in the next patch set.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":67,"context_line":"   the RabbitMQ service."},{"line_number":68,"context_line":"#. Open OpenStack Dashboard and launch an instance. If the instance launches,"},{"line_number":69,"context_line":"   the issue is resolved."},{"line_number":70,"context_line":"#. If you cannot launch an instance, check the :file:`/var/log/rabbitmq` log"},{"line_number":71,"context_line":"   files for reported connection issues."},{"line_number":72,"context_line":"#. Restart the RabbitMQ service on all of the controller nodes:"},{"line_number":73,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_181abf18","line":70,"in_reply_to":"1aa78d24_ea7294e2","updated":"2016-07-15 04:35:06.000000000","message":"Thanks for the information. What is the error that occurs when these limits aren\u0027t adjusted?\n\nI\u0027ll add it in the next patch set.","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"0650b9f8304867049e0dd786d94ed307950f9c56","unresolved":false,"context_lines":[{"line_number":73,"context_line":""},{"line_number":74,"context_line":"   .. code-block:: console"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"      # service rabbitmq-server restart"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"   .. note::"},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_4a5e4d13","line":76,"updated":"2016-07-14 14:44:51.000000000","message":"Above you advocate a stop / start pair, instead of restart. That should be followed here as well (or just reference the top of this document for a restart procedure).","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"},{"author":{"_account_id":10705,"name":"Darren Chan","email":"dazzachan@yahoo.com.au","username":"daz"},"change_message_id":"5fa5125daf4a931db277da4c8b0da32cd3079fea","unresolved":false,"context_lines":[{"line_number":73,"context_line":""},{"line_number":74,"context_line":"   .. code-block:: console"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"      # service rabbitmq-server restart"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"   .. note::"},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"1aa78d24_18e11ffd","line":76,"in_reply_to":"1aa78d24_4a5e4d13","updated":"2016-07-15 04:35:06.000000000","message":"Done","commit_id":"004ae6b1a1332f6dfb58ffdc90cdabdf90ad9f5c"}]}