)]}'
{"doc/source/specs/redis_availability.rst":[{"author":{"_account_id":20297,"name":"duankebo","email":"duankebo@huawei.com","username":"duankebo"},"change_message_id":"cf9cd5f75603ff7521283f2e5a4f1b83f3288eeb","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Problem Description\r"},{"line_number":14,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\r"},{"line_number":15,"context_line":"\r"},{"line_number":16,"context_line":"Dragonflow\u0027s Redis driver read the Redis cluster topology and cache it locally in Redis driver¡¯s initialization and then\r"},{"line_number":17,"context_line":"it connects to the Redis master nodes to operate read/write/pub/sub commands.\r"},{"line_number":18,"context_line":"This cluster topology maybe changed and then start HA in some scenario like db master node restarting and\r"},{"line_number":19,"context_line":"Dragonflow should detect it that it could reset the connections from the old master node to the new one.\r"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ba0121b8_15987177","line":16,"range":{"start_line":16,"start_character":98,"end_line":16,"end_character":100},"updated":"2016-03-31 06:17:30.000000000","message":"Please no more than 80 characters per line.","commit_id":"667484be317b9b499b7e5510d127a2938c71e0df"},{"author":{"_account_id":20297,"name":"duankebo","email":"duankebo@huawei.com","username":"duankebo"},"change_message_id":"cf9cd5f75603ff7521283f2e5a4f1b83f3288eeb","unresolved":false,"context_lines":[{"line_number":16,"context_line":"Dragonflow\u0027s Redis driver read the Redis cluster topology and cache it locally in Redis driver¡¯s initialization and then\r"},{"line_number":17,"context_line":"it connects to the Redis master nodes to operate read/write/pub/sub commands.\r"},{"line_number":18,"context_line":"This cluster topology maybe changed and then start HA in some scenario like db master node restarting and\r"},{"line_number":19,"context_line":"Dragonflow should detect it that it could reset the connections from the old master node to the new one.\r"},{"line_number":20,"context_line":"\r"},{"line_number":21,"context_line":"There are two scenarios in Redis cluster topology changing:\r"},{"line_number":22,"context_line":"1. The connection will be lost when master node restarting.\r"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ba0121b8_35a9357c","line":19,"range":{"start_line":19,"start_character":42,"end_line":19,"end_character":47},"updated":"2016-03-31 06:17:30.000000000","message":"move","commit_id":"667484be317b9b499b7e5510d127a2938c71e0df"},{"author":{"_account_id":20297,"name":"duankebo","email":"duankebo@huawei.com","username":"duankebo"},"change_message_id":"cf9cd5f75603ff7521283f2e5a4f1b83f3288eeb","unresolved":false,"context_lines":[{"line_number":46,"context_line":"\r"},{"line_number":47,"context_line":"Description to step 2\r"},{"line_number":48,"context_line":"------------------------\r"},{"line_number":49,"context_line":"After detecting, a new thread will be started to read new Redis cluster topology information periodically\r"},{"line_number":50,"context_line":"because there is a few seconds in Redis cluster HA.\r"},{"line_number":51,"context_line":"If the cluster state is ok and master node status is not fail,\r"},{"line_number":52,"context_line":"the driver will update the topology information and reset connections, then the thread stops and a ¡°sync¡± message will be sent.\r"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ba0121b8_a40a3688","line":49,"range":{"start_line":49,"start_character":64,"end_line":49,"end_character":83},"updated":"2016-03-31 06:17:30.000000000","message":"This means all the compute node will query default node periodically. Is there a better way?","commit_id":"667484be317b9b499b7e5510d127a2938c71e0df"},{"author":{"_account_id":20297,"name":"duankebo","email":"duankebo@huawei.com","username":"duankebo"},"change_message_id":"cf9cd5f75603ff7521283f2e5a4f1b83f3288eeb","unresolved":false,"context_lines":[{"line_number":46,"context_line":"\r"},{"line_number":47,"context_line":"Description to step 2\r"},{"line_number":48,"context_line":"------------------------\r"},{"line_number":49,"context_line":"After detecting, a new thread will be started to read new Redis cluster topology information periodically\r"},{"line_number":50,"context_line":"because there is a few seconds in Redis cluster HA.\r"},{"line_number":51,"context_line":"If the cluster state is ok and master node status is not fail,\r"},{"line_number":52,"context_line":"the driver will update the topology information and reset connections, then the thread stops and a ¡°sync¡± message will be sent.\r"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ba0121b8_35ee55b4","line":49,"range":{"start_line":49,"start_character":6,"end_line":49,"end_character":15},"updated":"2016-03-31 06:17:30.000000000","message":"not clear, detecting what?","commit_id":"667484be317b9b499b7e5510d127a2938c71e0df"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8d3e6b12e441ed9c4e6aa53a08582d1d80dacfb6","unresolved":false,"context_lines":[{"line_number":1,"context_line":"..\r"},{"line_number":2,"context_line":" This work is licensed under a Creative Commons Attribution 3.0 Unsuported\r"},{"line_number":3,"context_line":" License.\r"},{"line_number":4,"context_line":"\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_7a931b5d","line":1,"updated":"2016-05-02 09:01:27.000000000","message":"Please verify that there are no restview syntax errors.\nYou can use e.g. https://mg.pov.lt/restview/ or http://rst.ninjs.org","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"2a22d33187201c18fc48fcae5eafe94eec23cf6b","unresolved":false,"context_lines":[{"line_number":1,"context_line":"..\r"},{"line_number":2,"context_line":" This work is licensed under a Creative Commons Attribution 3.0 Unsuported\r"},{"line_number":3,"context_line":" License.\r"},{"line_number":4,"context_line":"\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_8f9d040d","line":1,"in_reply_to":"1a122d0e_7a931b5d","updated":"2016-05-03 03:44:55.000000000","message":"Done","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8d3e6b12e441ed9c4e6aa53a08582d1d80dacfb6","unresolved":false,"context_lines":[{"line_number":14,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\r"},{"line_number":15,"context_line":"\r"},{"line_number":16,"context_line":"Dragonflow\u0027s Redis driver read the Redis cluster topology and cache it\r"},{"line_number":17,"context_line":"locally in Redis driver¡¯s initialization and then it connects to the Redis\r"},{"line_number":18,"context_line":"master nodes to operate read/write/pub/sub commands.\r"},{"line_number":19,"context_line":"This cluster topology maybe changed and then start HA in some scenario like\r"},{"line_number":20,"context_line":"db master node restarting and Dragonflow should detect it that it could\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_9a71d747","line":17,"range":{"start_line":17,"start_character":23,"end_line":17,"end_character":26},"updated":"2016-05-02 09:01:27.000000000","message":"Strange character. Consider replacing with \u0027. (This appears a few more times in this document).","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"84046c0a22f70eeaf7fb7f999db25690da7263e1","unresolved":false,"context_lines":[{"line_number":14,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\r"},{"line_number":15,"context_line":"\r"},{"line_number":16,"context_line":"Dragonflow\u0027s Redis driver read the Redis cluster topology and cache it\r"},{"line_number":17,"context_line":"locally in Redis driver¡¯s initialization and then it connects to the Redis\r"},{"line_number":18,"context_line":"master nodes to operate read/write/pub/sub commands.\r"},{"line_number":19,"context_line":"This cluster topology maybe changed and then start HA in some scenario like\r"},{"line_number":20,"context_line":"db master node restarting and Dragonflow should detect it that it could\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_2f15d09a","line":17,"range":{"start_line":17,"start_character":23,"end_line":17,"end_character":26},"in_reply_to":"1a122d0e_9a71d747","updated":"2016-05-03 01:43:23.000000000","message":"yes","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8d3e6b12e441ed9c4e6aa53a08582d1d80dacfb6","unresolved":false,"context_lines":[{"line_number":20,"context_line":"db master node restarting and Dragonflow should detect it that it could\r"},{"line_number":21,"context_line":"move the connections from the old master node to the new one.\r"},{"line_number":22,"context_line":"\r"},{"line_number":23,"context_line":"There are two scenarios in Redis cluster topology changing:\r"},{"line_number":24,"context_line":"1. The connection will be lost when master node restarting.\r"},{"line_number":25,"context_line":"2. The connection will not be lost while master node changed to slave\r"},{"line_number":26,"context_line":"without restarting as using ¡°CLUSTER FAILOVER¡± command.\r"},{"line_number":27,"context_line":"In this case one slave will be promoted to master and the client\r"},{"line_number":28,"context_line":"could not get connection error but a MOVED error from server after\r"},{"line_number":29,"context_line":"sending request to the new slave node.\r"},{"line_number":30,"context_line":"\r"},{"line_number":31,"context_line":"Some data maybe lost in Redis HA because Redis does not\r"},{"line_number":32,"context_line":"provide strong consistency. So for this case,\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_4e7144a8","line":29,"range":{"start_line":23,"start_character":0,"end_line":29,"end_character":38},"updated":"2016-05-02 09:01:27.000000000","message":"What about support for sentinel?\nhttp://redis.io/topics/sentinel\n\nThis way, we don\u0027t need to write the detection logic ourselves. Sentinel pub/sub will notify us upon redis topology changes.","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"84046c0a22f70eeaf7fb7f999db25690da7263e1","unresolved":false,"context_lines":[{"line_number":20,"context_line":"db master node restarting and Dragonflow should detect it that it could\r"},{"line_number":21,"context_line":"move the connections from the old master node to the new one.\r"},{"line_number":22,"context_line":"\r"},{"line_number":23,"context_line":"There are two scenarios in Redis cluster topology changing:\r"},{"line_number":24,"context_line":"1. The connection will be lost when master node restarting.\r"},{"line_number":25,"context_line":"2. The connection will not be lost while master node changed to slave\r"},{"line_number":26,"context_line":"without restarting as using ¡°CLUSTER FAILOVER¡± command.\r"},{"line_number":27,"context_line":"In this case one slave will be promoted to master and the client\r"},{"line_number":28,"context_line":"could not get connection error but a MOVED error from server after\r"},{"line_number":29,"context_line":"sending request to the new slave node.\r"},{"line_number":30,"context_line":"\r"},{"line_number":31,"context_line":"Some data maybe lost in Redis HA because Redis does not\r"},{"line_number":32,"context_line":"provide strong consistency. So for this case,\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_4f100cac","line":29,"range":{"start_line":23,"start_character":0,"end_line":29,"end_character":38},"in_reply_to":"1a122d0e_4e7144a8","updated":"2016-05-03 01:43:23.000000000","message":"Sentinel is not suitable for Redis official cluster. Before Redis 3.0 version, it could use sentinel to implement cluster. After Redis 3.0, sentinel heartbeat function is already integrated in cluster.","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8d3e6b12e441ed9c4e6aa53a08582d1d80dacfb6","unresolved":false,"context_lines":[{"line_number":44,"context_line":"\r"},{"line_number":45,"context_line":"Description to step 1\r"},{"line_number":46,"context_line":"-------------------------------------\r"},{"line_number":47,"context_line":"If this step is done in each controller, there may have too many\r"},{"line_number":48,"context_line":"Dragonflow compute nodes read the DB cluster in the same time and\r"},{"line_number":49,"context_line":"redis cluster could hardly handle it.\r"},{"line_number":50,"context_line":"So create a detecting thread in NB plugin to read the DB topology information\r"},{"line_number":51,"context_line":"periodically when Neutron server starting and then send the information\r"},{"line_number":52,"context_line":"to all Dragonflow controllers to check if the DB cluster nodes changed.\r"},{"line_number":53,"context_line":"And controllers should subscribe a ¡°HA¡± topic to receive messages from\r"},{"line_number":54,"context_line":"plugin.\r"},{"line_number":55,"context_line":"In Dragonflow controller, it never read nodes information from Redis cluster\r"},{"line_number":56,"context_line":"after initialization but only listen the messages from detecting task from plugin.\r"},{"line_number":57,"context_line":"\r"},{"line_number":58,"context_line":"There are 3 types of connections between Redis client and cluster:\r"},{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_ae3f38b4","line":56,"range":{"start_line":47,"start_character":0,"end_line":56,"end_character":82},"updated":"2016-05-02 09:01:27.000000000","message":"Same here re: sentinel.","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"84046c0a22f70eeaf7fb7f999db25690da7263e1","unresolved":false,"context_lines":[{"line_number":44,"context_line":"\r"},{"line_number":45,"context_line":"Description to step 1\r"},{"line_number":46,"context_line":"-------------------------------------\r"},{"line_number":47,"context_line":"If this step is done in each controller, there may have too many\r"},{"line_number":48,"context_line":"Dragonflow compute nodes read the DB cluster in the same time and\r"},{"line_number":49,"context_line":"redis cluster could hardly handle it.\r"},{"line_number":50,"context_line":"So create a detecting thread in NB plugin to read the DB topology information\r"},{"line_number":51,"context_line":"periodically when Neutron server starting and then send the information\r"},{"line_number":52,"context_line":"to all Dragonflow controllers to check if the DB cluster nodes changed.\r"},{"line_number":53,"context_line":"And controllers should subscribe a ¡°HA¡± topic to receive messages from\r"},{"line_number":54,"context_line":"plugin.\r"},{"line_number":55,"context_line":"In Dragonflow controller, it never read nodes information from Redis cluster\r"},{"line_number":56,"context_line":"after initialization but only listen the messages from detecting task from plugin.\r"},{"line_number":57,"context_line":"\r"},{"line_number":58,"context_line":"There are 3 types of connections between Redis client and cluster:\r"},{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_0fe454aa","line":56,"range":{"start_line":47,"start_character":0,"end_line":56,"end_character":82},"in_reply_to":"1a122d0e_ae3f38b4","updated":"2016-05-03 01:43:23.000000000","message":"Same to above.","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8d3e6b12e441ed9c4e6aa53a08582d1d80dacfb6","unresolved":false,"context_lines":[{"line_number":73,"context_line":"------------------------\r"},{"line_number":74,"context_line":"After detecting the node failure, if the cluster state is ok and\r"},{"line_number":75,"context_line":"master node status is not fail, the driver will update\r"},{"line_number":76,"context_line":"the topology information and connections,\r"},{"line_number":77,"context_line":"then a ¡°recover¡± message will be sent to db consist module.\r"},{"line_number":78,"context_line":"\r"},{"line_number":79,"context_line":"The following diagram shows the procedure of Dragonflow:\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_8e0ebc30","line":76,"updated":"2016-05-02 09:01:27.000000000","message":"A re-sync is needed even if the redis master has changed.\nWhich controller node does the re-sync?","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"84046c0a22f70eeaf7fb7f999db25690da7263e1","unresolved":false,"context_lines":[{"line_number":73,"context_line":"------------------------\r"},{"line_number":74,"context_line":"After detecting the node failure, if the cluster state is ok and\r"},{"line_number":75,"context_line":"master node status is not fail, the driver will update\r"},{"line_number":76,"context_line":"the topology information and connections,\r"},{"line_number":77,"context_line":"then a ¡°recover¡± message will be sent to db consist module.\r"},{"line_number":78,"context_line":"\r"},{"line_number":79,"context_line":"The following diagram shows the procedure of Dragonflow:\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_cf241c10","line":76,"in_reply_to":"1a122d0e_8e0ebc30","updated":"2016-05-03 01:43:23.000000000","message":"Yes, and each controller does the re-sync, see \"add data sync mechanism for keep db\"","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8d3e6b12e441ed9c4e6aa53a08582d1d80dacfb6","unresolved":false,"context_lines":[{"line_number":76,"context_line":"the topology information and connections,\r"},{"line_number":77,"context_line":"then a ¡°recover¡± message will be sent to db consist module.\r"},{"line_number":78,"context_line":"\r"},{"line_number":79,"context_line":"The following diagram shows the procedure of Dragonflow:\r"},{"line_number":80,"context_line":"\r"},{"line_number":81,"context_line":"NB\r"},{"line_number":82,"context_line":"+-------------------------------+\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_0e774ca0","line":79,"updated":"2016-05-02 09:01:27.000000000","message":"Note that we should use existing re-sync features in dragonflow, so that when new features are added (e.g. object versioning and diff re-sync, it will be used automatically).","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"2a22d33187201c18fc48fcae5eafe94eec23cf6b","unresolved":false,"context_lines":[{"line_number":76,"context_line":"the topology information and connections,\r"},{"line_number":77,"context_line":"then a ¡°recover¡± message will be sent to db consist module.\r"},{"line_number":78,"context_line":"\r"},{"line_number":79,"context_line":"The following diagram shows the procedure of Dragonflow:\r"},{"line_number":80,"context_line":"\r"},{"line_number":81,"context_line":"NB\r"},{"line_number":82,"context_line":"+-------------------------------+\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_45610d11","line":79,"in_reply_to":"1a122d0e_0e774ca0","updated":"2016-05-03 03:44:55.000000000","message":"In this spec, re-sync feature means the spec \"add data sync mechanism for keep db\".","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8d3e6b12e441ed9c4e6aa53a08582d1d80dacfb6","unresolved":false,"context_lines":[{"line_number":104,"context_line":"||driver |        |DB consist|  |\r"},{"line_number":105,"context_line":"|--------+        +----------+  |\r"},{"line_number":106,"context_line":"+-------------------------------+\r"},{"line_number":107,"context_line":"SB\r"},{"line_number":108,"context_line":"\r"},{"line_number":109,"context_line":"References\r"},{"line_number":110,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_4e2304a8","line":107,"updated":"2016-05-02 09:01:27.000000000","message":"Am I right in understand the south-bound resync is done by a compute node? Is that necessary?","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"84046c0a22f70eeaf7fb7f999db25690da7263e1","unresolved":false,"context_lines":[{"line_number":104,"context_line":"||driver |        |DB consist|  |\r"},{"line_number":105,"context_line":"|--------+        +----------+  |\r"},{"line_number":106,"context_line":"+-------------------------------+\r"},{"line_number":107,"context_line":"SB\r"},{"line_number":108,"context_line":"\r"},{"line_number":109,"context_line":"References\r"},{"line_number":110,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\r"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a122d0e_4f5e4c7a","line":107,"in_reply_to":"1a122d0e_4e2304a8","updated":"2016-05-03 01:43:23.000000000","message":"Yes, see \"add data sync mechanism for keep db\"","commit_id":"809cf788717501f67296086a44beb6a3601981a6"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"8fd9e4326d8f943ddedf8fcaf974e633320553d2","unresolved":false,"context_lines":[{"line_number":26,"context_line":"without restarting as using \"CLUSTER FAILOVER\" command.\r"},{"line_number":27,"context_line":"In this case one slave will be promoted to master and the client\r"},{"line_number":28,"context_line":"could not get connection error but a MOVED error from server after\r"},{"line_number":29,"context_line":"sending request to the new slave node.\r"},{"line_number":30,"context_line":"\r"},{"line_number":31,"context_line":"Some data maybe lost in Redis HA because Redis does not\r"},{"line_number":32,"context_line":"provide strong consistency. So for this case,\r"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_61e06fcd","line":29,"updated":"2016-05-03 13:51:16.000000000","message":"Continuing the discussion about sentinel: I am not sure I understand. I have an environment where the sentinel instances monitor the redis servers, and send events when one is down. I think they are even capable of initiating HA (but this didn\u0027t work in my environment).","commit_id":"5eccfbcf477dd10e32d345aff01400380092dbaf"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"acc003f6058d6d0af8657d4c0811fd136df7a9ef","unresolved":false,"context_lines":[{"line_number":26,"context_line":"without restarting as using \"CLUSTER FAILOVER\" command.\r"},{"line_number":27,"context_line":"In this case one slave will be promoted to master and the client\r"},{"line_number":28,"context_line":"could not get connection error but a MOVED error from server after\r"},{"line_number":29,"context_line":"sending request to the new slave node.\r"},{"line_number":30,"context_line":"\r"},{"line_number":31,"context_line":"Some data maybe lost in Redis HA because Redis does not\r"},{"line_number":32,"context_line":"provide strong consistency. So for this case,\r"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_1c83c7aa","line":29,"in_reply_to":"1a122d0e_61e06fcd","updated":"2016-05-04 02:02:48.000000000","message":"Yes of course sentinel can do this too. Redis did not support a cluster function until last year. So there are many Redis cluster solutions: Codis, Twemproxy, Sentinel and the official Redis Cluster(version 3.0 or higher from last year) and so on. In Dragonflow i implemented the official Redis Cluster, it supports data-sharding, master-slave mode, some but not strong consistency, failover and other features just like other solutions. If you deploy the cluster according to this guide http://redis.io/topics/cluster-tutorial, then your cluster is the official Redis Cluster. And you can also use some commands as \"cluster info\" to check if or not because only this cluster solution supports this kind of commands. By the way in the years without Redis Cluster, most people liked using Twemproxy(twitter) and Codis to implement redis cluster.","commit_id":"5eccfbcf477dd10e32d345aff01400380092dbaf"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"c3d5243013f4c218f6752b40c117bd38dc39b1c2","unresolved":false,"context_lines":[{"line_number":31,"context_line":"Some data maybe lost in Redis HA because Redis does not\r"},{"line_number":32,"context_line":"provide strong consistency. So for this case,\r"},{"line_number":33,"context_line":"driver should notify DB Consistency module to resynchronize\r"},{"line_number":34,"context_line":"local data to Redis cluster after the Redis cluster finishing HA.\r"},{"line_number":35,"context_line":"\r"},{"line_number":36,"context_line":"The goal of this design is to describe how to\r"},{"line_number":37,"context_line":"keep available of Redis cluster if node crashing occurred.\r"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_cf27ceb3","line":34,"range":{"start_line":34,"start_character":0,"end_line":34,"end_character":10},"updated":"2016-05-04 08:57:23.000000000","message":"not the data in neutron db?","commit_id":"5eccfbcf477dd10e32d345aff01400380092dbaf"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"b73c4930bdf2b446ed29aaf270b197ac53a2e2d5","unresolved":false,"context_lines":[{"line_number":31,"context_line":"Some data maybe lost in Redis HA because Redis does not\r"},{"line_number":32,"context_line":"provide strong consistency. So for this case,\r"},{"line_number":33,"context_line":"driver should notify DB Consistency module to resynchronize\r"},{"line_number":34,"context_line":"local data to Redis cluster after the Redis cluster finishing HA.\r"},{"line_number":35,"context_line":"\r"},{"line_number":36,"context_line":"The goal of this design is to describe how to\r"},{"line_number":37,"context_line":"keep available of Redis cluster if node crashing occurred.\r"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_3f2f8b89","line":34,"range":{"start_line":34,"start_character":0,"end_line":34,"end_character":10},"in_reply_to":"1a122d0e_bb9c50c0","updated":"2016-05-04 09:27:12.000000000","message":"OK, I see.","commit_id":"5eccfbcf477dd10e32d345aff01400380092dbaf"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"75f7073b609c3ae9a11072f932a92d50dbb6ca36","unresolved":false,"context_lines":[{"line_number":31,"context_line":"Some data maybe lost in Redis HA because Redis does not\r"},{"line_number":32,"context_line":"provide strong consistency. So for this case,\r"},{"line_number":33,"context_line":"driver should notify DB Consistency module to resynchronize\r"},{"line_number":34,"context_line":"local data to Redis cluster after the Redis cluster finishing HA.\r"},{"line_number":35,"context_line":"\r"},{"line_number":36,"context_line":"The goal of this design is to describe how to\r"},{"line_number":37,"context_line":"keep available of Redis cluster if node crashing occurred.\r"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_bb9c50c0","line":34,"range":{"start_line":34,"start_character":0,"end_line":34,"end_character":10},"in_reply_to":"1a122d0e_cf27ceb3","updated":"2016-05-04 09:03:41.000000000","message":"Yes, in northbound it is the data in neutron db, and in southbound it is the local data of controller. This is done by the db consistency module witch spec is https://review.openstack.org/#/c/300877/","commit_id":"5eccfbcf477dd10e32d345aff01400380092dbaf"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"149b3c0b35196396ad146a6336f734b60e2105e7","unresolved":false,"context_lines":[{"line_number":47,"context_line":"If this step is done in each controller, there may have too many\r"},{"line_number":48,"context_line":"Dragonflow compute nodes read the DB cluster in the same time and\r"},{"line_number":49,"context_line":"redis cluster could hardly handle it.\r"},{"line_number":50,"context_line":"So create a detecting thread in NB plugin to read the DB topology information\r"},{"line_number":51,"context_line":"periodically when Neutron server starting and then send the information\r"},{"line_number":52,"context_line":"to all Dragonflow controllers to check if the DB cluster nodes changed.\r"},{"line_number":53,"context_line":"And controllers should subscribe a \"HA\" topic to receive messages from\r"},{"line_number":54,"context_line":"plugin.\r"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_58620ad9","line":51,"range":{"start_line":50,"start_character":0,"end_line":51,"end_character":41},"updated":"2016-05-22 14:33:56.000000000","message":"Why not send the info when it detect the status changed instead of send it peroidically?","commit_id":"45c15f8ad2b52e0b52e1ba2730474fc92d0bd7b5"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"089c981cae4ae4321218030a11b42403da91a4ef","unresolved":false,"context_lines":[{"line_number":47,"context_line":"If this step is done in each controller, there may have too many\r"},{"line_number":48,"context_line":"Dragonflow compute nodes read the DB cluster in the same time and\r"},{"line_number":49,"context_line":"redis cluster could hardly handle it.\r"},{"line_number":50,"context_line":"So create a detecting thread in NB plugin to read the DB topology information\r"},{"line_number":51,"context_line":"periodically when Neutron server starting and then send the information\r"},{"line_number":52,"context_line":"to all Dragonflow controllers to check if the DB cluster nodes changed.\r"},{"line_number":53,"context_line":"And controllers should subscribe a \"HA\" topic to receive messages from\r"},{"line_number":54,"context_line":"plugin.\r"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_6481278f","line":51,"range":{"start_line":50,"start_character":0,"end_line":51,"end_character":41},"in_reply_to":"bab6814e_58620ad9","updated":"2016-05-23 02:34:26.000000000","message":"Because db status maybe changed in a very short time and the detection task in plugin could not get it, but local controller may have been error.","commit_id":"45c15f8ad2b52e0b52e1ba2730474fc92d0bd7b5"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"2963c5a0d845c5e837d62568b6c59e30d06c73f6","unresolved":false,"context_lines":[{"line_number":47,"context_line":"If this step is done in each controller, there may have too many\r"},{"line_number":48,"context_line":"Dragonflow compute nodes read the DB cluster in the same time and\r"},{"line_number":49,"context_line":"redis cluster could hardly handle it.\r"},{"line_number":50,"context_line":"So create a detecting thread in NB plugin to read the DB topology information\r"},{"line_number":51,"context_line":"periodically when Neutron server starting and then send the information\r"},{"line_number":52,"context_line":"to all Dragonflow controllers to check if the DB cluster nodes changed.\r"},{"line_number":53,"context_line":"And controllers should subscribe a \"HA\" topic to receive messages from\r"},{"line_number":54,"context_line":"plugin.\r"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_8a6d35ce","line":51,"range":{"start_line":50,"start_character":0,"end_line":51,"end_character":41},"in_reply_to":"bab6814e_6481278f","updated":"2016-05-23 02:45:43.000000000","message":"I mean the detect task could detect the status periodically, but it should pub the info only when it finds the topology of cluster is changed.","commit_id":"45c15f8ad2b52e0b52e1ba2730474fc92d0bd7b5"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"149b3c0b35196396ad146a6336f734b60e2105e7","unresolved":false,"context_lines":[{"line_number":57,"context_line":"\r"},{"line_number":58,"context_line":"There are 3 types of connections between Redis client and cluster:\r"},{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"},{"line_number":60,"context_line":"2. pub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":61,"context_line":"3. sub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":62,"context_line":"\r"},{"line_number":63,"context_line":"For type 2/3 connection failure, it should hash to other node immediately.\r"},{"line_number":64,"context_line":"For type 1 connection failure, it will be updated after receiving messages sent\r"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_385d8615","line":61,"range":{"start_line":60,"start_character":0,"end_line":61,"end_character":71},"updated":"2016-05-22 14:33:56.000000000","message":"If the pub/sub description is the same, you can merge it :)","commit_id":"45c15f8ad2b52e0b52e1ba2730474fc92d0bd7b5"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"089c981cae4ae4321218030a11b42403da91a4ef","unresolved":false,"context_lines":[{"line_number":57,"context_line":"\r"},{"line_number":58,"context_line":"There are 3 types of connections between Redis client and cluster:\r"},{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"},{"line_number":60,"context_line":"2. pub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":61,"context_line":"3. sub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":62,"context_line":"\r"},{"line_number":63,"context_line":"For type 2/3 connection failure, it should hash to other node immediately.\r"},{"line_number":64,"context_line":"For type 1 connection failure, it will be updated after receiving messages sent\r"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_84508b33","line":61,"range":{"start_line":60,"start_character":0,"end_line":61,"end_character":71},"in_reply_to":"bab6814e_385d8615","updated":"2016-05-23 02:34:26.000000000","message":"yes","commit_id":"45c15f8ad2b52e0b52e1ba2730474fc92d0bd7b5"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"149b3c0b35196396ad146a6336f734b60e2105e7","unresolved":false,"context_lines":[{"line_number":71,"context_line":"\r"},{"line_number":72,"context_line":"Description to step 2\r"},{"line_number":73,"context_line":"------------------------\r"},{"line_number":74,"context_line":"After detecting the node failure, if the cluster state is ok and\r"},{"line_number":75,"context_line":"master node status is not fail, the driver will update\r"},{"line_number":76,"context_line":"the topology information and connections,\r"},{"line_number":77,"context_line":"then a \"recover\" message will be sent to db consist module.\r"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_78a1cef7","line":74,"range":{"start_line":74,"start_character":34,"end_line":74,"end_character":61},"updated":"2016-05-22 14:33:56.000000000","message":"How do the driver know the cluster is ok after detect the node failure, and how to judge the cluster state is ok? Could you please describe it more detail?","commit_id":"45c15f8ad2b52e0b52e1ba2730474fc92d0bd7b5"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"089c981cae4ae4321218030a11b42403da91a4ef","unresolved":false,"context_lines":[{"line_number":71,"context_line":"\r"},{"line_number":72,"context_line":"Description to step 2\r"},{"line_number":73,"context_line":"------------------------\r"},{"line_number":74,"context_line":"After detecting the node failure, if the cluster state is ok and\r"},{"line_number":75,"context_line":"master node status is not fail, the driver will update\r"},{"line_number":76,"context_line":"the topology information and connections,\r"},{"line_number":77,"context_line":"then a \"recover\" message will be sent to db consist module.\r"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_841ecb6d","line":74,"range":{"start_line":74,"start_character":34,"end_line":74,"end_character":61},"in_reply_to":"bab6814e_78a1cef7","updated":"2016-05-23 02:34:26.000000000","message":"yes, here\u0027s description is not clear","commit_id":"45c15f8ad2b52e0b52e1ba2730474fc92d0bd7b5"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"f1a49790b9e502ea5a9f9e028e5582b1598fac6f","unresolved":false,"context_lines":[{"line_number":55,"context_line":"In Dragonflow controller, it never read nodes information from Redis cluster\r"},{"line_number":56,"context_line":"after initialization but only listen the messages from detecting task from plugin.\r"},{"line_number":57,"context_line":"\r"},{"line_number":58,"context_line":"There are 3 types of connections between Redis client and cluster:\r"},{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"},{"line_number":60,"context_line":"2. pub/sub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":61,"context_line":"\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_ca23bd54","line":58,"range":{"start_line":58,"start_character":10,"end_line":58,"end_character":12},"updated":"2016-05-23 03:02:04.000000000","message":"2","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"5a3ed70751aeab1a877550854b92cda25199d404","unresolved":false,"context_lines":[{"line_number":55,"context_line":"In Dragonflow controller, it never read nodes information from Redis cluster\r"},{"line_number":56,"context_line":"after initialization but only listen the messages from detecting task from plugin.\r"},{"line_number":57,"context_line":"\r"},{"line_number":58,"context_line":"There are 3 types of connections between Redis client and cluster:\r"},{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"},{"line_number":60,"context_line":"2. pub/sub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":61,"context_line":"\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_ea704142","line":58,"range":{"start_line":58,"start_character":10,"end_line":58,"end_character":12},"in_reply_to":"bab6814e_ca23bd54","updated":"2016-05-23 03:19:39.000000000","message":"Done","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"f1a49790b9e502ea5a9f9e028e5582b1598fac6f","unresolved":false,"context_lines":[{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"},{"line_number":60,"context_line":"2. pub/sub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":61,"context_line":"\r"},{"line_number":62,"context_line":"For type 2/3 connection failure, it should hash to other node immediately.\r"},{"line_number":63,"context_line":"For type 1 connection failure, it will be updated after receiving messages sent\r"},{"line_number":64,"context_line":"by detecting task.\r"},{"line_number":65,"context_line":"Either connection error or MOVED error detected in Redis driver refers to\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_6a28d133","line":62,"range":{"start_line":62,"start_character":9,"end_line":62,"end_character":12},"updated":"2016-05-23 03:02:04.000000000","message":"2","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"5a3ed70751aeab1a877550854b92cda25199d404","unresolved":false,"context_lines":[{"line_number":59,"context_line":"1. read/write connection, client connects to every Redis master nodes.\r"},{"line_number":60,"context_line":"2. pub/sub connection, client connects to one of the cluster nodes by hash.\r"},{"line_number":61,"context_line":"\r"},{"line_number":62,"context_line":"For type 2/3 connection failure, it should hash to other node immediately.\r"},{"line_number":63,"context_line":"For type 1 connection failure, it will be updated after receiving messages sent\r"},{"line_number":64,"context_line":"by detecting task.\r"},{"line_number":65,"context_line":"Either connection error or MOVED error detected in Redis driver refers to\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_4a922d4d","line":62,"range":{"start_line":62,"start_character":9,"end_line":62,"end_character":12},"in_reply_to":"bab6814e_6a28d133","updated":"2016-05-23 03:19:39.000000000","message":"Done","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"f1a49790b9e502ea5a9f9e028e5582b1598fac6f","unresolved":false,"context_lines":[{"line_number":70,"context_line":"\r"},{"line_number":71,"context_line":"Description to step 2\r"},{"line_number":72,"context_line":"------------------------\r"},{"line_number":73,"context_line":"After receiving the cluster information from plugin, locall controller will\r"},{"line_number":74,"context_line":"compare the new nodes with the old nodes and update the topology information\r"},{"line_number":75,"context_line":"and connections,\r"},{"line_number":76,"context_line":"then a \"dbrestart\" message will be sent to db consist module.\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_8a25355d","line":73,"range":{"start_line":73,"start_character":53,"end_line":73,"end_character":59},"updated":"2016-05-23 03:02:04.000000000","message":"local","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"5a3ed70751aeab1a877550854b92cda25199d404","unresolved":false,"context_lines":[{"line_number":70,"context_line":"\r"},{"line_number":71,"context_line":"Description to step 2\r"},{"line_number":72,"context_line":"------------------------\r"},{"line_number":73,"context_line":"After receiving the cluster information from plugin, locall controller will\r"},{"line_number":74,"context_line":"compare the new nodes with the old nodes and update the topology information\r"},{"line_number":75,"context_line":"and connections,\r"},{"line_number":76,"context_line":"then a \"dbrestart\" message will be sent to db consist module.\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_6a9bb166","line":73,"range":{"start_line":73,"start_character":53,"end_line":73,"end_character":59},"in_reply_to":"bab6814e_8a25355d","updated":"2016-05-23 03:19:39.000000000","message":"Done","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"f1a49790b9e502ea5a9f9e028e5582b1598fac6f","unresolved":false,"context_lines":[{"line_number":73,"context_line":"After receiving the cluster information from plugin, locall controller will\r"},{"line_number":74,"context_line":"compare the new nodes with the old nodes and update the topology information\r"},{"line_number":75,"context_line":"and connections,\r"},{"line_number":76,"context_line":"then a \"dbrestart\" message will be sent to db consist module.\r"},{"line_number":77,"context_line":"\r"},{"line_number":78,"context_line":"The following diagram shows the procedure of Dragonflow:\r"},{"line_number":79,"context_line":"\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_2a3249a7","line":76,"range":{"start_line":76,"start_character":0,"end_line":76,"end_character":61},"updated":"2016-05-23 03:02:04.000000000","message":"Will the driver send out dbrestart message after it receives the cluster info from plugin every time? I think it should notify the message only when it finds the error happened before has been recovered.","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"5a3ed70751aeab1a877550854b92cda25199d404","unresolved":false,"context_lines":[{"line_number":73,"context_line":"After receiving the cluster information from plugin, locall controller will\r"},{"line_number":74,"context_line":"compare the new nodes with the old nodes and update the topology information\r"},{"line_number":75,"context_line":"and connections,\r"},{"line_number":76,"context_line":"then a \"dbrestart\" message will be sent to db consist module.\r"},{"line_number":77,"context_line":"\r"},{"line_number":78,"context_line":"The following diagram shows the procedure of Dragonflow:\r"},{"line_number":79,"context_line":"\r"}],"source_content_type":"text/x-rst","patch_set":12,"id":"bab6814e_aa3d992c","line":76,"range":{"start_line":76,"start_character":0,"end_line":76,"end_character":61},"in_reply_to":"bab6814e_2a3249a7","updated":"2016-05-23 03:19:39.000000000","message":"yes, it sends the dbrestart message only when nodes changed and all nodes connecton are working normally. This means that db cluster may have been recovered.","commit_id":"eb29c06b1538730ba5a81dd4e511e19f66674b02"}]}