)]}'
{"doc/source/specs/keep_db_consistency.rst":[{"author":{"_account_id":20405,"name":"liuhaixia","email":"liuhaixia315@huawei.com","username":"liuhaixia"},"change_message_id":"4af1c01efcf5ddc12f0a2a41d186481416172484","unresolved":false,"context_lines":[{"line_number":20,"context_line":"As a result, the question becomes how to guarantee the atomicity of DB"},{"line_number":21,"context_line":"operations across two distinct kinds of database in the one API session."},{"line_number":22,"context_line":"Here we will propose a simple and feasible distributed lock solution and"},{"line_number":23,"context_line":"improve it later according to evaluation in production. We also provide a"},{"line_number":24,"context_line":"effective data synchronization mechanism to resolve the data inconsistency"},{"line_number":25,"context_line":"problem which may happen in the communication among above three type of"},{"line_number":26,"context_line":"data stores."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_cb36f5b3","line":23,"range":{"start_line":23,"start_character":71,"end_line":23,"end_character":73},"updated":"2016-04-04 10:07:02.000000000","message":"an","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"298e38e0ef89c197bb48bc5fdc59d8fc6dfe92d7","unresolved":false,"context_lines":[{"line_number":20,"context_line":"As a result, the question becomes how to guarantee the atomicity of DB"},{"line_number":21,"context_line":"operations across two distinct kinds of database in the one API session."},{"line_number":22,"context_line":"Here we will propose a simple and feasible distributed lock solution and"},{"line_number":23,"context_line":"improve it later according to evaluation in production. We also provide a"},{"line_number":24,"context_line":"effective data synchronization mechanism to resolve the data inconsistency"},{"line_number":25,"context_line":"problem which may happen in the communication among above three type of"},{"line_number":26,"context_line":"data stores."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_ce2535ac","line":23,"range":{"start_line":23,"start_character":71,"end_line":23,"end_character":73},"in_reply_to":"9a061dce_cb36f5b3","updated":"2016-04-05 01:56:56.000000000","message":"ok:)","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20405,"name":"liuhaixia","email":"liuhaixia315@huawei.com","username":"liuhaixia"},"change_message_id":"4af1c01efcf5ddc12f0a2a41d186481416172484","unresolved":false,"context_lines":[{"line_number":115,"context_line":"the complexity of introducing other sophisticated synchronization mechanism"},{"line_number":116,"context_line":"from scratch."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"The distributed lock is tenant-based and each tenant has its own lock in the"},{"line_number":119,"context_line":"database. Due to this design, we can allow concurrency to a certain degree."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"In Neutron plugin:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_266fe893","line":118,"range":{"start_line":118,"start_character":23,"end_line":118,"end_character":41},"updated":"2016-04-04 10:07:02.000000000","message":"If so , all the tenant\u0027s resource operation need to wait lock release.\nNetwork/Subnet/Port/Router...\nWhy not use tenant plus neutron resource type as lock?","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"298e38e0ef89c197bb48bc5fdc59d8fc6dfe92d7","unresolved":false,"context_lines":[{"line_number":115,"context_line":"the complexity of introducing other sophisticated synchronization mechanism"},{"line_number":116,"context_line":"from scratch."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"The distributed lock is tenant-based and each tenant has its own lock in the"},{"line_number":119,"context_line":"database. Due to this design, we can allow concurrency to a certain degree."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"In Neutron plugin:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_4e49255f","line":118,"range":{"start_line":118,"start_character":23,"end_line":118,"end_character":41},"in_reply_to":"9a061dce_266fe893","updated":"2016-04-05 01:56:56.000000000","message":"this is phase 1 design commited by Li Ma, maybe the we can have specifical and better dssign in future:)","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20405,"name":"liuhaixia","email":"liuhaixia315@huawei.com","username":"liuhaixia"},"change_message_id":"4af1c01efcf5ddc12f0a2a41d186481416172484","unresolved":false,"context_lines":[{"line_number":133,"context_line":"* When acquiring the distributed lock, it first issue SELECT-FOR-UPDATE to"},{"line_number":134,"context_line":"check the lock has been obtained or not. If not, it updates the lock state"},{"line_number":135,"context_line":"and commits the transaction. If exception happens, it will re-try it for"},{"line_number":136,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":139,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_e6169054","line":136,"range":{"start_line":136,"start_character":45,"end_line":136,"end_character":69},"updated":"2016-04-04 10:07:02.000000000","message":"If it get lock failed, does it mean lock owned by others?\nIf so, plugin or driver needs to check if corresponding records has existed.","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"298e38e0ef89c197bb48bc5fdc59d8fc6dfe92d7","unresolved":false,"context_lines":[{"line_number":133,"context_line":"* When acquiring the distributed lock, it first issue SELECT-FOR-UPDATE to"},{"line_number":134,"context_line":"check the lock has been obtained or not. If not, it updates the lock state"},{"line_number":135,"context_line":"and commits the transaction. If exception happens, it will re-try it for"},{"line_number":136,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":139,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_8e0a1dd1","line":136,"range":{"start_line":136,"start_character":45,"end_line":136,"end_character":69},"in_reply_to":"9a061dce_e6169054","updated":"2016-04-05 01:56:56.000000000","message":"yes, if someone else owned the lock, it will get lock failed, plugins will do the operation as usual after they get the lock one by one","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20405,"name":"liuhaixia","email":"liuhaixia315@huawei.com","username":"liuhaixia"},"change_message_id":"4af1c01efcf5ddc12f0a2a41d186481416172484","unresolved":false,"context_lines":[{"line_number":233,"context_line":"When DB driver and pub\\sub driver find the read\\write connection between"},{"line_number":234,"context_line":"Neutron plugin and DF DB, and also the pub\\sub connection between Neutron"},{"line_number":235,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":236,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"    Start handle the revcover message:"},{"line_number":239,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_f1bac86d","line":236,"range":{"start_line":236,"start_character":24,"end_line":236,"end_character":66},"updated":"2016-04-04 10:07:02.000000000","message":"Does each plugin do recover operation, or just one plugin do it?","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"298e38e0ef89c197bb48bc5fdc59d8fc6dfe92d7","unresolved":false,"context_lines":[{"line_number":233,"context_line":"When DB driver and pub\\sub driver find the read\\write connection between"},{"line_number":234,"context_line":"Neutron plugin and DF DB, and also the pub\\sub connection between Neutron"},{"line_number":235,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":236,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"    Start handle the revcover message:"},{"line_number":239,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_2e14b142","line":236,"range":{"start_line":236,"start_character":24,"end_line":236,"end_character":66},"in_reply_to":"9a061dce_f1bac86d","updated":"2016-04-05 01:56:56.000000000","message":"please see the Neutron plugin election below, only the master Neutron plugin will do the work","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20405,"name":"liuhaixia","email":"liuhaixia315@huawei.com","username":"liuhaixia"},"change_message_id":"4af1c01efcf5ddc12f0a2a41d186481416172484","unresolved":false,"context_lines":[{"line_number":263,"context_line":"    def get_master_neutron_plugin(context):"},{"line_number":264,"context_line":"        nb_lock \u003d lock_db.DBLock(context.election_key)"},{"line_number":265,"context_line":"        with nb_lock:"},{"line_number":266,"context_line":"            if db_api.get_master_plugin_name() \u003d\u003d self.plugin_name:"},{"line_number":267,"context_line":"                db_api.set_master_plugin_time(self.current_time)"},{"line_number":268,"context_line":"                return True"},{"line_number":269,"context_line":"            elif self.current_time \u003e master_old_time + timeout:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_4c203b4a","line":266,"range":{"start_line":266,"start_character":55,"end_line":266,"end_character":66},"updated":"2016-04-04 10:07:02.000000000","message":"How does each neutron server get plugin name ? By configuration?\nHow to deal time synchronization?","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"298e38e0ef89c197bb48bc5fdc59d8fc6dfe92d7","unresolved":false,"context_lines":[{"line_number":263,"context_line":"    def get_master_neutron_plugin(context):"},{"line_number":264,"context_line":"        nb_lock \u003d lock_db.DBLock(context.election_key)"},{"line_number":265,"context_line":"        with nb_lock:"},{"line_number":266,"context_line":"            if db_api.get_master_plugin_name() \u003d\u003d self.plugin_name:"},{"line_number":267,"context_line":"                db_api.set_master_plugin_time(self.current_time)"},{"line_number":268,"context_line":"                return True"},{"line_number":269,"context_line":"            elif self.current_time \u003e master_old_time + timeout:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a061dce_ee09192e","line":266,"range":{"start_line":266,"start_character":55,"end_line":266,"end_character":66},"in_reply_to":"9a061dce_4c203b4a","updated":"2016-04-05 01:56:56.000000000","message":"yes, we can assign a unique name for each plugin like the chassis, If Neutron plugins support NTP, it is good, if not, each plugin should keep the time data for each plugin.","commit_id":"8c413a07fe2f827b1b9cbdf1587ecc86df22dbfc"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2e34edb290cc3b658adaac802315b1883f4f0b1a","unresolved":false,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Keep DB Consistency between Neutron and Dragonflow - Phase 1"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"The URL of your launchpad Blueprint:"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_04394ea4","line":8,"range":{"start_line":8,"start_character":50,"end_line":8,"end_character":60},"updated":"2016-04-08 13:11:54.000000000","message":"Remove it because you are working on the same spec. There is no Phase 2.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"c9d605a23ae6e00849f5f8882bdb034ab2c32ae6","unresolved":false,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Keep DB Consistency between Neutron and Dragonflow - Phase 1"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"The URL of your launchpad Blueprint:"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_ded4e748","line":8,"range":{"start_line":8,"start_character":50,"end_line":8,"end_character":60},"in_reply_to":"9a061dce_04394ea4","updated":"2016-04-11 11:07:05.000000000","message":"ok","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"c8ff245e50e43638b39146013ee5436640d7bef1","unresolved":false,"context_lines":[{"line_number":79,"context_line":"After Dragonflow local controller subscribe its concerned data, the sub/pub"},{"line_number":80,"context_line":"connection between local controller and sub/pub system is down because of the"},{"line_number":81,"context_line":"breakdown of network or sub/pub system, so local controller will lose the new"},{"line_number":82,"context_line":"data. Another case is after local controller receives the pub message, it"},{"line_number":83,"context_line":"happens internal exception in the data process, then the new data will be"},{"line_number":84,"context_line":"dropped."},{"line_number":85,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_6a5bec2e","line":82,"range":{"start_line":82,"start_character":6,"end_line":82,"end_character":18},"updated":"2016-04-06 08:37:26.000000000","message":"I cannot figure out this case.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"dbae38bc92e6dad6950c8a85c157fe684adb2402","unresolved":false,"context_lines":[{"line_number":79,"context_line":"After Dragonflow local controller subscribe its concerned data, the sub/pub"},{"line_number":80,"context_line":"connection between local controller and sub/pub system is down because of the"},{"line_number":81,"context_line":"breakdown of network or sub/pub system, so local controller will lose the new"},{"line_number":82,"context_line":"data. Another case is after local controller receives the pub message, it"},{"line_number":83,"context_line":"happens internal exception in the data process, then the new data will be"},{"line_number":84,"context_line":"dropped."},{"line_number":85,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_fb04ecc0","line":82,"range":{"start_line":82,"start_character":6,"end_line":82,"end_character":18},"in_reply_to":"9a061dce_6a5bec2e","updated":"2016-04-06 09:04:44.000000000","message":"If there happened some internal exception when process the message, the data maybe dropped before it is stored into memory cache, so it is a possible case:)","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"c8ff245e50e43638b39146013ee5436640d7bef1","unresolved":false,"context_lines":[{"line_number":136,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":139,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."},{"line_number":140,"context_line":"If deadlock exception happens, it will retry it to make sure it will be"},{"line_number":141,"context_line":"committed later."},{"line_number":142,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_051fc35f","line":139,"range":{"start_line":139,"start_character":19,"end_line":139,"end_character":29},"updated":"2016-04-06 08:37:26.000000000","message":"i think it is blocked","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"e2f5057f65539547e12d051ce4ff9c1fdfa3516c","unresolved":false,"context_lines":[{"line_number":136,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":139,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."},{"line_number":140,"context_line":"If deadlock exception happens, it will retry it to make sure it will be"},{"line_number":141,"context_line":"committed later."},{"line_number":142,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_8e86bab7","line":139,"range":{"start_line":139,"start_character":19,"end_line":139,"end_character":29},"in_reply_to":"7a1b2928_78ca0b51","updated":"2016-04-11 12:10:03.000000000","message":"https://www.percona.com/blog/2014/09/11/openstack-users-shed-light-on-percona-xtradb-cluster-deadlock-issues/\nThis is the simple analysis provided by percona. Percona cluster has a similar behavior of Mysql Galera cluster. The solution in the post is just saying \u0027do not use it\u0027. But another solution provided by oslo.db is \u0027retry_on_deadlock\u0027 which is used in this distributed lock.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"dbae38bc92e6dad6950c8a85c157fe684adb2402","unresolved":false,"context_lines":[{"line_number":136,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":139,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."},{"line_number":140,"context_line":"If deadlock exception happens, it will retry it to make sure it will be"},{"line_number":141,"context_line":"committed later."},{"line_number":142,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_7bb57cec","line":139,"range":{"start_line":139,"start_character":19,"end_line":139,"end_character":29},"in_reply_to":"9a061dce_051fc35f","updated":"2016-04-06 09:04:44.000000000","message":"yes","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"c9d605a23ae6e00849f5f8882bdb034ab2c32ae6","unresolved":false,"context_lines":[{"line_number":136,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":139,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."},{"line_number":140,"context_line":"If deadlock exception happens, it will retry it to make sure it will be"},{"line_number":141,"context_line":"committed later."},{"line_number":142,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_78ca0b51","line":139,"range":{"start_line":139,"start_character":19,"end_line":139,"end_character":29},"in_reply_to":"9a061dce_1872efad","updated":"2016-04-11 11:07:05.000000000","message":"are there any resource race in the DB cluster? if deadlock happened as you describe, how the program could be released from the deadlock, who will release the racing resource firstly in the db?","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2e34edb290cc3b658adaac802315b1883f4f0b1a","unresolved":false,"context_lines":[{"line_number":136,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":139,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."},{"line_number":140,"context_line":"If deadlock exception happens, it will retry it to make sure it will be"},{"line_number":141,"context_line":"committed later."},{"line_number":142,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_1872efad","line":139,"range":{"start_line":139,"start_character":19,"end_line":139,"end_character":29},"in_reply_to":"9a061dce_7bb57cec","updated":"2016-04-08 13:11:54.000000000","message":"The lock is local state and cannot join the certification process of Galera. It will be deadlocked in Galera, not blocked.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2e34edb290cc3b658adaac802315b1883f4f0b1a","unresolved":false,"context_lines":[{"line_number":203,"context_line":"    except:"},{"line_number":204,"context_line":"        retry to do DF DB operations and PUB/SUB."},{"line_number":205,"context_line":"        if beyond max retry times:"},{"line_number":206,"context_line":"            rollback Neutron DB operations."},{"line_number":207,"context_line":"            raise creation exception."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_64392ad7","line":206,"range":{"start_line":206,"start_character":12,"end_line":206,"end_character":43},"updated":"2016-04-08 13:11:54.000000000","message":"You cannot rollback the committed Neutron transaction. If you wanna do it, you should start a new transaction for reverse operations.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"c9d605a23ae6e00849f5f8882bdb034ab2c32ae6","unresolved":false,"context_lines":[{"line_number":203,"context_line":"    except:"},{"line_number":204,"context_line":"        retry to do DF DB operations and PUB/SUB."},{"line_number":205,"context_line":"        if beyond max retry times:"},{"line_number":206,"context_line":"            rollback Neutron DB operations."},{"line_number":207,"context_line":"            raise creation exception."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_38353346","line":206,"range":{"start_line":206,"start_character":12,"end_line":206,"end_character":43},"in_reply_to":"9a061dce_64392ad7","updated":"2016-04-11 11:07:05.000000000","message":"yes, I\u0027ll start another transaction to rollback it, not rollback in old transaction","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2e34edb290cc3b658adaac802315b1883f4f0b1a","unresolved":false,"context_lines":[{"line_number":209,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":210,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"},{"line_number":211,"context_line":"process, Neutron plugin should retry several times to finish commits and"},{"line_number":212,"context_line":"PUB/SUB, if it failed after all the attempts, Neutron plugin should rollback"},{"line_number":213,"context_line":"the previous commit, and raise a creation exception."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"When Neutron plugin receives a update\\delete object(router\\network\\subnet\\port,"},{"line_number":216,"context_line":"etc) invoke:"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_e4537a15","line":213,"range":{"start_line":212,"start_character":46,"end_line":213,"end_character":52},"updated":"2016-04-08 13:11:54.000000000","message":"ditto.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20297,"name":"duankebo","email":"duankebo@huawei.com","username":"duankebo"},"change_message_id":"9f76d5a82457944656713915505528e518574900","unresolved":false,"context_lines":[{"line_number":236,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"    Start handle the revcover message:"},{"line_number":239,"context_line":"        pull data from DF DB."},{"line_number":240,"context_line":"        pull data from Neutron DB."},{"line_number":241,"context_line":"        do compare with two data set."},{"line_number":242,"context_line":"        if found object create\\update\\delete :"},{"line_number":243,"context_line":"            try:"},{"line_number":244,"context_line":"                do DF DB operations."}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_50f9f4c7","line":241,"range":{"start_line":239,"start_character":8,"end_line":241,"end_character":14},"updated":"2016-04-11 03:02:00.000000000","message":"during this period, df/neutron db are still changing. So you are not comparing two static db. How to make sure no errors are introduced?","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"c9d605a23ae6e00849f5f8882bdb034ab2c32ae6","unresolved":false,"context_lines":[{"line_number":236,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"    Start handle the revcover message:"},{"line_number":239,"context_line":"        pull data from DF DB."},{"line_number":240,"context_line":"        pull data from Neutron DB."},{"line_number":241,"context_line":"        do compare with two data set."},{"line_number":242,"context_line":"        if found object create\\update\\delete :"},{"line_number":243,"context_line":"            try:"},{"line_number":244,"context_line":"                do DF DB operations."}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_7eddd36f","line":241,"range":{"start_line":239,"start_character":8,"end_line":241,"end_character":14},"in_reply_to":"7a1b2928_50f9f4c7","updated":"2016-04-11 11:07:05.000000000","message":"we can mark the create\\update\\delete status for each object at the first time db comparison, and after the second time db comparison, if the status of one object is unchanged, we can make sure the create\\update\\delete operation, and if the status is changed, we should remark the status for the object by the latest status and then wait for next db comparison:)","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2e34edb290cc3b658adaac802315b1883f4f0b1a","unresolved":false,"context_lines":[{"line_number":250,"context_line":""},{"line_number":251,"context_line":"* Optionally, we could start a green thread to do the data comparison between"},{"line_number":252,"context_line":"Neutron DB and DF DB periodically in Neutron plugin, if we found the"},{"line_number":253,"context_line":"performance bottleneck for Neutron plugin, we could consider the third"},{"line_number":254,"context_line":"software to do this."},{"line_number":255,"context_line":""},{"line_number":256,"context_line":"Neutron Plugin election"},{"line_number":257,"context_line":"-----------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_049dcebb","line":254,"range":{"start_line":253,"start_character":61,"end_line":254,"end_character":8},"updated":"2016-04-08 13:11:54.000000000","message":"what is the 3rd software?","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"c9d605a23ae6e00849f5f8882bdb034ab2c32ae6","unresolved":false,"context_lines":[{"line_number":250,"context_line":""},{"line_number":251,"context_line":"* Optionally, we could start a green thread to do the data comparison between"},{"line_number":252,"context_line":"Neutron DB and DF DB periodically in Neutron plugin, if we found the"},{"line_number":253,"context_line":"performance bottleneck for Neutron plugin, we could consider the third"},{"line_number":254,"context_line":"software to do this."},{"line_number":255,"context_line":""},{"line_number":256,"context_line":"Neutron Plugin election"},{"line_number":257,"context_line":"-----------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_d8999f21","line":254,"range":{"start_line":253,"start_character":61,"end_line":254,"end_character":8},"in_reply_to":"9a061dce_049dcebb","updated":"2016-04-11 11:07:05.000000000","message":"3rd-part software, maybe an additional process or system OM tools, this senaria may refer to the dragonflow deployment in production environment:)","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"c8ff245e50e43638b39146013ee5436640d7bef1","unresolved":false,"context_lines":[{"line_number":273,"context_line":"            else:"},{"line_number":274,"context_line":"                return False"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"* Each Neutron plugin will detect the current master periodically, and when"},{"line_number":277,"context_line":"it find the master is itself, it will handle the recover message as discussed"},{"line_number":278,"context_line":"above, else it will do nothing for the message."},{"line_number":279,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_05e7c3b0","line":276,"range":{"start_line":276,"start_character":2,"end_line":276,"end_character":65},"updated":"2016-04-06 08:37:26.000000000","message":"what will you do if there is a network partition.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"dbae38bc92e6dad6950c8a85c157fe684adb2402","unresolved":false,"context_lines":[{"line_number":273,"context_line":"            else:"},{"line_number":274,"context_line":"                return False"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"* Each Neutron plugin will detect the current master periodically, and when"},{"line_number":277,"context_line":"it find the master is itself, it will handle the recover message as discussed"},{"line_number":278,"context_line":"above, else it will do nothing for the message."},{"line_number":279,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_7b9a9c64","line":276,"range":{"start_line":276,"start_character":2,"end_line":276,"end_character":65},"in_reply_to":"9a061dce_05e7c3b0","updated":"2016-04-06 09:04:44.000000000","message":"Neutron plugin detect each other by read DF DB, not communicate by each other directly, so it is not concerned about the Neutron plugins network, it depends on the network between Neutron plugin and DF DB","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2e34edb290cc3b658adaac802315b1883f4f0b1a","unresolved":false,"context_lines":[{"line_number":280,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"},{"line_number":281,"context_line":"not be released, so each Neutron plugin should check the created-time in the"},{"line_number":282,"context_line":"db-lock and if it found the db-lock is timeout, it could own the db-lock"},{"line_number":283,"context_line":"instead of the crashed Neutron plugin."},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"Local Controller Data Sync"},{"line_number":286,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_e4889a73","line":283,"updated":"2016-04-08 13:11:54.000000000","message":"+1, set a timeout for each db-lock at creation.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"c9d605a23ae6e00849f5f8882bdb034ab2c32ae6","unresolved":false,"context_lines":[{"line_number":280,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"},{"line_number":281,"context_line":"not be released, so each Neutron plugin should check the created-time in the"},{"line_number":282,"context_line":"db-lock and if it found the db-lock is timeout, it could own the db-lock"},{"line_number":283,"context_line":"instead of the crashed Neutron plugin."},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"Local Controller Data Sync"},{"line_number":286,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_d8c07fe3","line":283,"in_reply_to":"9a061dce_e4889a73","updated":"2016-04-11 11:07:05.000000000","message":"yes","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2e34edb290cc3b658adaac802315b1883f4f0b1a","unresolved":false,"context_lines":[{"line_number":315,"context_line":"2. Create DB schema for distributed lock. (DONE)"},{"line_number":316,"context_line":"3. Implement distributed lock. (DONE)"},{"line_number":317,"context_line":"4. Protect all the API operations by distributed lock. (DONE)"},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Potential Improvements"},{"line_number":320,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":321,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a061dce_64ae4ae7","line":318,"updated":"2016-04-08 13:11:54.000000000","message":"You can list your work items here.\n\nBy the way, you should add a section for ML2 compatibility. Because ML2 split the core plugin logic into two parts, one is precommit, the other is postcommit. The precommit operation is inside the ML2 DB transaction, while the postcommit operation is after the ML2 DB transaction is done.","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"c9d605a23ae6e00849f5f8882bdb034ab2c32ae6","unresolved":false,"context_lines":[{"line_number":315,"context_line":"2. Create DB schema for distributed lock. (DONE)"},{"line_number":316,"context_line":"3. Implement distributed lock. (DONE)"},{"line_number":317,"context_line":"4. Protect all the API operations by distributed lock. (DONE)"},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Potential Improvements"},{"line_number":320,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":321,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a1b2928_18179752","line":318,"in_reply_to":"9a061dce_64ae4ae7","updated":"2016-04-11 11:07:05.000000000","message":"yes, I have consider this in previous description, but maybe more detail as you suggest in the additional ML2 section","commit_id":"098d3d2f8d521121cbd207fde3629632e5864127"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"a32d6e3cb019fbcff4ae87e1c1d068fe2b0b50f1","unresolved":false,"context_lines":[{"line_number":238,"context_line":"is changing dynamically, for example, if we find an additional port in Neutron"},{"line_number":239,"context_line":"DB than DF DB during data comparison, then the port may be deleted, if"},{"line_number":240,"context_line":"the delete operation is happened earlier than the create operation, the dirty"},{"line_number":241,"context_line":"data of this port will be stored in DF DB, so we introduce a verification"},{"line_number":242,"context_line":"mechanism for the message process."},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"We can store the create\\update\\delete status for each object at the first time"},{"line_number":245,"context_line":"db comparison, and after the second time db comparison, if the status of one"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a18252c_da1cbcc3","line":242,"range":{"start_line":241,"start_character":43,"end_line":242,"end_character":34},"updated":"2016-04-13 09:13:25.000000000","message":"During the synchronization period (data pulling and comparison period), we need to fence the related objects to prevent from concurrent read/write APIs. It can be done by the distributed lock.\n\nI\u0027m not sure how you synchronize the data. I suggest you do object-based synchronization. It can allow concurrency. If you pull the full database, you have to lock the full database. It is not optimal.","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"7ffdbefad41164a68bc7734a972d8984f6c7455a","unresolved":false,"context_lines":[{"line_number":238,"context_line":"is changing dynamically, for example, if we find an additional port in Neutron"},{"line_number":239,"context_line":"DB than DF DB during data comparison, then the port may be deleted, if"},{"line_number":240,"context_line":"the delete operation is happened earlier than the create operation, the dirty"},{"line_number":241,"context_line":"data of this port will be stored in DF DB, so we introduce a verification"},{"line_number":242,"context_line":"mechanism for the message process."},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"We can store the create\\update\\delete status for each object at the first time"},{"line_number":245,"context_line":"db comparison, and after the second time db comparison, if the status of one"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1a122d0e_de09632e","line":242,"range":{"start_line":241,"start_character":43,"end_line":242,"end_character":34},"in_reply_to":"5a18252c_da1cbcc3","updated":"2016-04-18 01:38:57.000000000","message":"All the operations are DB read during the sync period, we do not need lock to block any process, but this may introduce the problem that duankebo said, and we can deal with it:)","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"a32d6e3cb019fbcff4ae87e1c1d068fe2b0b50f1","unresolved":false,"context_lines":[{"line_number":241,"context_line":"data of this port will be stored in DF DB, so we introduce a verification"},{"line_number":242,"context_line":"mechanism for the message process."},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"We can store the create\\update\\delete status for each object at the first time"},{"line_number":245,"context_line":"db comparison, and after the second time db comparison, if the status of one"},{"line_number":246,"context_line":"object is unchanged, we can confirm the create\\update\\delete operation and do"},{"line_number":247,"context_line":"the corresponding operations, but if the status is changed, we should flush"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a18252c_d5c00f2d","line":244,"range":{"start_line":244,"start_character":17,"end_line":244,"end_character":44},"updated":"2016-04-13 09:13:25.000000000","message":"neutron db object has timestamps which can be compared. you don\u0027t need to add a new status for objects.","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"7ffdbefad41164a68bc7734a972d8984f6c7455a","unresolved":false,"context_lines":[{"line_number":241,"context_line":"data of this port will be stored in DF DB, so we introduce a verification"},{"line_number":242,"context_line":"mechanism for the message process."},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"We can store the create\\update\\delete status for each object at the first time"},{"line_number":245,"context_line":"db comparison, and after the second time db comparison, if the status of one"},{"line_number":246,"context_line":"object is unchanged, we can confirm the create\\update\\delete operation and do"},{"line_number":247,"context_line":"the corresponding operations, but if the status is changed, we should flush"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1a122d0e_3e3217f2","line":244,"range":{"start_line":244,"start_character":17,"end_line":244,"end_character":44},"in_reply_to":"5a18252c_d5c00f2d","updated":"2016-04-18 01:38:57.000000000","message":"the status here is just create\\update\\delete, it defines the temporary status for each object, for example, if the status of a port is delete in the first time db comparison, we should keep it and if we find the port is deleted in the second time db comparison, we can make sure the deleted port and delete it in DF DB and pub the event","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"a32d6e3cb019fbcff4ae87e1c1d068fe2b0b50f1","unresolved":false,"context_lines":[{"line_number":248,"context_line":"the status for the object by the latest status and then wait for next db"},{"line_number":249,"context_line":"comparison."},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    Start handle the revcover message:"},{"line_number":252,"context_line":"        pull data from DF DB."},{"line_number":253,"context_line":"        pull data from Neutron DB."},{"line_number":254,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a18252c_badc60e5","line":251,"range":{"start_line":251,"start_character":21,"end_line":251,"end_character":29},"updated":"2016-04-13 09:13:25.000000000","message":"--\u003e recover\n\nCan you describe the details about this recover message?\nThe format, payload, and how you send/receive the message?","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"7ffdbefad41164a68bc7734a972d8984f6c7455a","unresolved":false,"context_lines":[{"line_number":248,"context_line":"the status for the object by the latest status and then wait for next db"},{"line_number":249,"context_line":"comparison."},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"    Start handle the revcover message:"},{"line_number":252,"context_line":"        pull data from DF DB."},{"line_number":253,"context_line":"        pull data from Neutron DB."},{"line_number":254,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":5,"id":"1a122d0e_9e740b9d","line":251,"range":{"start_line":251,"start_character":21,"end_line":251,"end_character":29},"in_reply_to":"5a18252c_badc60e5","updated":"2016-04-18 01:38:57.000000000","message":"the recover event is sent from db driver, it is just a simple notify message which means the db and pub\\sub system is ready.","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"a32d6e3cb019fbcff4ae87e1c1d068fe2b0b50f1","unresolved":false,"context_lines":[{"line_number":267,"context_line":""},{"line_number":268,"context_line":"* Optionally, we could start a green thread to do the data comparison between"},{"line_number":269,"context_line":"Neutron DB and DF DB periodically in Neutron plugin, if we found the"},{"line_number":270,"context_line":"performance bottleneck for Neutron plugin, we could consider the 3rd-part"},{"line_number":271,"context_line":"software (such as an additional process or system OM tools) to do this."},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"Neutron Plugin election"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a18252c_1c2c7401","line":270,"range":{"start_line":270,"start_character":69,"end_line":270,"end_character":73},"updated":"2016-04-13 09:13:25.000000000","message":"party","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"a32d6e3cb019fbcff4ae87e1c1d068fe2b0b50f1","unresolved":false,"context_lines":[{"line_number":292,"context_line":""},{"line_number":293,"context_line":"* Each Neutron plugin will detect the current master by read and check the"},{"line_number":294,"context_line":"data record in DF DB which describe the info of master plugin periodically,"},{"line_number":295,"context_line":"and when it find the master is itself, it will handle the recover message"},{"line_number":296,"context_line":"as discussed above, else it will do nothing for the message."},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a18252c_f091d99a","line":295,"range":{"start_line":295,"start_character":47,"end_line":295,"end_character":73},"updated":"2016-04-13 09:13:25.000000000","message":"Only one neutron plugin can handle the recover message. If you do the object-based synchronization, each neutron plugin can sync different objects. We can do a distributed hash to assign different object to different neutron plugin (running on different hosts). Is it more efficient?","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"7ffdbefad41164a68bc7734a972d8984f6c7455a","unresolved":false,"context_lines":[{"line_number":292,"context_line":""},{"line_number":293,"context_line":"* Each Neutron plugin will detect the current master by read and check the"},{"line_number":294,"context_line":"data record in DF DB which describe the info of master plugin periodically,"},{"line_number":295,"context_line":"and when it find the master is itself, it will handle the recover message"},{"line_number":296,"context_line":"as discussed above, else it will do nothing for the message."},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"}],"source_content_type":"text/x-rst","patch_set":5,"id":"1a122d0e_def32309","line":295,"range":{"start_line":295,"start_character":47,"end_line":295,"end_character":73},"in_reply_to":"5a18252c_f091d99a","updated":"2016-04-18 01:38:57.000000000","message":"yes, good ideal, I\u0027ll consider it:)","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"a32d6e3cb019fbcff4ae87e1c1d068fe2b0b50f1","unresolved":false,"context_lines":[{"line_number":304,"context_line":"--------------------------"},{"line_number":305,"context_line":""},{"line_number":306,"context_line":"* When initialize or restart the local controller, ovsdb monitor module will"},{"line_number":307,"context_line":"notify all the exist local VM ports, and local controller will fetch the"},{"line_number":308,"context_line":"corresponding data according to the tenants that these VMs belong to from"},{"line_number":309,"context_line":"DF DB."},{"line_number":310,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a18252c_303871a0","line":307,"range":{"start_line":307,"start_character":15,"end_line":307,"end_character":20},"updated":"2016-04-13 09:13:25.000000000","message":"existing","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"a32d6e3cb019fbcff4ae87e1c1d068fe2b0b50f1","unresolved":false,"context_lines":[{"line_number":318,"context_line":"        publish local chassis data."},{"line_number":319,"context_line":"        get tenant list according to local VM ports."},{"line_number":320,"context_line":"        pull data from DF DB according to tenant list."},{"line_number":321,"context_line":"        compare data between local cache and the got data."},{"line_number":322,"context_line":"        if found object create\\update\\delete :"},{"line_number":323,"context_line":"            notify to local apps"},{"line_number":324,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"5a18252c_d0c01d2d","line":321,"range":{"start_line":321,"start_character":8,"end_line":321,"end_character":58},"updated":"2016-04-13 09:13:25.000000000","message":"when you compare the data between local cache and DF DB, do you need to fence the local controller? What if the data is being changed in DF DB at the mean time?","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"7ffdbefad41164a68bc7734a972d8984f6c7455a","unresolved":false,"context_lines":[{"line_number":318,"context_line":"        publish local chassis data."},{"line_number":319,"context_line":"        get tenant list according to local VM ports."},{"line_number":320,"context_line":"        pull data from DF DB according to tenant list."},{"line_number":321,"context_line":"        compare data between local cache and the got data."},{"line_number":322,"context_line":"        if found object create\\update\\delete :"},{"line_number":323,"context_line":"            notify to local apps"},{"line_number":324,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"1a122d0e_5e56f3d2","line":321,"range":{"start_line":321,"start_character":8,"end_line":321,"end_character":58},"in_reply_to":"5a18252c_d0c01d2d","updated":"2016-04-18 01:38:57.000000000","message":"we do not need to block the local controller, the mechanism is similar with Neutron DB and DF DB which need the verification mechanism described above, the changed data in DF DB will be handled in the next db comparison","commit_id":"d93bfb6c788d4d8cdfe0f1b34712b04b19458c67"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"7e5779253e069d65761e2ae42642d27c7a10a72f","unresolved":false,"context_lines":[{"line_number":187,"context_line":"We discussed the Data Synchronization Mechanism from two aspects:"},{"line_number":188,"context_line":""},{"line_number":189,"context_line":"    1. Neutron plugin;"},{"line_number":190,"context_line":"    2. Dragonflow local controller."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"Neutron Plugin Data Sync"},{"line_number":193,"context_line":"------------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_5913ba0d","line":190,"range":{"start_line":190,"start_character":7,"end_line":190,"end_character":35},"updated":"2016-04-22 08:30:28.000000000","message":"I suggest you add a user-defined system signal to reload all the concerned data from df db to local controller. It will greatly help maintain the datapath. If something unexpected happened, and it is not easy to identify which data is missing, just reload everything without restarting the local controller.\n\nThis is what I\u0027m thinking, but it can be considered as a plus. We can add it in the later phase.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"46508dbb0ab9ecf10255e2fb399e22c08b559ad8","unresolved":false,"context_lines":[{"line_number":187,"context_line":"We discussed the Data Synchronization Mechanism from two aspects:"},{"line_number":188,"context_line":""},{"line_number":189,"context_line":"    1. Neutron plugin;"},{"line_number":190,"context_line":"    2. Dragonflow local controller."},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"Neutron Plugin Data Sync"},{"line_number":193,"context_line":"------------------------"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_410c3c42","line":190,"range":{"start_line":190,"start_character":7,"end_line":190,"end_character":35},"in_reply_to":"1a122d0e_5913ba0d","updated":"2016-04-23 03:37:50.000000000","message":"I have discuss this below, the user-defined system signal you mentioned may be the discover message in my description which could be used by both Neutron plugin and local controller","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"7e5779253e069d65761e2ae42642d27c7a10a72f","unresolved":false,"context_lines":[{"line_number":197,"context_line":""},{"line_number":198,"context_line":"    Start Neutron DB transaction for creation operations."},{"line_number":199,"context_line":"        Do Neutron DB operations."},{"line_number":200,"context_line":"        try:"},{"line_number":201,"context_line":"            Do DF DB operations."},{"line_number":202,"context_line":"            Emit messages via PUB/SUB."},{"line_number":203,"context_line":"        except:"},{"line_number":204,"context_line":"            retry to do DF DB operations and PUB/SUB."},{"line_number":205,"context_line":"            if beyond max retry times:"},{"line_number":206,"context_line":"                rollback Neutron DB operations."},{"line_number":207,"context_line":"                raise creation exception."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":210,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_19bae2c0","line":207,"range":{"start_line":200,"start_character":8,"end_line":207,"end_character":41},"updated":"2016-04-22 08:30:28.000000000","message":"This will lead the situation that the DF DB operations are committed in advance.\n\nYou have to add another try-catch for Neutron DB error.\n\ntry:\n    Start Neutron DB transaction for creation operations.\n        Do Neutron DB operations.\n        try:\n            Do DF DB operations.\n            Emit messages via PUB/SUB.\n        except:\n            retry to do DF DB operations and PUB/SUB.\n            if beyond max retry times:\n                rollback Neutron DB operations.\n                raise creation exception.\nexcept SQLAlchemyError:\n    Do reverse operations on DF DB.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"46508dbb0ab9ecf10255e2fb399e22c08b559ad8","unresolved":false,"context_lines":[{"line_number":197,"context_line":""},{"line_number":198,"context_line":"    Start Neutron DB transaction for creation operations."},{"line_number":199,"context_line":"        Do Neutron DB operations."},{"line_number":200,"context_line":"        try:"},{"line_number":201,"context_line":"            Do DF DB operations."},{"line_number":202,"context_line":"            Emit messages via PUB/SUB."},{"line_number":203,"context_line":"        except:"},{"line_number":204,"context_line":"            retry to do DF DB operations and PUB/SUB."},{"line_number":205,"context_line":"            if beyond max retry times:"},{"line_number":206,"context_line":"                rollback Neutron DB operations."},{"line_number":207,"context_line":"                raise creation exception."},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":210,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_7c62bbd9","line":207,"range":{"start_line":200,"start_character":8,"end_line":207,"end_character":41},"in_reply_to":"1a122d0e_19bae2c0","updated":"2016-04-23 03:37:50.000000000","message":"What I mean here is using the default process mode similar in ML2 plugin, if there are some exception happen in do Neutron DB operation, the plugin should rollback the operation automatically by transaction mechanism, and should not run the following code anymore, so the df db operation will not be executed, I will add the code here to avoid confuse:)","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"7e5779253e069d65761e2ae42642d27c7a10a72f","unresolved":false,"context_lines":[{"line_number":215,"context_line":"When Neutron plugin receives a update\\delete object(router\\network\\subnet\\port,"},{"line_number":216,"context_line":"etc) invoke:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"    Start Neutron DB transaction for DB operations."},{"line_number":219,"context_line":"        Do Neutron DB operations."},{"line_number":220,"context_line":"        try:"},{"line_number":221,"context_line":"            Do DF DB operations."},{"line_number":222,"context_line":"            Emit messages via PUB/SUB."},{"line_number":223,"context_line":"        except:"},{"line_number":224,"context_line":"            retry to do DF DB operations and PUB/SUB."},{"line_number":225,"context_line":"            if beyond max retry times:"},{"line_number":226,"context_line":"                raise update\\delete exception."},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"* The difference between update\\delete invoke and creation invoke is there is"},{"line_number":229,"context_line":"no need to rollback when beyond max retry times, for instance, it is impossible"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_19e142d1","line":226,"range":{"start_line":218,"start_character":4,"end_line":226,"end_character":46},"updated":"2016-04-22 08:30:28.000000000","message":"ditto.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"7e5779253e069d65761e2ae42642d27c7a10a72f","unresolved":false,"context_lines":[{"line_number":238,"context_line":"    Start handle the recover message:"},{"line_number":239,"context_line":"        pull data from DF DB."},{"line_number":240,"context_line":"        pull data from Neutron DB."},{"line_number":241,"context_line":"        do compare with two data set."},{"line_number":242,"context_line":"        if found object create\\update\\delete :"},{"line_number":243,"context_line":"            do DF DB operations."},{"line_number":244,"context_line":"            Emit messages via PUB/SUB."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_5939fa40","line":241,"range":{"start_line":241,"start_character":8,"end_line":241,"end_character":37},"updated":"2016-04-22 08:30:28.000000000","message":"Please specify how to compare the two data set.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"46508dbb0ab9ecf10255e2fb399e22c08b559ad8","unresolved":false,"context_lines":[{"line_number":238,"context_line":"    Start handle the recover message:"},{"line_number":239,"context_line":"        pull data from DF DB."},{"line_number":240,"context_line":"        pull data from Neutron DB."},{"line_number":241,"context_line":"        do compare with two data set."},{"line_number":242,"context_line":"        if found object create\\update\\delete :"},{"line_number":243,"context_line":"            do DF DB operations."},{"line_number":244,"context_line":"            Emit messages via PUB/SUB."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_5c5d3f14","line":241,"range":{"start_line":241,"start_character":8,"end_line":241,"end_character":37},"in_reply_to":"1a122d0e_5939fa40","updated":"2016-04-23 03:37:50.000000000","message":"ok, I\u0027ll add it","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"7e5779253e069d65761e2ae42642d27c7a10a72f","unresolved":false,"context_lines":[{"line_number":280,"context_line":"            else:"},{"line_number":281,"context_line":"                refresh object status in cache."},{"line_number":282,"context_line":""},{"line_number":283,"context_line":"Neutron Plugin election"},{"line_number":284,"context_line":"-----------------------"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"We could use the distribute lock mechanism discussed above for the election."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_b953ee6b","line":283,"updated":"2016-04-22 08:30:28.000000000","message":"This can be considered as a membership management module. It is very good. We can track the local controller\u0027s runtime status via this table. Not only for election, but also for health check.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"46508dbb0ab9ecf10255e2fb399e22c08b559ad8","unresolved":false,"context_lines":[{"line_number":280,"context_line":"            else:"},{"line_number":281,"context_line":"                refresh object status in cache."},{"line_number":282,"context_line":""},{"line_number":283,"context_line":"Neutron Plugin election"},{"line_number":284,"context_line":"-----------------------"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"We could use the distribute lock mechanism discussed above for the election."}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_7cb91b2e","line":283,"in_reply_to":"1a122d0e_b953ee6b","updated":"2016-04-23 03:37:50.000000000","message":"yes, master and normal plugins should monitor each other by this record in df db for election and load balance","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"3bb2a60d0bc1f3cf504145541daf58f824887e4a","unresolved":false,"context_lines":[{"line_number":328,"context_line":"node bottleneck."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Master Neutron plugin will get tenant list from Neutron DB and DF DB"},{"line_number":331,"context_line":"periodically, however, the tenant list from Neutron DB may not be the same as"},{"line_number":332,"context_line":"the tenant list from DF DB, so master Neutron plugin will combine the two"},{"line_number":333,"context_line":"tenant list into one set. For instance, if tenant list from Neutron DB is"},{"line_number":334,"context_line":"(A, B), tenant list from DF DB is (B, C), after combination the tenant list is"},{"line_number":335,"context_line":"(A, B, C). Master Neutron plugin will assign the tenants to be well-distributed"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_f4ff2da8","line":332,"range":{"start_line":331,"start_character":23,"end_line":332,"end_character":27},"updated":"2016-04-22 08:45:18.000000000","message":"how to get the tenant list? both neutron db and df db do not have such a table to store all the tenants.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"2d8fb17e9f07a8f240c03361ed8aa24e5bd945a6","unresolved":false,"context_lines":[{"line_number":328,"context_line":"node bottleneck."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Master Neutron plugin will get tenant list from Neutron DB and DF DB"},{"line_number":331,"context_line":"periodically, however, the tenant list from Neutron DB may not be the same as"},{"line_number":332,"context_line":"the tenant list from DF DB, so master Neutron plugin will combine the two"},{"line_number":333,"context_line":"tenant list into one set. For instance, if tenant list from Neutron DB is"},{"line_number":334,"context_line":"(A, B), tenant list from DF DB is (B, C), after combination the tenant list is"},{"line_number":335,"context_line":"(A, B, C). Master Neutron plugin will assign the tenants to be well-distributed"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_499b9566","line":332,"range":{"start_line":331,"start_character":23,"end_line":332,"end_character":27},"in_reply_to":"1a122d0e_046349dd","updated":"2016-04-26 06:46:23.000000000","message":"If the tenant has no resource, when Neutron plugin try to get the data by the tenant, it will get nothing, so it doesn\u0027t matter when do db comparison","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"f00a8d7d40e4a9abf3c64a5bb363df7f393b353d","unresolved":false,"context_lines":[{"line_number":328,"context_line":"node bottleneck."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Master Neutron plugin will get tenant list from Neutron DB and DF DB"},{"line_number":331,"context_line":"periodically, however, the tenant list from Neutron DB may not be the same as"},{"line_number":332,"context_line":"the tenant list from DF DB, so master Neutron plugin will combine the two"},{"line_number":333,"context_line":"tenant list into one set. For instance, if tenant list from Neutron DB is"},{"line_number":334,"context_line":"(A, B), tenant list from DF DB is (B, C), after combination the tenant list is"},{"line_number":335,"context_line":"(A, B, C). Master Neutron plugin will assign the tenants to be well-distributed"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_5cb81ac8","line":332,"range":{"start_line":331,"start_character":23,"end_line":332,"end_character":27},"in_reply_to":"1a122d0e_499b9566","updated":"2016-04-28 19:21:01.000000000","message":"Currently df db doesn\u0027t have the index of tenant-id. I suggest we do it in the neutron, but not keystone. You can use neutron api to get all the objects.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"a5fc22d9c0d975e4757767ede3ef746396cb5d69","unresolved":false,"context_lines":[{"line_number":328,"context_line":"node bottleneck."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Master Neutron plugin will get tenant list from Neutron DB and DF DB"},{"line_number":331,"context_line":"periodically, however, the tenant list from Neutron DB may not be the same as"},{"line_number":332,"context_line":"the tenant list from DF DB, so master Neutron plugin will combine the two"},{"line_number":333,"context_line":"tenant list into one set. For instance, if tenant list from Neutron DB is"},{"line_number":334,"context_line":"(A, B), tenant list from DF DB is (B, C), after combination the tenant list is"},{"line_number":335,"context_line":"(A, B, C). Master Neutron plugin will assign the tenants to be well-distributed"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_f5993420","line":332,"range":{"start_line":331,"start_character":23,"end_line":332,"end_character":27},"in_reply_to":"1a122d0e_5cb81ac8","updated":"2016-04-29 02:49:41.000000000","message":"yes, I agree use keystone here is not suitable, my previous reply means we do not need to determine which tenant has the network resource if we use keystone, I have modified here to use static hash LB mechanism in the new patch:)","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"c4f64d3d0c642d6abdae7db1d302caff251b03e1","unresolved":false,"context_lines":[{"line_number":328,"context_line":"node bottleneck."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Master Neutron plugin will get tenant list from Neutron DB and DF DB"},{"line_number":331,"context_line":"periodically, however, the tenant list from Neutron DB may not be the same as"},{"line_number":332,"context_line":"the tenant list from DF DB, so master Neutron plugin will combine the two"},{"line_number":333,"context_line":"tenant list into one set. For instance, if tenant list from Neutron DB is"},{"line_number":334,"context_line":"(A, B), tenant list from DF DB is (B, C), after combination the tenant list is"},{"line_number":335,"context_line":"(A, B, C). Master Neutron plugin will assign the tenants to be well-distributed"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_046349dd","line":332,"range":{"start_line":331,"start_character":23,"end_line":332,"end_character":27},"in_reply_to":"1a122d0e_dcb66ff8","updated":"2016-04-25 19:27:24.000000000","message":"Using keystone-api to get tenant-id list is not suitable here. you need to determine which tenant has the network resource.","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"46508dbb0ab9ecf10255e2fb399e22c08b559ad8","unresolved":false,"context_lines":[{"line_number":328,"context_line":"node bottleneck."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Master Neutron plugin will get tenant list from Neutron DB and DF DB"},{"line_number":331,"context_line":"periodically, however, the tenant list from Neutron DB may not be the same as"},{"line_number":332,"context_line":"the tenant list from DF DB, so master Neutron plugin will combine the two"},{"line_number":333,"context_line":"tenant list into one set. For instance, if tenant list from Neutron DB is"},{"line_number":334,"context_line":"(A, B), tenant list from DF DB is (B, C), after combination the tenant list is"},{"line_number":335,"context_line":"(A, B, C). Master Neutron plugin will assign the tenants to be well-distributed"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_dcb66ff8","line":332,"range":{"start_line":331,"start_character":23,"end_line":332,"end_character":27},"in_reply_to":"1a122d0e_f4ff2da8","updated":"2016-04-23 03:37:50.000000000","message":"Master plugin should collect the tenant info from openstack by keystone api, and we could store the tenant info in DF db so that the master plugin could fetch them, but in my current thinking, I think use tenant to do LB is not best method, the amount of tenant info maybe large and the performance of keystone api is a problem, so we can consider the hash for object id as you suggest:)","commit_id":"cfa4ac7c8905facf0d42ae14cf73205b10b8580f"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":22,"context_line":"Here we will propose a simple and feasible distributed lock solution and"},{"line_number":23,"context_line":"improve it later according to evaluation in production. We also provide an"},{"line_number":24,"context_line":"effective data synchronization mechanism to resolve the data inconsistency"},{"line_number":25,"context_line":"problem which may happen in the communication among above three type of"},{"line_number":26,"context_line":"data stores."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_a675abfb","line":25,"range":{"start_line":25,"start_character":64,"end_line":25,"end_character":68},"updated":"2016-05-01 09:15:18.000000000","message":"type -\u003e types","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":29,"context_line":"Problem Description"},{"line_number":30,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"Currently, Dragonflow architecture has three type of data stores. One is"},{"line_number":33,"context_line":"Neutron DB responsible for storing virtualized network topology. This database"},{"line_number":34,"context_line":"is relational and all the operations on it are based upon atomic transactions."},{"line_number":35,"context_line":"It is considered as the master DB of Dragonflow. The other is distributed NoSQL"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_c662673d","line":32,"range":{"start_line":32,"start_character":45,"end_line":32,"end_character":49},"updated":"2016-05-01 09:15:18.000000000","message":"type -\u003e types","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":46,"context_line":""},{"line_number":47,"context_line":"* Scenario 1, in Bug [1]:"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"    with transaction of Neutron DB:"},{"line_number":50,"context_line":"        call Neutron DB API"},{"line_number":51,"context_line":"    call Dragonflow DB API"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"When an exception is thrown from Dragonflow DB API, however, Neutron DB"},{"line_number":54,"context_line":"transaction is committed. We need to call re-sync to make sure the two DB is"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_c6c9c727","line":51,"range":{"start_line":49,"start_character":0,"end_line":51,"end_character":26},"updated":"2016-05-01 09:15:18.000000000","message":"The restview syntax here is illegal. Consider adding:\n\n::\n\nbefore the stanza.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":46,"context_line":""},{"line_number":47,"context_line":"* Scenario 1, in Bug [1]:"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"    with transaction of Neutron DB:"},{"line_number":50,"context_line":"        call Neutron DB API"},{"line_number":51,"context_line":"    call Dragonflow DB API"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"When an exception is thrown from Dragonflow DB API, however, Neutron DB"},{"line_number":54,"context_line":"transaction is committed. We need to call re-sync to make sure the two DB is"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_ca240a10","line":51,"range":{"start_line":49,"start_character":0,"end_line":51,"end_character":26},"in_reply_to":"1a122d0e_c6c9c727","updated":"2016-05-03 06:49:53.000000000","message":"here you mean use :: to replace : ?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":56,"context_line":"disconnected after Neutron DB transaction is committed."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"* Scenario 2, in Bug [2]:"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"    with transaction of Neutron DB:"},{"line_number":61,"context_line":"        call Neutron DB API"},{"line_number":62,"context_line":"    call _some_operation(...)"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_46d657c4","line":59,"updated":"2016-05-01 09:15:18.000000000","message":"Same here.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"After Neutron DB is committed, concurrent calling of _some_operation function"},{"line_number":70,"context_line":"will still cause inconsistency, because all the operations in that function are"},{"line_number":71,"context_line":"not atomic. This phenomenon is discovered in multi-node deployments."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* Scenario 3:"},{"line_number":74,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_01e0c9f9","line":71,"updated":"2016-05-01 09:15:18.000000000","message":"I think I am missing something - looking at dragonflow.neutron.plugin.DFPlugin, it looks like all API calls are wrapped with lock_db.wrap_db_lock(). Doesn\u0027t that mean there is a lock on the call to the method?\n\nI think this issue was resolved by: https://review.openstack.org/#/c/282290/","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":68,"context_line":""},{"line_number":69,"context_line":"After Neutron DB is committed, concurrent calling of _some_operation function"},{"line_number":70,"context_line":"will still cause inconsistency, because all the operations in that function are"},{"line_number":71,"context_line":"not atomic. This phenomenon is discovered in multi-node deployments."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* Scenario 3:"},{"line_number":74,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_6bf4f7ce","line":71,"in_reply_to":"1a122d0e_01e0c9f9","updated":"2016-05-01 19:07:27.000000000","message":"Sorry, please ignore. Meant to remove this comment before posting.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":81,"context_line":"breakdown of network or sub/pub system, so local controller will lose the new"},{"line_number":82,"context_line":"data. Another case is after local controller receives the pub message, it"},{"line_number":83,"context_line":"happens internal exception in the data process, then the new data will be"},{"line_number":84,"context_line":"dropped."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"* Scenario 4:"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_a10115b2","line":84,"updated":"2016-05-01 09:15:18.000000000","message":"I am not sure I understand this paragraph. Is this what you meant?:\n\nAfter Dragonflow local controller receives its concerned data, the publish/subscribe connection between the local controller and the publisher fails, possibly due to network or publish/subscribe system failure. In this case, the local controller will lose the new data.\n\nAnother case is after the local controller receives the published message, there is an internal exception in the data process. In this case, the new data will be lost.\n\n----\nIf this is the case, why would the new data be lost if the failure happens *after* it is received?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":81,"context_line":"breakdown of network or sub/pub system, so local controller will lose the new"},{"line_number":82,"context_line":"data. Another case is after local controller receives the pub message, it"},{"line_number":83,"context_line":"happens internal exception in the data process, then the new data will be"},{"line_number":84,"context_line":"dropped."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"* Scenario 4:"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_4ac59a08","line":84,"in_reply_to":"1a122d0e_a10115b2","updated":"2016-05-03 06:49:53.000000000","message":"First case means before local controller receive its concerned data it happens some exception, so the new data will be lost, the second case is like what you said, if the local controller raise some exception during handling the new data before store it into local cache, the new data maybe lost","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":86,"context_line":"* Scenario 4:"},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"    1. A VM online on a Dragonflow local controller host while it is the first"},{"line_number":89,"context_line":"    VM of one tenant on the host;"},{"line_number":90,"context_line":"    2. The local controller will fetch all the data belong to the tenant from"},{"line_number":91,"context_line":"    Dragonflow NoSQL DB."},{"line_number":92,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_c107f1c7","line":89,"updated":"2016-05-01 09:15:18.000000000","message":"Add newline","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":93,"context_line":"After the VM online, the data read/write connection between local controller"},{"line_number":94,"context_line":"and Dragonflow NoSQL DB is down because of the breakdown of network or the"},{"line_number":95,"context_line":"problem of DB itself(restart or node crash), then the local controller will"},{"line_number":96,"context_line":"lose the tenant data it concerned."},{"line_number":97,"context_line":""},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_81fe99ac","line":96,"updated":"2016-05-01 09:15:18.000000000","message":"Won\u0027t the controller try to re-sync the data in this case, until it succeeds (when the connection is restored?)","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":93,"context_line":"After the VM online, the data read/write connection between local controller"},{"line_number":94,"context_line":"and Dragonflow NoSQL DB is down because of the breakdown of network or the"},{"line_number":95,"context_line":"problem of DB itself(restart or node crash), then the local controller will"},{"line_number":96,"context_line":"lose the tenant data it concerned."},{"line_number":97,"context_line":""},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"Proposed Change"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_2a665e29","line":96,"in_reply_to":"1a122d0e_81fe99ac","updated":"2016-05-03 06:49:53.000000000","message":"When the connection is restored, the db driver will notify a recover message as described below and controller will handle it, here we just mention the possibility that happens data inconsistency","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":117,"context_line":""},{"line_number":118,"context_line":"The distributed lock is tenant-based and each tenant has its own lock in the"},{"line_number":119,"context_line":"database. Due to this design, we can allow concurrency to a certain degree."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"In Neutron plugin:"},{"line_number":122,"context_line":"    When an API is processing:"},{"line_number":123,"context_line":"        Acquire the distributed lock for the Neutron object."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_61cc1d8b","line":120,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: ::","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":126,"context_line":"            Do DF DB operations."},{"line_number":127,"context_line":"            Emit messages via PUB/SUB."},{"line_number":128,"context_line":"        Release the distributed lock."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"* When creating the distributed lock record, it starts a DB transaction and"},{"line_number":131,"context_line":"inserts a lock record into Neutron DB according to the current object."},{"line_number":132,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_41c92179","line":129,"updated":"2016-05-01 09:15:18.000000000","message":"Consider indenting the wrapped lines of the bullets, to fix restview syntax.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":126,"context_line":"            Do DF DB operations."},{"line_number":127,"context_line":"            Emit messages via PUB/SUB."},{"line_number":128,"context_line":"        Release the distributed lock."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"* When creating the distributed lock record, it starts a DB transaction and"},{"line_number":131,"context_line":"inserts a lock record into Neutron DB according to the current object."},{"line_number":132,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_aa59ce5d","line":129,"in_reply_to":"1a122d0e_41c92179","updated":"2016-05-03 06:49:53.000000000","message":"Sorry, would you please give me a simple example, how to implement the indenting as you suggest, thx:)","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20336,"name":"heshan","email":"shan.he@huawei.com","username":"shanhe"},"change_message_id":"194a648ed52ab9f0959ef642da0ae2cf54ade8f7","unresolved":false,"context_lines":[{"line_number":141,"context_line":"committed later."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"* Potential Issue: When concurrent write operations on a certain key happen,"},{"line_number":144,"context_line":"due to the inconsistency window of DF DB. If the update on DF DB is always"},{"line_number":145,"context_line":"delayed because the previous operations have already been delayed."},{"line_number":146,"context_line":"The root cause is that Neutron DB is strongly consistent but DF DB is"},{"line_number":147,"context_line":"eventually consistent. We cannot guarantee the updates on DF DB is committed."},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_e868b623","line":145,"range":{"start_line":144,"start_character":42,"end_line":145,"end_character":66},"updated":"2016-04-25 03:47:04.000000000","message":"I\u0027m not sure I understand this part","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"2b459f24fcff4fa4c29480e6a545bbdf5fd5772b","unresolved":false,"context_lines":[{"line_number":141,"context_line":"committed later."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"* Potential Issue: When concurrent write operations on a certain key happen,"},{"line_number":144,"context_line":"due to the inconsistency window of DF DB. If the update on DF DB is always"},{"line_number":145,"context_line":"delayed because the previous operations have already been delayed."},{"line_number":146,"context_line":"The root cause is that Neutron DB is strongly consistent but DF DB is"},{"line_number":147,"context_line":"eventually consistent. We cannot guarantee the updates on DF DB is committed."},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_5ed6c027","line":145,"range":{"start_line":144,"start_character":42,"end_line":145,"end_character":66},"in_reply_to":"1a122d0e_e868b623","updated":"2016-04-25 06:59:16.000000000","message":"because DF db is eventually consistent, it need time to sync the data to other nodes in the cluster, so if use the sync interface, the operation will be delayed util eventually consistent","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Pseudo Code in Core Plugin"},{"line_number":150,"context_line":"--------------------------"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"    def CUD_object(context, obj):"},{"line_number":153,"context_line":"        nb_lock \u003d lock_db.DBLock(context.tenant_id)"},{"line_number":154,"context_line":"        with nb_lock:"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_01ae0998","line":151,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: ::","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":162,"context_line":"* CUD means create, update or delete."},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"* This can be simplified by a decorator:"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":"    @lock_db.wrap_db_lock()"},{"line_number":167,"context_line":"    def CUD_object(self, context, obj):"},{"line_number":168,"context_line":"        pass"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_61937d5d","line":165,"updated":"2016-05-01 09:15:18.000000000","message":"And here","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":191,"context_line":""},{"line_number":192,"context_line":"Neutron Plugin Data Sync"},{"line_number":193,"context_line":"------------------------"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"When Neutron plugin receives a creation object(router\\network\\subnet\\port, etc)"},{"line_number":196,"context_line":"invoke:"},{"line_number":197,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_21aea581","line":194,"updated":"2016-05-01 09:15:18.000000000","message":"Consider replacing all \\ with /. I think \u0027\\\u0027 is a special character in restview.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":191,"context_line":""},{"line_number":192,"context_line":"Neutron Plugin Data Sync"},{"line_number":193,"context_line":"------------------------"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"When Neutron plugin receives a creation object(router\\network\\subnet\\port, etc)"},{"line_number":196,"context_line":"invoke:"},{"line_number":197,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_4a2f5aa7","line":194,"in_reply_to":"1a122d0e_21aea581","updated":"2016-05-03 06:49:53.000000000","message":"ok","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":194,"context_line":""},{"line_number":195,"context_line":"When Neutron plugin receives a creation object(router\\network\\subnet\\port, etc)"},{"line_number":196,"context_line":"invoke:"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"    Start Neutron DB transaction for creation operations."},{"line_number":199,"context_line":"        with session.begin(subtransactions\u003dTrue):"},{"line_number":200,"context_line":"            Do Neutron DB operations."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_a19d9568","line":197,"updated":"2016-05-01 09:15:18.000000000","message":"And here (re: ::)","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":204,"context_line":"        except:"},{"line_number":205,"context_line":"            retry to do DF DB operations and PUB/SUB."},{"line_number":206,"context_line":"            if beyond max retry times:"},{"line_number":207,"context_line":"                rollback Neutron DB operations."},{"line_number":208,"context_line":"                raise creation exception."},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_819a196d","line":207,"updated":"2016-05-01 09:15:18.000000000","message":"Wouldn\u0027t it be better to re-sync the databases? Neutron DB is the master. We don\u0027t want to rollback a change just because the slave couldn\u0027t commit it.\n\nIf Neutron allowed the change, Dragonflow must accept it one way or another.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":204,"context_line":"        except:"},{"line_number":205,"context_line":"            retry to do DF DB operations and PUB/SUB."},{"line_number":206,"context_line":"            if beyond max retry times:"},{"line_number":207,"context_line":"                rollback Neutron DB operations."},{"line_number":208,"context_line":"                raise creation exception."},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_c5ad1dec","line":207,"in_reply_to":"1a122d0e_819a196d","updated":"2016-05-03 06:49:53.000000000","message":"1. As an important component in the whole dragonflow system, if the DF DB could not work, it\u0027s unnecessary to allow the tenant use Neutron to create a vm(for example) which could not work as well;\n2. It will add the workload of Neutron plugin/driver to support re-sync for every exception event;\n3. In current implement of ML2 in Openstack, if it happens exception in postcommit method, the previously created resource will be deleted, so we could be compatible with ML2\n4. The rollback here is meaning to delete the previously created resource in another transaction","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":206,"context_line":"            if beyond max retry times:"},{"line_number":207,"context_line":"                rollback Neutron DB operations."},{"line_number":208,"context_line":"                raise creation exception."},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":211,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"},{"line_number":212,"context_line":"process, Neutron plugin should retry several times to finish commits and"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_e185cd0a","line":209,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: indentation","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":209,"context_line":""},{"line_number":210,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":211,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"},{"line_number":212,"context_line":"process, Neutron plugin should retry several times to finish commits and"},{"line_number":213,"context_line":"PUB/SUB, if it failed after all the attempts, Neutron plugin should rollback"},{"line_number":214,"context_line":"the previous commit, and raise a creation exception."},{"line_number":215,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_4177613b","line":212,"range":{"start_line":212,"start_character":9,"end_line":212,"end_character":23},"updated":"2016-05-01 09:15:18.000000000","message":"Is Neutron plugin responsible for Dragonflow, or Neutron db? Shouldn\u0027t Dragonflow retry to finish the commit (Didn\u0027t Neutron already commit to the Neutron DB)?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":209,"context_line":""},{"line_number":210,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":211,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"},{"line_number":212,"context_line":"process, Neutron plugin should retry several times to finish commits and"},{"line_number":213,"context_line":"PUB/SUB, if it failed after all the attempts, Neutron plugin should rollback"},{"line_number":214,"context_line":"the previous commit, and raise a creation exception."},{"line_number":215,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_e5e2d92e","line":212,"range":{"start_line":212,"start_character":9,"end_line":212,"end_character":23},"in_reply_to":"1a122d0e_4177613b","updated":"2016-05-03 06:49:53.000000000","message":"Here means Neutron plugin responsible for DF DB or pub/sub system, indeed, this should be discussed in DB driver spec, Neutron plugin just see the exception:)","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":215,"context_line":""},{"line_number":216,"context_line":"When Neutron plugin receives a update\\delete object(router\\network\\subnet\\port,"},{"line_number":217,"context_line":"etc) invoke:"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"    Start Neutron DB transaction for DB operations."},{"line_number":220,"context_line":"        with session.begin(subtransactions\u003dTrue):"},{"line_number":221,"context_line":"            Do Neutron DB operations."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_c18a511a","line":218,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: ::","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":225,"context_line":"        except:"},{"line_number":226,"context_line":"            retry to do DF DB operations and PUB/SUB."},{"line_number":227,"context_line":"            if beyond max retry times:"},{"line_number":228,"context_line":"                raise update\\delete exception."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"* The difference between update\\delete invoke and creation invoke is there is"},{"line_number":231,"context_line":"no need to rollback when beyond max retry times, for instance, it is impossible"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_a1c2b532","line":228,"updated":"2016-05-01 09:15:18.000000000","message":"Shouldn\u0027t a re-sync be attempted as well?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":226,"context_line":"            retry to do DF DB operations and PUB/SUB."},{"line_number":227,"context_line":"            if beyond max retry times:"},{"line_number":228,"context_line":"                raise update\\delete exception."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"* The difference between update\\delete invoke and creation invoke is there is"},{"line_number":231,"context_line":"no need to rollback when beyond max retry times, for instance, it is impossible"},{"line_number":232,"context_line":"and unnecessary to rollback all the Neutron DB data for a deleted VM, and we"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_01ea89ac","line":229,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: indentation","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20336,"name":"heshan","email":"shan.he@huawei.com","username":"shanhe"},"change_message_id":"194a648ed52ab9f0959ef642da0ae2cf54ade8f7","unresolved":false,"context_lines":[{"line_number":235,"context_line":"When DB driver and pub\\sub driver find the read\\write connection between"},{"line_number":236,"context_line":"Neutron plugin and DF DB, and also the pub\\sub connection between Neutron"},{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_08dc12e3","line":238,"range":{"start_line":238,"start_character":26,"end_line":238,"end_character":68},"updated":"2016-04-25 03:47:04.000000000","message":"how to notify plugin?\nI don\u0027t think plugin could do this work, especially ml2 plugin","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"2b459f24fcff4fa4c29480e6a545bbdf5fd5772b","unresolved":false,"context_lines":[{"line_number":235,"context_line":"When DB driver and pub\\sub driver find the read\\write connection between"},{"line_number":236,"context_line":"Neutron plugin and DF DB, and also the pub\\sub connection between Neutron"},{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_3e1b041d","line":238,"range":{"start_line":238,"start_character":26,"end_line":238,"end_character":68},"in_reply_to":"1a122d0e_08dc12e3","updated":"2016-04-25 06:59:16.000000000","message":"df driver will notify neutron plugin, as to ML2 plugin, df driver will notify the message to our own mechanism driver","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"01a04911d1b597195772b9cd4b87423a86961721","unresolved":false,"context_lines":[{"line_number":235,"context_line":"When DB driver and pub\\sub driver find the read\\write connection between"},{"line_number":236,"context_line":"Neutron plugin and DF DB, and also the pub\\sub connection between Neutron"},{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_647f9d66","line":238,"range":{"start_line":238,"start_character":26,"end_line":238,"end_character":68},"in_reply_to":"1a122d0e_3e1b041d","updated":"2016-04-25 19:31:36.000000000","message":"what does \u0027df driver\u0027 mean here? and how to send the recover message? you should elaborate it here to prevent from misunderstanding.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"20e239c5f90a2d20a66e39ca5af892011ce677c8","unresolved":false,"context_lines":[{"line_number":235,"context_line":"When DB driver and pub\\sub driver find the read\\write connection between"},{"line_number":236,"context_line":"Neutron plugin and DF DB, and also the pub\\sub connection between Neutron"},{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_12054210","line":238,"range":{"start_line":238,"start_character":26,"end_line":238,"end_character":68},"in_reply_to":"1a122d0e_647f9d66","updated":"2016-04-26 06:45:26.000000000","message":"df driver means dragonflow DB driver which could communicate with both Neutron plugin and local controller, dragonflow core plugin or our mechanism driver for ML2 will register callback to df driver for recover message hander, if df driver detect that it meet the requirement for sending a recover message, it will invoke the callback to core plugin or our mechanism driver","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":236,"context_line":"Neutron plugin and DF DB, and also the pub\\sub connection between Neutron"},{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_e1014d6f","line":239,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: ::","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."},{"line_number":243,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_21fce571","line":240,"updated":"2016-05-01 09:15:18.000000000","message":"This is a resync operation, right?. Shouldn\u0027t all resync operations behave in this manner?\n\nAdditionally, can this operation be improved by adding a version to each object?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"e1e0999029da4b640933f60615fb9019bbb1c849","unresolved":false,"context_lines":[{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."},{"line_number":243,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_a07eb318","line":240,"in_reply_to":"1a122d0e_1f5e0f45","updated":"2016-05-05 01:23:54.000000000","message":"When updating the version, we need to update it atomically to prevent from race. NoSQL backends support compare-and-swap a given blob, like etcd. Zookeeper can also do the same thing. However if we add \u0027version\u0027 to the blob, we cannot do atomic update, because updating the whole blob with object and version is not feasible. As a result, we add a namespace for that purpose.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."},{"line_number":243,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_655b69f7","line":240,"in_reply_to":"1a122d0e_21fce571","updated":"2016-05-03 06:49:53.000000000","message":"Yes, the DF db driver should support this and I have discuss with zhuweiqiong who design the DB driver, a recover message will resolve the inconsistency for many resource objects during the exception period, and it\u0027s best to add a version in each object so that we can make our comparison more effective, but I don\u0027t think it is supported in Neutron DB:(","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"b1696cc644c79c42e97b3ca692982bb796431bba","unresolved":false,"context_lines":[{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."},{"line_number":243,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_db5e6c78","line":240,"in_reply_to":"1a122d0e_655b69f7","updated":"2016-05-04 09:23:15.000000000","message":"We discussed it during summit. The versioning per object is necessary because it is also needed in pub/sub system.\n\nWe can add a db table in neutron for versioning, with object id and its version integer. Everytime we create an object(network, subnet, port, router, secgroup...), we create a record in this table.\n\nThen we add a version namespace in df db for versioning, \\versions\\secgroup\\id: version, which representing \\secgroup\\id: blob value.\n\nThen we add a function cas_key(table, key, old_val, new_val) in df db interface to update the version atomically. (cas means compare_and_swap).\n\nThis is a basic idea behind the object versioning. If you have any question, we can discuss with it offline, via phone or email or meeting.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"2927bec2b1b1532232c0224d2a2b92bafaf58589","unresolved":false,"context_lines":[{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."},{"line_number":243,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_c0c76f43","line":240,"in_reply_to":"1a122d0e_a07eb318","updated":"2016-05-05 01:26:51.000000000","message":"By the way, for object comparison, we can only directly read the version and compare it. If we save version into blob, we have to read the whole object and extract version from it.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"d4914632e141a8b8cce625e0c9f6c745c39fb148","unresolved":false,"context_lines":[{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."},{"line_number":243,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_1f5e0f45","line":240,"in_reply_to":"1a122d0e_db5e6c78","updated":"2016-05-04 09:46:57.000000000","message":"I do not understand why we should maintain a namespace as you mentioned to store the version info, I think we just add a version field into each object(lswitch/subnet/lport/lrouter), it\u0027s more simple","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"0691faaf57e85fe356a72ecf2f6175119099c5a5","unresolved":false,"context_lines":[{"line_number":237,"context_line":"plugin and pub\\sub system are recovered, the driver should notify Neutron"},{"line_number":238,"context_line":"plugin a recover message, Neutron plugin should process the message:"},{"line_number":239,"context_line":""},{"line_number":240,"context_line":"    Start handle the recover message:"},{"line_number":241,"context_line":"        pull data from DF DB."},{"line_number":242,"context_line":"        pull data from Neutron DB."},{"line_number":243,"context_line":"        do compare with two data set."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_1fca8f4f","line":240,"in_reply_to":"1a122d0e_db5e6c78","updated":"2016-05-04 09:39:27.000000000","message":"yes, that\u0027s great, I have mentioned that we need the version for each object in the patch 11, and if we could add a table into Neutron DB and record the version for each object, that\u0027s great, every time we update the object, we should update the version field, the version will make the comparison more effective and we can judge whether the object has been updated, should I add a section to describe it or you do it by yourself:)","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":244,"context_line":"        if found object create\\update\\delete :"},{"line_number":245,"context_line":"            do DF DB operations."},{"line_number":246,"context_line":"            Emit messages via PUB/SUB."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"* During the db comparison, plugin will iterator each object in the two DB,"},{"line_number":249,"context_line":"if an object in Neutron DB could not be found in DF DB, the object should be"},{"line_number":250,"context_line":"considered create, if an object in DF DB could not be found in Neutron DB,"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_41f3e19f","line":247,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: indentation.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":245,"context_line":"            do DF DB operations."},{"line_number":246,"context_line":"            Emit messages via PUB/SUB."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"* During the db comparison, plugin will iterator each object in the two DB,"},{"line_number":249,"context_line":"if an object in Neutron DB could not be found in DF DB, the object should be"},{"line_number":250,"context_line":"considered create, if an object in DF DB could not be found in Neutron DB,"},{"line_number":251,"context_line":"the object should be considered delete, while if an object exists in both DF"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_a1f0f592","line":248,"range":{"start_line":248,"start_character":40,"end_line":248,"end_character":48},"updated":"2016-05-01 09:15:18.000000000","message":"iterator -\u003e iterate","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":257,"context_line":"DB data is changing dynamically, for example, if we find an additional port"},{"line_number":258,"context_line":"in Neutron DB than DF DB during data comparison, then the port may be deleted,"},{"line_number":259,"context_line":"if the delete operation is happened earlier than the create operation, the"},{"line_number":260,"context_line":"dirty data of this port will be stored in DF DB, so we start a green thread"},{"line_number":261,"context_line":"to do the data comparison between Neutron DB and DF DB periodically in Neutron"},{"line_number":262,"context_line":"plugin, if we found the performance bottleneck for Neutron plugin, we could"},{"line_number":263,"context_line":"consider the 3rd-party software (such as an additional process or system OM"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_0138c93c","line":260,"updated":"2016-05-01 09:15:18.000000000","message":"I am not sure I understand how this solves the problem. Could you please elaborate?\nIf the re-sync runs on a separate thread, how do you make sure race-conditions do not override new data with old data?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":257,"context_line":"DB data is changing dynamically, for example, if we find an additional port"},{"line_number":258,"context_line":"in Neutron DB than DF DB during data comparison, then the port may be deleted,"},{"line_number":259,"context_line":"if the delete operation is happened earlier than the create operation, the"},{"line_number":260,"context_line":"dirty data of this port will be stored in DF DB, so we start a green thread"},{"line_number":261,"context_line":"to do the data comparison between Neutron DB and DF DB periodically in Neutron"},{"line_number":262,"context_line":"plugin, if we found the performance bottleneck for Neutron plugin, we could"},{"line_number":263,"context_line":"consider the 3rd-party software (such as an additional process or system OM"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_8f77bf9e","line":260,"in_reply_to":"1a122d0e_0138c93c","updated":"2016-05-03 06:49:53.000000000","message":"The periodically comparison introduces the verificaiton mechanism described below, if confirmed the status of object after two times of comparison, it should get the distributed-lock and check the object from DF DB and do the comfirmed operations, release the lock at last, this could avoid the race-conditions, I\u0027ll describe it detail in next patch:)","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":270,"context_line":"and do the corresponding operations, but if the status is changed, we should"},{"line_number":271,"context_line":"flush the status for the object by the latest status and then wait for next db"},{"line_number":272,"context_line":"comparison. So we need two times of db comparison to confirm the status"},{"line_number":273,"context_line":"of object."},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"    Start db comparison periodically:"},{"line_number":276,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_e1478d78","line":273,"updated":"2016-05-01 09:15:18.000000000","message":"What happens if the Neutron object changes between iterations? Or does this happen under Neutron DB lock?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":270,"context_line":"and do the corresponding operations, but if the status is changed, we should"},{"line_number":271,"context_line":"flush the status for the object by the latest status and then wait for next db"},{"line_number":272,"context_line":"comparison. So we need two times of db comparison to confirm the status"},{"line_number":273,"context_line":"of object."},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"    Start db comparison periodically:"},{"line_number":276,"context_line":"        pull data from DF DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_afe71b45","line":273,"in_reply_to":"1a122d0e_e1478d78","updated":"2016-05-03 06:49:53.000000000","message":"ok, i\u0027ll describe it","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"f76609d5dff51173155e8552657faa2df241fed7","unresolved":false,"context_lines":[{"line_number":271,"context_line":"flush the status for the object by the latest status and then wait for next db"},{"line_number":272,"context_line":"comparison. So we need two times of db comparison to confirm the status"},{"line_number":273,"context_line":"of object."},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"    Start db comparison periodically:"},{"line_number":276,"context_line":"        pull data from DF DB."},{"line_number":277,"context_line":"        pull data from Neutron DB."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_21422569","line":274,"updated":"2016-05-01 09:15:18.000000000","message":"Same here re: ::","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":290,"context_line":"            else:"},{"line_number":291,"context_line":"                refresh object status in cache."},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"Neutron Plugin election"},{"line_number":294,"context_line":"-----------------------"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"We could use the distribute lock mechanism discussed above for the election."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_6bc99779","line":293,"updated":"2016-05-01 19:07:27.000000000","message":"Add: Master/Leader","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":298,"context_line":"distribute lock table, and we should store one data record for each Neutron"},{"line_number":299,"context_line":"plugin in DF DB, the record should be like this, the total_number property"},{"line_number":300,"context_line":"will only be contained in master plugin record:"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    plugin_name, String, primary key"},{"line_number":303,"context_line":"    role, String, master or normal"},{"line_number":304,"context_line":"    status, String, active or down"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_cb01cbb1","line":301,"updated":"2016-05-01 19:07:27.000000000","message":"Consider making a proper restview table, e.g. adding:\n.. csv-table::\n    :header: Attribute,Type,Description","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":304,"context_line":"    status, String, active or down"},{"line_number":305,"context_line":"    hash_factor, int, used to LB"},{"line_number":306,"context_line":"    total_number, int, total active plugin number"},{"line_number":307,"context_line":"    plugin_time, the latest update DateTime"},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"The election process should be like this:"},{"line_number":310,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_2b295f38","line":307,"range":{"start_line":307,"start_character":17,"end_line":307,"end_character":43},"updated":"2016-05-01 19:07:27.000000000","message":"the latest update DateTime -\u003e DateTime, the last update time.","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":307,"context_line":"    plugin_time, the latest update DateTime"},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"The election process should be like this:"},{"line_number":310,"context_line":""},{"line_number":311,"context_line":"    def get_master_neutron_plugin(context):"},{"line_number":312,"context_line":"        nb_lock \u003d lock_db.DBLock(context.election_key)"},{"line_number":313,"context_line":"        with nb_lock:"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_0b2aa32f","line":310,"updated":"2016-05-01 19:07:27.000000000","message":"Same here re: ::","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":320,"context_line":"                return True"},{"line_number":321,"context_line":"            else:"},{"line_number":322,"context_line":"                return False"},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"* Each Neutron plugin will update its own data record in DF DB and detect"},{"line_number":325,"context_line":"the current master data record which describe the info of master plugin"},{"line_number":326,"context_line":"periodically. If normal plugin found master plugin break down, it will update"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_4bdbfb4f","line":323,"updated":"2016-05-01 19:07:27.000000000","message":"Same here re: indentation","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":340,"context_line":"do the db comparison work, it can make the work more effective and avoid single"},{"line_number":341,"context_line":"node bottleneck."},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"Master Neutron plugin will assign hash factor for each existing active plugin"},{"line_number":344,"context_line":"and store the assign result into DF DB. For example, if there are three"},{"line_number":345,"context_line":"active plugins (include master plugin) A, B and C, master plugin could"},{"line_number":346,"context_line":"assign hash factor 0 to A, 1 to B, 2 to C, if a new plugin D join, master"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_8bccf38b","line":343,"range":{"start_line":343,"start_character":0,"end_line":343,"end_character":6},"updated":"2016-05-01 19:07:27.000000000","message":"If this is the only use of the Master Neutron plugin, wouldn\u0027t a distributed algorithm be better?","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":340,"context_line":"do the db comparison work, it can make the work more effective and avoid single"},{"line_number":341,"context_line":"node bottleneck."},{"line_number":342,"context_line":""},{"line_number":343,"context_line":"Master Neutron plugin will assign hash factor for each existing active plugin"},{"line_number":344,"context_line":"and store the assign result into DF DB. For example, if there are three"},{"line_number":345,"context_line":"active plugins (include master plugin) A, B and C, master plugin could"},{"line_number":346,"context_line":"assign hash factor 0 to A, 1 to B, 2 to C, if a new plugin D join, master"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_0f472f1e","line":343,"range":{"start_line":343,"start_character":0,"end_line":343,"end_character":6},"in_reply_to":"1a122d0e_8bccf38b","updated":"2016-05-03 06:49:53.000000000","message":"Here we just describe a simple static hash algorithm to implement the LB, we could also consider better distributed algorithm in the future","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20229,"name":"Omer Anson","email":"omer.anson@toganetworks.com","username":"oanson"},"change_message_id":"9a6bed099fc7eb51703ea6f34a56d81757b01180","unresolved":false,"context_lines":[{"line_number":349,"context_line":"value by object_uuid for each object and get the corresponding result by"},{"line_number":350,"context_line":"using the hash value to mod the total active plugin number, if the result"},{"line_number":351,"context_line":"is equal to the hash factor of the plugin, the object will be processed,"},{"line_number":352,"context_line":"otherwise the object will be passed."},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"Local Controller Data Sync"},{"line_number":355,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_ebbc87dd","line":352,"updated":"2016-05-01 19:07:27.000000000","message":"In this scenario, wouldn\u0027t the data in B be lost? Consider adding a redundancy mechanism (where some plugins are slaves to others and only hold their data, or that objects are stored on more than one Neutron server)","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"023aee9ad347e68ec12c9e3944ceba4dad3c8312","unresolved":false,"context_lines":[{"line_number":349,"context_line":"value by object_uuid for each object and get the corresponding result by"},{"line_number":350,"context_line":"using the hash value to mod the total active plugin number, if the result"},{"line_number":351,"context_line":"is equal to the hash factor of the plugin, the object will be processed,"},{"line_number":352,"context_line":"otherwise the object will be passed."},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"Local Controller Data Sync"},{"line_number":355,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1a122d0e_4fff67aa","line":352,"in_reply_to":"1a122d0e_ebbc87dd","updated":"2016-05-03 06:49:53.000000000","message":"indeed, B is replaced with C because hash factor 1 has been assigned to C instead of B, each plugin just process its concerned data according to the hash factor, so no data will be lost","commit_id":"f04d02570ea9d20c3fb33f004be5315c274fe079"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"f453d6214f1f0aa76dbf140b5e9a0cec05582367","unresolved":false,"context_lines":[{"line_number":400,"context_line":"controller a recover message, local controller process the recover message:"},{"line_number":401,"context_line":""},{"line_number":402,"context_line":"::"},{"line_number":403,"context_line":"    Start handle the recover message:"},{"line_number":404,"context_line":"        get tenant list according to local VM ports."},{"line_number":405,"context_line":"        pull data from DF DB according to tenant list."},{"line_number":406,"context_line":"        compare data between local cache and the got data."},{"line_number":407,"context_line":"        if found object create/update/delete :"},{"line_number":408,"context_line":"            notify to local apps."},{"line_number":409,"context_line":""},{"line_number":410,"context_line":"We should start a green thread to do the data comparison between"},{"line_number":411,"context_line":"local controller cache and DF DB periodically by using the similar"}],"source_content_type":"text/x-rst","patch_set":12,"id":"dab17558_83ec2ebc","line":408,"range":{"start_line":403,"start_character":4,"end_line":408,"end_character":33},"updated":"2016-05-16 14:18:50.000000000","message":"when updating the local cache only update with newer version object and throw away older versions.\n\nWhen using the plug pub-sub older event can reach the compute node after a newer event from the Neutron DB.\nwe need to always compare versions when updating local cache.\n\nWe will need to update the pub sub code to send the value and not only notification","commit_id":"dc15030ec700fbb72adfe0cd57cd864412a7be7b"},{"author":{"_account_id":20188,"name":"zhuweiqiong","email":"zhuweiqiong@huawei.com","username":"zhuweiqiong"},"change_message_id":"040d6bbb8064ed95d536701f32f06b04626924b9","unresolved":false,"context_lines":[{"line_number":414,"context_line":"Comparison By Version ID"},{"line_number":415,"context_line":"------------------------"},{"line_number":416,"context_line":""},{"line_number":417,"context_line":"We add the version_id value for every object, it will be generated when"},{"line_number":418,"context_line":"object is created and updated when the object is updated. We will add an"},{"line_number":419,"context_line":"additional table in Neutron DB to store the version info for each object:"},{"line_number":420,"context_line":""},{"line_number":421,"context_line":".. csv-table::"}],"source_content_type":"text/x-rst","patch_set":12,"id":"dab17558_4ec16280","line":418,"range":{"start_line":417,"start_character":67,"end_line":418,"end_character":56},"updated":"2016-05-06 03:37:13.000000000","message":"what does this mean","commit_id":"dc15030ec700fbb72adfe0cd57cd864412a7be7b"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"876be077578cb85a6e6aeec1f66a9228e0c757c8","unresolved":false,"context_lines":[{"line_number":414,"context_line":"Comparison By Version ID"},{"line_number":415,"context_line":"------------------------"},{"line_number":416,"context_line":""},{"line_number":417,"context_line":"We add the version_id value for every object, it will be generated when"},{"line_number":418,"context_line":"object is created and updated when the object is updated. We will add an"},{"line_number":419,"context_line":"additional table in Neutron DB to store the version info for each object:"},{"line_number":420,"context_line":""},{"line_number":421,"context_line":".. csv-table::"}],"source_content_type":"text/x-rst","patch_set":12,"id":"dab17558_6e72667e","line":418,"range":{"start_line":417,"start_character":67,"end_line":418,"end_character":56},"in_reply_to":"dab17558_4ec16280","updated":"2016-05-06 03:40:34.000000000","message":"when create object, generate and initial the version_id, update object, the version_id will be updated","commit_id":"dc15030ec700fbb72adfe0cd57cd864412a7be7b"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"8bf035625643720ae35a6436b816f639ae8929ca","unresolved":false,"context_lines":[{"line_number":253,"context_line":"if an object in Neutron DB could not be found in DF DB, the object should be"},{"line_number":254,"context_line":"considered create, if an object in DF DB could not be found in Neutron DB,"},{"line_number":255,"context_line":"the object should be considered delete, while if an object exists in both DF"},{"line_number":256,"context_line":"DB and Neutron DB, but some properties are different, the object should be"},{"line_number":257,"context_line":"considered update, if all properties of the two objects are same, the object"},{"line_number":258,"context_line":"should be considered same and pass it."},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"As we know, during the data pulling and comparison period, both of the two"},{"line_number":261,"context_line":"DB data is changing dynamically, for example, a new port data has been written"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_4c04bb7d","line":258,"range":{"start_line":256,"start_character":54,"end_line":258,"end_character":38},"updated":"2016-05-18 14:06:10.000000000","message":"This check should be done using the object version,\n just compare the version and let Neutron DB win in case of conflict","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"78d4aa074db8e8b8a78b4c135dda5641cd067486","unresolved":false,"context_lines":[{"line_number":253,"context_line":"if an object in Neutron DB could not be found in DF DB, the object should be"},{"line_number":254,"context_line":"considered create, if an object in DF DB could not be found in Neutron DB,"},{"line_number":255,"context_line":"the object should be considered delete, while if an object exists in both DF"},{"line_number":256,"context_line":"DB and Neutron DB, but some properties are different, the object should be"},{"line_number":257,"context_line":"considered update, if all properties of the two objects are same, the object"},{"line_number":258,"context_line":"should be considered same and pass it."},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"As we know, during the data pulling and comparison period, both of the two"},{"line_number":261,"context_line":"DB data is changing dynamically, for example, a new port data has been written"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_b11e3a61","line":258,"range":{"start_line":256,"start_character":54,"end_line":258,"end_character":38},"in_reply_to":"bab6814e_4c04bb7d","updated":"2016-05-19 02:46:10.000000000","message":"yes, indeed version different means some properties are different, ok, I\u0027ll modify the description to make it clear","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"8bf035625643720ae35a6436b816f639ae8929ca","unresolved":false,"context_lines":[{"line_number":304,"context_line":"::"},{"line_number":305,"context_line":"    Start db comparison periodically:"},{"line_number":306,"context_line":"        pull data from DF DB."},{"line_number":307,"context_line":"        pull data from Neutron DB."},{"line_number":308,"context_line":"        do compare with two data set."},{"line_number":309,"context_line":"        if found object create/update/delete :"},{"line_number":310,"context_line":"            verification the object."}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_cc89cbcc","line":307,"range":{"start_line":307,"start_character":8,"end_line":307,"end_character":33},"updated":"2016-05-18 14:06:10.000000000","message":"you just need to pull the version table, in this stage","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"78d4aa074db8e8b8a78b4c135dda5641cd067486","unresolved":false,"context_lines":[{"line_number":304,"context_line":"::"},{"line_number":305,"context_line":"    Start db comparison periodically:"},{"line_number":306,"context_line":"        pull data from DF DB."},{"line_number":307,"context_line":"        pull data from Neutron DB."},{"line_number":308,"context_line":"        do compare with two data set."},{"line_number":309,"context_line":"        if found object create/update/delete :"},{"line_number":310,"context_line":"            verification the object."}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_51c0ae7c","line":307,"range":{"start_line":307,"start_character":8,"end_line":307,"end_character":33},"in_reply_to":"bab6814e_cc89cbcc","updated":"2016-05-19 02:46:10.000000000","message":"If the number of different object is few, you can read the version value first and get the different objects one by one from Neutron DB, but If the number is large, for example, the initial stage of the system or the whole df db restart, get the different objects one by one is not proper, so if the memory is enough and we could use for example select * from networks to get all networks first","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"8bf035625643720ae35a6436b816f639ae8929ca","unresolved":false,"context_lines":[{"line_number":305,"context_line":"    Start db comparison periodically:"},{"line_number":306,"context_line":"        pull data from DF DB."},{"line_number":307,"context_line":"        pull data from Neutron DB."},{"line_number":308,"context_line":"        do compare with two data set."},{"line_number":309,"context_line":"        if found object create/update/delete :"},{"line_number":310,"context_line":"            verification the object."},{"line_number":311,"context_line":"            if confirmed object status:"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_4cc79b12","line":308,"range":{"start_line":308,"start_character":8,"end_line":308,"end_character":37},"updated":"2016-05-18 14:06:10.000000000","message":"just compare version","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"78d4aa074db8e8b8a78b4c135dda5641cd067486","unresolved":false,"context_lines":[{"line_number":305,"context_line":"    Start db comparison periodically:"},{"line_number":306,"context_line":"        pull data from DF DB."},{"line_number":307,"context_line":"        pull data from Neutron DB."},{"line_number":308,"context_line":"        do compare with two data set."},{"line_number":309,"context_line":"        if found object create/update/delete :"},{"line_number":310,"context_line":"            verification the object."},{"line_number":311,"context_line":"            if confirmed object status:"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_d13ade97","line":308,"range":{"start_line":308,"start_character":8,"end_line":308,"end_character":37},"in_reply_to":"bab6814e_4cc79b12","updated":"2016-05-19 02:46:10.000000000","message":"yes, that\u0027s what I mean, I\u0027ll describe it more clearly","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"8bf035625643720ae35a6436b816f639ae8929ca","unresolved":false,"context_lines":[{"line_number":363,"context_line":"normal and set status to down. Also master Neutron plugin should detect"},{"line_number":364,"context_line":"other plugins to confirm they are alive periodically."},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"},{"line_number":367,"context_line":"not be released, so each Neutron plugin should check the created-time in the"},{"line_number":368,"context_line":"db-lock and if it found the db-lock is timeout, it could own the db-lock"},{"line_number":369,"context_line":"instead of the crashed Neutron plugin."},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"Neutron Plugin load balance"},{"line_number":372,"context_line":"---------------------------"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_75d69ec6","line":369,"range":{"start_line":366,"start_character":2,"end_line":369,"end_character":38},"updated":"2016-05-18 14:06:10.000000000","message":"Is this still valid with the new compare and swap lock we are using ?","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"78d4aa074db8e8b8a78b4c135dda5641cd067486","unresolved":false,"context_lines":[{"line_number":363,"context_line":"normal and set status to down. Also master Neutron plugin should detect"},{"line_number":364,"context_line":"other plugins to confirm they are alive periodically."},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"},{"line_number":367,"context_line":"not be released, so each Neutron plugin should check the created-time in the"},{"line_number":368,"context_line":"db-lock and if it found the db-lock is timeout, it could own the db-lock"},{"line_number":369,"context_line":"instead of the crashed Neutron plugin."},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"Neutron Plugin load balance"},{"line_number":372,"context_line":"---------------------------"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_f19562e4","line":369,"range":{"start_line":366,"start_character":2,"end_line":369,"end_character":38},"in_reply_to":"bab6814e_75d69ec6","updated":"2016-05-19 02:46:10.000000000","message":"yes, compare and swap is just replacement of previous select-for-update, but it still has the problem that the lock owner crash which lead to un-release the lock, for example, one plugin use compare and swap transaction to get the lock and modify the \"lock\" field to True and it crashed, other plugins could still use compare and swap to fetch the lock, but when they found the \"lock\" field is still True, they could do nothing.","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"8bf035625643720ae35a6436b816f639ae8929ca","unresolved":false,"context_lines":[{"line_number":368,"context_line":"db-lock and if it found the db-lock is timeout, it could own the db-lock"},{"line_number":369,"context_line":"instead of the crashed Neutron plugin."},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"Neutron Plugin load balance"},{"line_number":372,"context_line":"---------------------------"},{"line_number":373,"context_line":""},{"line_number":374,"context_line":"In multi nodes environment, we should consider Neutron plugin load balance to"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_554a7a84","line":371,"range":{"start_line":371,"start_character":1,"end_line":371,"end_character":27},"updated":"2016-05-18 14:06:10.000000000","message":"If I understand correctly this is load balancing of the db consistency check per object correct ?","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"78d4aa074db8e8b8a78b4c135dda5641cd067486","unresolved":false,"context_lines":[{"line_number":368,"context_line":"db-lock and if it found the db-lock is timeout, it could own the db-lock"},{"line_number":369,"context_line":"instead of the crashed Neutron plugin."},{"line_number":370,"context_line":""},{"line_number":371,"context_line":"Neutron Plugin load balance"},{"line_number":372,"context_line":"---------------------------"},{"line_number":373,"context_line":""},{"line_number":374,"context_line":"In multi nodes environment, we should consider Neutron plugin load balance to"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_b11bda1a","line":371,"range":{"start_line":371,"start_character":1,"end_line":371,"end_character":27},"in_reply_to":"bab6814e_554a7a84","updated":"2016-05-19 02:46:10.000000000","message":"yes, I\u0027ll modify the description","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"8bf035625643720ae35a6436b816f639ae8929ca","unresolved":false,"context_lines":[{"line_number":419,"context_line":"Comparison By Version ID"},{"line_number":420,"context_line":"------------------------"},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"We add the version_id value for every object, it will be generated when"},{"line_number":423,"context_line":"object is created and updated when the object is updated. We will add an"},{"line_number":424,"context_line":"additional table in Neutron DB to store the version info for each object:"},{"line_number":425,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_f5556e1d","line":422,"range":{"start_line":422,"start_character":6,"end_line":422,"end_character":45},"updated":"2016-05-18 14:06:10.000000000","message":"It is all ready added","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"78d4aa074db8e8b8a78b4c135dda5641cd067486","unresolved":false,"context_lines":[{"line_number":419,"context_line":"Comparison By Version ID"},{"line_number":420,"context_line":"------------------------"},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"We add the version_id value for every object, it will be generated when"},{"line_number":423,"context_line":"object is created and updated when the object is updated. We will add an"},{"line_number":424,"context_line":"additional table in Neutron DB to store the version info for each object:"},{"line_number":425,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_118746d8","line":422,"range":{"start_line":422,"start_character":6,"end_line":422,"end_character":45},"in_reply_to":"bab6814e_f5556e1d","updated":"2016-05-19 02:46:10.000000000","message":"you mean if the code has implement it, I should delete the description from spec?","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":13070,"name":"Eran Gampel","email":"eran@gampel.net","username":"Gampel"},"change_message_id":"1315a5c2e723818f24af3075c8ba86b43706e354","unresolved":false,"context_lines":[{"line_number":488,"context_line":"Potential Improvements"},{"line_number":489,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":490,"context_line":""},{"line_number":491,"context_line":"1. The SELECT-FOR-UPDATE consumes much computing resources in clustering"},{"line_number":492,"context_line":"by Galera certification process. This can be improved by a SQL-based"},{"line_number":493,"context_line":"compare-and-swap operation which is currently used in Nova [3]."},{"line_number":494,"context_line":""},{"line_number":495,"context_line":"2. For simplicity, we protect the whole API session by distributed lock."},{"line_number":496,"context_line":"This is definitely not optimal. We can use distributed lock to only protect"}],"source_content_type":"text/x-rst","patch_set":13,"id":"dab17558_ff6f545d","line":493,"range":{"start_line":491,"start_character":3,"end_line":493,"end_character":58},"updated":"2016-05-18 13:14:43.000000000","message":"This was all ready merged","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"78d4aa074db8e8b8a78b4c135dda5641cd067486","unresolved":false,"context_lines":[{"line_number":488,"context_line":"Potential Improvements"},{"line_number":489,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":490,"context_line":""},{"line_number":491,"context_line":"1. The SELECT-FOR-UPDATE consumes much computing resources in clustering"},{"line_number":492,"context_line":"by Galera certification process. This can be improved by a SQL-based"},{"line_number":493,"context_line":"compare-and-swap operation which is currently used in Nova [3]."},{"line_number":494,"context_line":""},{"line_number":495,"context_line":"2. For simplicity, we protect the whole API session by distributed lock."},{"line_number":496,"context_line":"This is definitely not optimal. We can use distributed lock to only protect"}],"source_content_type":"text/x-rst","patch_set":13,"id":"bab6814e_1c80fd45","line":493,"range":{"start_line":491,"start_character":3,"end_line":493,"end_character":58},"in_reply_to":"dab17558_ff6f545d","updated":"2016-05-19 02:46:10.000000000","message":"ok","commit_id":"f3b432b552dec497ddb93930d73347371984307d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":128,"context_line":"                Start Neutron DB transaction for network operations."},{"line_number":129,"context_line":"                    Do Neutron DB operations."},{"line_number":130,"context_line":"                Do DF DB operations."},{"line_number":131,"context_line":"                Emit messages via PUB/SUB."},{"line_number":132,"context_line":"            Release the distributed lock."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"* When creating the distributed lock record, it starts a DB transaction and"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_492c9bab","line":131,"updated":"2016-06-10 03:55:13.000000000","message":"message sending should not be under the lock (even in the example)","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"87c9709f8a4dfc72f8519af906d48ca38039c62a","unresolved":false,"context_lines":[{"line_number":128,"context_line":"                Start Neutron DB transaction for network operations."},{"line_number":129,"context_line":"                    Do Neutron DB operations."},{"line_number":130,"context_line":"                Do DF DB operations."},{"line_number":131,"context_line":"                Emit messages via PUB/SUB."},{"line_number":132,"context_line":"            Release the distributed lock."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"* When creating the distributed lock record, it starts a DB transaction and"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_81ea9aaa","line":131,"in_reply_to":"7aa08908_492c9bab","updated":"2016-06-11 12:04:48.000000000","message":"The distribute lock section is committed by Li Ma, and he will discuss it with you :)","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"bd31af36b402f17960cce40cbfc71f011b5da311","unresolved":false,"context_lines":[{"line_number":128,"context_line":"                Start Neutron DB transaction for network operations."},{"line_number":129,"context_line":"                    Do Neutron DB operations."},{"line_number":130,"context_line":"                Do DF DB operations."},{"line_number":131,"context_line":"                Emit messages via PUB/SUB."},{"line_number":132,"context_line":"            Release the distributed lock."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"* When creating the distributed lock record, it starts a DB transaction and"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_86270511","line":131,"in_reply_to":"7aa08908_492c9bab","updated":"2016-06-14 05:37:04.000000000","message":"This is the initial implementation, so I put everything in the lock. If we move it outside, we need to make sure the controller can get the updated object from other channel, because message delivery is not always stable.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"75876102a14f00d935d344c2ed9a5f2e6e070bd6","unresolved":false,"context_lines":[{"line_number":128,"context_line":"                Start Neutron DB transaction for network operations."},{"line_number":129,"context_line":"                    Do Neutron DB operations."},{"line_number":130,"context_line":"                Do DF DB operations."},{"line_number":131,"context_line":"                Emit messages via PUB/SUB."},{"line_number":132,"context_line":"            Release the distributed lock."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"* When creating the distributed lock record, it starts a DB transaction and"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_c67e2dae","line":131,"in_reply_to":"7aa08908_86270511","updated":"2016-06-14 05:40:37.000000000","message":"And make sure the message is in the sequence (but I know it is not always the case). If we want to make sure it is sequential, we need to include the object version in the message and let the receiver decide.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":135,"context_line":"inserts a lock record into Neutron DB according to the current object."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* When acquiring the distributed lock, it first issue SELECT-FOR-UPDATE to"},{"line_number":138,"context_line":"check the lock has been obtained or not. If not, it updates the lock state"},{"line_number":139,"context_line":"and commits the transaction. If exception happens, it will re-try it for"},{"line_number":140,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_9425faaf","line":138,"updated":"2016-06-10 03:55:13.000000000","message":"s/has been obtained/is free/\n\"has been obtained\" sounds like the current process obtained it, not that the lock is already taken.\n\nNote that this step is an optimization only and can be removed from this rst to make it easier to review.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":137,"context_line":"* When acquiring the distributed lock, it first issue SELECT-FOR-UPDATE to"},{"line_number":138,"context_line":"check the lock has been obtained or not. If not, it updates the lock state"},{"line_number":139,"context_line":"and commits the transaction. If exception happens, it will re-try it for"},{"line_number":140,"context_line":"several times. If the lock has been obtained, it will wait and re-try."},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":143,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_b42476af","line":140,"updated":"2016-06-10 03:55:13.000000000","message":"s/If the lock has been obtained/If the lock is already taken/","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":142,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":143,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."},{"line_number":144,"context_line":"If deadlock exception happens, it will retry it to make sure it will be"},{"line_number":145,"context_line":"committed later."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"* Potential Issue: When concurrent write operations on a certain key happen,"},{"line_number":148,"context_line":"due to the inconsistency window of DF DB. If the update on DF DB is always"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_74516e0e","line":145,"updated":"2016-06-10 03:55:13.000000000","message":"This part is not clear, I do not understand what you mean.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"bd31af36b402f17960cce40cbfc71f011b5da311","unresolved":false,"context_lines":[{"line_number":142,"context_line":"* If MySQL clustering involves, only one lock transaction will be committed"},{"line_number":143,"context_line":"and others will be deadlocked. Here we introduce DB retry mechanism."},{"line_number":144,"context_line":"If deadlock exception happens, it will retry it to make sure it will be"},{"line_number":145,"context_line":"committed later."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"* Potential Issue: When concurrent write operations on a certain key happen,"},{"line_number":148,"context_line":"due to the inconsistency window of DF DB. If the update on DF DB is always"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_26ea9174","line":145,"in_reply_to":"7aa08908_74516e0e","updated":"2016-06-14 05:37:04.000000000","message":"The select-for-update lock is local state in MySQL clustering. If one of the node in the cluster has the lock, it will not propagate to other nodes. If a conflict commit happens at another node, it will go through the galera certification process and finally returns a deadlock exception which means \u0027 a lock is there and i cannot process the current statement\u0027. In the neutron, we have to catch that exception, wait a moment and retry the statement to see whether the lock is released.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":148,"context_line":"due to the inconsistency window of DF DB. If the update on DF DB is always"},{"line_number":149,"context_line":"delayed because the previous operations have already been delayed."},{"line_number":150,"context_line":"The root cause is that Neutron DB is strongly consistent but DF DB is"},{"line_number":151,"context_line":"eventually consistent. We cannot guarantee the updates on DF DB is committed."},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"Pseudo Code in Core Plugin"},{"line_number":154,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_5406f220","line":151,"updated":"2016-06-10 03:55:13.000000000","message":"s/ * Potential Issue:.*committed/\nPotential Issue: When using eventually consistent KV store as the DF DB, write conflicts can occur as a result of concurrent Neutron updates. There are potential solutions to this problem which are out of scope of this spec/","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":207,"context_line":"            Do Neutron DB operations."},{"line_number":208,"context_line":"        try:"},{"line_number":209,"context_line":"            Do DF DB operations."},{"line_number":210,"context_line":"            Emit messages via PUB/SUB."},{"line_number":211,"context_line":"        except:"},{"line_number":212,"context_line":"            rollback Neutron DB operations."},{"line_number":213,"context_line":"            raise creation exception."}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_4f4bfb59","line":210,"updated":"2016-06-10 03:55:13.000000000","message":"Again, should be outside of the lock","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"87c9709f8a4dfc72f8519af906d48ca38039c62a","unresolved":false,"context_lines":[{"line_number":207,"context_line":"            Do Neutron DB operations."},{"line_number":208,"context_line":"        try:"},{"line_number":209,"context_line":"            Do DF DB operations."},{"line_number":210,"context_line":"            Emit messages via PUB/SUB."},{"line_number":211,"context_line":"        except:"},{"line_number":212,"context_line":"            rollback Neutron DB operations."},{"line_number":213,"context_line":"            raise creation exception."}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_a1feb668","line":210,"in_reply_to":"7aa08908_4f4bfb59","updated":"2016-06-11 12:04:48.000000000","message":"In current implementation, the df db oper and pub msg is doing together in one single api provided by nb_api module, and pub msg here is just a asynchronous oper","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":215,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":216,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"},{"line_number":217,"context_line":"process, Neutron plugin should rollback the previous commit, and raise"},{"line_number":218,"context_line":"a creation exception."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"When Neutron plugin receives a update/delete object(router/network/subnet/port,"},{"line_number":221,"context_line":"etc) invoke:"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_4fbe5b00","line":218,"updated":"2016-06-10 03:55:13.000000000","message":"Is this rollback performed under the lock?\nRegardless, this can lead to further inconsistencies (rollback can fail for various reasons).\nThe assumption was that the Neutron DB is the master of the data and hence if committed then the action succeeded and from that point on is roll-forward only.\n\nIMO, since you have this mechanism below, you should converge on it for all cases","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"87c9709f8a4dfc72f8519af906d48ca38039c62a","unresolved":false,"context_lines":[{"line_number":215,"context_line":"* After Neutron plugin commit the creation operation to Neutron DB"},{"line_number":216,"context_line":"successfully, if there happened some exceptions in DF DB operations or PUB/SUB"},{"line_number":217,"context_line":"process, Neutron plugin should rollback the previous commit, and raise"},{"line_number":218,"context_line":"a creation exception."},{"line_number":219,"context_line":""},{"line_number":220,"context_line":"When Neutron plugin receives a update/delete object(router/network/subnet/port,"},{"line_number":221,"context_line":"etc) invoke:"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_61120eb6","line":218,"in_reply_to":"7aa08908_4fbe5b00","updated":"2016-06-11 12:04:48.000000000","message":"rollback here is just try to delete the created data in Neutron DB previously which is similar with Ml2 plugin when there happened exception in xxx_postcommit, create operation could be easily rollback which update/delete is hard and no need to rollback, we could deal with it by other method","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":226,"context_line":"            Do Neutron DB operations."},{"line_number":227,"context_line":"        try:"},{"line_number":228,"context_line":"            Do DF DB operations."},{"line_number":229,"context_line":"            Emit messages via PUB/SUB."},{"line_number":230,"context_line":"        except:"},{"line_number":231,"context_line":"            raise update/delete exception."},{"line_number":232,"context_line":""}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_8f3683b2","line":229,"updated":"2016-06-10 03:55:13.000000000","message":"Ditto.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":233,"context_line":"* The difference between update/delete invoke and creation invoke is there is"},{"line_number":234,"context_line":"no need to rollback when catch exception in DF DB operations, for instance,"},{"line_number":235,"context_line":"it is impossible and unnecessary to rollback all the Neutron DB data for a"},{"line_number":236,"context_line":"deleted VM, and we can deal with the dirty data in DF DB by other methods."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"When DB driver and pub/sub driver find the read/write connection between"},{"line_number":239,"context_line":"Neutron plugin and DF DB, and also the pub/sub connection between Neutron"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_6f6237af","line":236,"updated":"2016-06-10 03:55:13.000000000","message":"Deleted data is garbage, but update is a different story.\nHow do you propose to deal with out of date info in DF DB here?","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"87c9709f8a4dfc72f8519af906d48ca38039c62a","unresolved":false,"context_lines":[{"line_number":233,"context_line":"* The difference between update/delete invoke and creation invoke is there is"},{"line_number":234,"context_line":"no need to rollback when catch exception in DF DB operations, for instance,"},{"line_number":235,"context_line":"it is impossible and unnecessary to rollback all the Neutron DB data for a"},{"line_number":236,"context_line":"deleted VM, and we can deal with the dirty data in DF DB by other methods."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"When DB driver and pub/sub driver find the read/write connection between"},{"line_number":239,"context_line":"Neutron plugin and DF DB, and also the pub/sub connection between Neutron"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_810f3a87","line":236,"in_reply_to":"7aa08908_6f6237af","updated":"2016-06-11 12:04:48.000000000","message":"I think the update and delete situation is similar, it\u0027s hard and unnecessary to rollback, we can deal with it by other methods","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":246,"context_line":"        pull data from Neutron DB."},{"line_number":247,"context_line":"        do compare with two data set."},{"line_number":248,"context_line":"        if found object create/update/delete :"},{"line_number":249,"context_line":"            do DF DB operations."},{"line_number":250,"context_line":"            Emit messages via PUB/SUB."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"* During the db comparison, plugin will iterate each object in the two DB,"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_2f694f8c","line":249,"updated":"2016-06-10 03:55:13.000000000","message":"What happens if these operations fail?","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"87c9709f8a4dfc72f8519af906d48ca38039c62a","unresolved":false,"context_lines":[{"line_number":246,"context_line":"        pull data from Neutron DB."},{"line_number":247,"context_line":"        do compare with two data set."},{"line_number":248,"context_line":"        if found object create/update/delete :"},{"line_number":249,"context_line":"            do DF DB operations."},{"line_number":250,"context_line":"            Emit messages via PUB/SUB."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"* During the db comparison, plugin will iterate each object in the two DB,"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_417c32eb","line":249,"in_reply_to":"7aa08908_2f694f8c","updated":"2016-06-11 12:04:48.000000000","message":"if the opers fail, it means there are some exceptions in DF db, we could do nothing at this time, and wait for next recover msg or periodically db comparison","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":247,"context_line":"        do compare with two data set."},{"line_number":248,"context_line":"        if found object create/update/delete :"},{"line_number":249,"context_line":"            do DF DB operations."},{"line_number":250,"context_line":"            Emit messages via PUB/SUB."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"* During the db comparison, plugin will iterate each object in the two DB,"},{"line_number":253,"context_line":"if an object in Neutron DB could not be found in DF DB, the object should be"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_4f6c9b9c","line":250,"updated":"2016-06-10 03:55:13.000000000","message":"ditto.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":310,"context_line":"            verification the object."},{"line_number":311,"context_line":"            if confirmed object status:"},{"line_number":312,"context_line":"                try:"},{"line_number":313,"context_line":"                    db_lock \u003d lock_db.DBLock(context.tenant_id)"},{"line_number":314,"context_line":"                    with db_lock:"},{"line_number":315,"context_line":"                        read and check object from DF DB."},{"line_number":316,"context_line":"                        if everything confirmed:"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_afb89fe6","line":313,"updated":"2016-06-10 03:55:13.000000000","message":"taking the lock per object would be extremely slow.\nwhy not create a list of objects to update per tenant and then run through each list under the lock? (each list can later also be processed in parallel if we want to increase the speed)","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"87c9709f8a4dfc72f8519af906d48ca38039c62a","unresolved":false,"context_lines":[{"line_number":310,"context_line":"            verification the object."},{"line_number":311,"context_line":"            if confirmed object status:"},{"line_number":312,"context_line":"                try:"},{"line_number":313,"context_line":"                    db_lock \u003d lock_db.DBLock(context.tenant_id)"},{"line_number":314,"context_line":"                    with db_lock:"},{"line_number":315,"context_line":"                        read and check object from DF DB."},{"line_number":316,"context_line":"                        if everything confirmed:"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_815d7a7b","line":313,"in_reply_to":"7aa08908_afb89fe6","updated":"2016-06-11 12:04:48.000000000","message":"yes, this is just a initial solution which the lock is fetch by the tenant, In future we could get the lock by router or network (by smaller range), and what you suggest will speed up the process:)","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":12257,"name":"Ayal Baron","email":"ayal.baron@huawei.com","username":"AyalBaron"},"change_message_id":"84ccac2c0d2b1bbdf174b6aaca2c7678008b2d7d","unresolved":false,"context_lines":[{"line_number":361,"context_line":"periodically. If normal plugin found master plugin break down, it will update"},{"line_number":362,"context_line":"its own data record to become new master plugin and change the old master to"},{"line_number":363,"context_line":"normal and set status to down. Also master Neutron plugin should detect"},{"line_number":364,"context_line":"other plugins to confirm they are alive periodically."},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"},{"line_number":367,"context_line":"not be released, so each Neutron plugin should check the created-time in the"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_ef78e72d","line":364,"updated":"2016-06-10 03:55:13.000000000","message":"Iiuc, then you are relying on time synchronization between the neutron API servers.\nThis is a fragile assumption that can lead to multiple servers thinking they are the master.\n\nAlso, you are relying on the DB lock here to obtain this lock. I thought the whole point of this mechanism is that the db lock mechanism is not good enough when using clustered mysql as the neutron DB, so why is the lock good enough here?","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":7805,"name":"Li Ma","email":"skywalker.nick@gmail.com","username":"Nick"},"change_message_id":"bd31af36b402f17960cce40cbfc71f011b5da311","unresolved":false,"context_lines":[{"line_number":361,"context_line":"periodically. If normal plugin found master plugin break down, it will update"},{"line_number":362,"context_line":"its own data record to become new master plugin and change the old master to"},{"line_number":363,"context_line":"normal and set status to down. Also master Neutron plugin should detect"},{"line_number":364,"context_line":"other plugins to confirm they are alive periodically."},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"},{"line_number":367,"context_line":"not be released, so each Neutron plugin should check the created-time in the"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_e6a6c9bf","line":364,"in_reply_to":"7aa08908_ef78e72d","updated":"2016-06-14 05:37:04.000000000","message":"Actually, OpenStack rely on time synchronization to work. I don\u0027t think it is the big problem in dragonflow. We switched db lock from sql lock to compare and swap. As a result, it doesn\u0027t care if the db is clustered or not. The lock depends on atomic update statement not on select for update statement.","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"},{"author":{"_account_id":20287,"name":"hujie","email":"mike.hu@huawei.com","username":"hujie"},"change_message_id":"87c9709f8a4dfc72f8519af906d48ca38039c62a","unresolved":false,"context_lines":[{"line_number":361,"context_line":"periodically. If normal plugin found master plugin break down, it will update"},{"line_number":362,"context_line":"its own data record to become new master plugin and change the old master to"},{"line_number":363,"context_line":"normal and set status to down. Also master Neutron plugin should detect"},{"line_number":364,"context_line":"other plugins to confirm they are alive periodically."},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"* If a Neutron plugin has got the db-lock, but it is crashed, the db-lock may"},{"line_number":367,"context_line":"not be released, so each Neutron plugin should check the created-time in the"}],"source_content_type":"text/x-rst","patch_set":14,"id":"7aa08908_41ca721b","line":364,"in_reply_to":"7aa08908_ef78e72d","updated":"2016-06-11 12:04:48.000000000","message":"yes, if the lock mechanism is good enough, this could be done","commit_id":"e79e218e20d2e9355869a9aef9588f307759134d"}]}