)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":14957,"name":"Brad Behle","email":"behle@us.ibm.com","username":"behle"},"change_message_id":"b37e89da18f3e044933feead5d4252ad4a3ee035","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Oleg Bondarev \u003cobondarev@mirantis.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2015-10-19 14:57:24 +0300"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"L3 agent: sync routers by chunks to prevent fatal memory consumption"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"In case there are thousands of routers attached to thousands of"},{"line_number":10,"context_line":"networks, full l3 agent resync might become a big problem on server"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"3a29b11f_257d831c","line":7,"updated":"2015-10-19 18:26:30.000000000","message":"To make this a bit more clear, change this to: Get router info in small chunks to prevent fatal memory consumption by neutron-server","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"4ca41f6a3b3ed3cdd4a75a9a09a33f41426f9390","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Oleg Bondarev \u003cobondarev@mirantis.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2015-10-19 14:57:24 +0300"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"L3 agent: sync routers by chunks to prevent fatal memory consumption"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"In case there are thousands of routers attached to thousands of"},{"line_number":10,"context_line":"networks, full l3 agent resync might become a big problem on server"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"3a29b11f_28ab011f","line":7,"in_reply_to":"3a29b11f_257d831c","updated":"2015-10-20 08:34:01.000000000","message":"This might be too long according to [1], will try to rephrase to make it more clear\n\n[1] https://wiki.openstack.org/wiki/GitCommitMessages#Summary_of_Git_commit_message_structure","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":704,"name":"Edgar Magana","email":"emagana@gmail.com","username":"emagana"},"change_message_id":"fee6b96e86a83b667b729a1338ec07bbe3344052","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"In case there are thousands of routers attached to thousands of"},{"line_number":10,"context_line":"networks, full l3 agent resync might become a big problem on server"},{"line_number":11,"context_line":"side due to serious memory consumption."},{"line_number":12,"context_line":"This patch makes l3 agent first check how many routers are assigned to"},{"line_number":13,"context_line":"it and then start to fetch routers by chunks not exceeding configured"},{"line_number":14,"context_line":"value."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"3a29b11f_a7e99b1c","line":11,"updated":"2015-10-19 17:29:12.000000000","message":"Yes! Thanks  :-)","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":11682,"name":"Ryan Moats","email":"rmoats@us.ibm.com","username":"regXboi"},"change_message_id":"a3ebf5c0f0769708924321ed4ed3099f4a87bcb3","unresolved":false,"context_lines":[{"line_number":12,"context_line":"This patch makes l3 agent first check how many routers are assigned to"},{"line_number":13,"context_line":"it and then start to fetch routers by chunks not exceeding configured"},{"line_number":14,"context_line":"value."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Closes-Bug: #1505575"},{"line_number":17,"context_line":"Change-Id: Id675910c2a0b862bfb9e6f4fdaf3cd9fe337e52f"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"3a29b11f_b8674324","line":15,"updated":"2015-10-20 16:39:19.000000000","message":"Since we are adding a new configuration parameter, shouldn\u0027t we also have a DocImpact tag here?","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":11682,"name":"Ryan Moats","email":"rmoats@us.ibm.com","username":"regXboi"},"change_message_id":"441f986b616fe33f8e6c2593d8f8ba084b05fa6b","unresolved":false,"context_lines":[{"line_number":12,"context_line":"This patch makes l3 agent first check how many routers are assigned to"},{"line_number":13,"context_line":"it and then start to fetch routers by chunks not exceeding configured"},{"line_number":14,"context_line":"value."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Closes-Bug: #1505575"},{"line_number":17,"context_line":"Change-Id: Id675910c2a0b862bfb9e6f4fdaf3cd9fe337e52f"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"3a29b11f_b4d06a20","line":15,"in_reply_to":"3a29b11f_2568cab8","updated":"2015-10-21 20:30:47.000000000","message":"Ack- I\u0027ll look for the final result","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"9638ad296cb7fb5394444ad20f98d6128b7e779b","unresolved":false,"context_lines":[{"line_number":12,"context_line":"This patch makes l3 agent first check how many routers are assigned to"},{"line_number":13,"context_line":"it and then start to fetch routers by chunks not exceeding configured"},{"line_number":14,"context_line":"value."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Closes-Bug: #1505575"},{"line_number":17,"context_line":"Change-Id: Id675910c2a0b862bfb9e6f4fdaf3cd9fe337e52f"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"3a29b11f_2568cab8","line":15,"in_reply_to":"3a29b11f_b8674324","updated":"2015-10-21 10:00:13.000000000","message":"There are concerns regarding config parameter so it may be reconsidered","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"}],"etc/l3_agent.ini":[{"author":{"_account_id":704,"name":"Edgar Magana","email":"emagana@gmail.com","username":"emagana"},"change_message_id":"fee6b96e86a83b667b729a1338ec07bbe3344052","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"# Number of routers to fetch from server at a time on resync."},{"line_number":33,"context_line":"# Needed to prevent fatal memory consumption on server side."},{"line_number":34,"context_line":"# sync_routers_chunk_size \u003d 200"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"# When external_network_bridge is set, each L3 agent can be associated"},{"line_number":37,"context_line":"# with no more than one external network. This value should be set to the UUID"}],"source_content_type":"text/x-properties","patch_set":3,"id":"3a29b11f_27782bc4","line":34,"updated":"2015-10-19 17:29:12.000000000","message":"I still feel that 200 is way big. I will suggest to move it to 50 as the default value.","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"0e555f761e88515db5e52e30285dd94181baeb23","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"# Number of routers to fetch from server at a time on resync."},{"line_number":33,"context_line":"# Needed to prevent fatal memory consumption on server side."},{"line_number":34,"context_line":"# sync_routers_chunk_size \u003d 200"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"# When external_network_bridge is set, each L3 agent can be associated"},{"line_number":37,"context_line":"# with no more than one external network. This value should be set to the UUID"}],"source_content_type":"text/x-properties","patch_set":3,"id":"3a29b11f_0bb35f99","line":34,"in_reply_to":"3a29b11f_15909942","updated":"2015-10-21 10:52:39.000000000","message":"I start to think that 50 might be better default: it should suit for most (production) environments and still be quite transparent. Making it confugarable or not is still questionable however. Armando suggested to find a way to figure it out dynamically but that\u0027s quite nontrivial.","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":14957,"name":"Brad Behle","email":"behle@us.ibm.com","username":"behle"},"change_message_id":"b37e89da18f3e044933feead5d4252ad4a3ee035","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"# Number of routers to fetch from server at a time on resync."},{"line_number":33,"context_line":"# Needed to prevent fatal memory consumption on server side."},{"line_number":34,"context_line":"# sync_routers_chunk_size \u003d 200"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"# When external_network_bridge is set, each L3 agent can be associated"},{"line_number":37,"context_line":"# with no more than one external network. This value should be set to the UUID"}],"source_content_type":"text/x-properties","patch_set":3,"id":"3a29b11f_a5b673da","line":34,"in_reply_to":"3a29b11f_27782bc4","updated":"2015-10-19 18:26:30.000000000","message":"Have you tested the memory usage of a scenario like this with the chunk size at 200?  If the memory use doesn\u0027t go up much, then maybe 200 is fine, but it seems like 50 or 100 is a better default.  Since it is configurable it is better to be on the safe side, and customers can always raise it if they decide a higher value works better for them.","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"4ca41f6a3b3ed3cdd4a75a9a09a33f41426f9390","unresolved":false,"context_lines":[{"line_number":31,"context_line":""},{"line_number":32,"context_line":"# Number of routers to fetch from server at a time on resync."},{"line_number":33,"context_line":"# Needed to prevent fatal memory consumption on server side."},{"line_number":34,"context_line":"# sync_routers_chunk_size \u003d 200"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"# When external_network_bridge is set, each L3 agent can be associated"},{"line_number":37,"context_line":"# with no more than one external network. This value should be set to the UUID"}],"source_content_type":"text/x-properties","patch_set":3,"id":"3a29b11f_15909942","line":34,"in_reply_to":"3a29b11f_a5b673da","updated":"2015-10-20 08:34:01.000000000","message":"200 was tested and proved working fine, decreasing it will increase chattiness on the bus/db (please see discussion in PS1). Agree with Brad: once admin face the problem of memory consumption he/she can tune the parameter, otherwise we better be closer to original behavior.","commit_id":"0906cbfb6fa9745a0fa47067185155768c785a8c"}],"neutron/agent/l3/agent.py":[{"author":{"_account_id":7448,"name":"Carl Baldwin","email":"carl@ecbaldwin.net","username":"carl-baldwin"},"change_message_id":"e0e6eb2112ad1022f90c80ea99ee59e4157331cc","unresolved":false,"context_lines":[{"line_number":557,"context_line":"        timestamp \u003d timeutils.utcnow()"},{"line_number":558,"context_line":""},{"line_number":559,"context_line":"        try:"},{"line_number":560,"context_line":"            router_ids \u003d (self.plugin_rpc.get_router_ids(context) if"},{"line_number":561,"context_line":"                          self.conf.use_namespaces else [self.conf.router_id])"},{"line_number":562,"context_line":"            routers \u003d []"},{"line_number":563,"context_line":"            # fetch routers by chunks to prevent fatal memory consumption on"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a2ca52d_b8c2c832","line":560,"updated":"2015-10-15 19:59:59.000000000","message":"This really should be a list comprehension rather than a generator.  Calling len(...) and indexing a generator is going to be expensive, no?\n\nSo, just change ( to [?","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"55786f8ae3fe5098080809f95399f57125df02e9","unresolved":false,"context_lines":[{"line_number":557,"context_line":"        timestamp \u003d timeutils.utcnow()"},{"line_number":558,"context_line":""},{"line_number":559,"context_line":"        try:"},{"line_number":560,"context_line":"            router_ids \u003d (self.plugin_rpc.get_router_ids(context) if"},{"line_number":561,"context_line":"                          self.conf.use_namespaces else [self.conf.router_id])"},{"line_number":562,"context_line":"            routers \u003d []"},{"line_number":563,"context_line":"            # fetch routers by chunks to prevent fatal memory consumption on"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a2ca52d_ec386fa2","line":560,"in_reply_to":"5a2ca52d_b8c2c832","updated":"2015-10-16 07:13:30.000000000","message":"Actually it\u0027s not a generator, plugin_rpc.get_router_ids() returns a list, () are used for line breaking to avoid \\ and are indeed confusing, it\u0027s probably better to go with \\ in this case..","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"edc96b211d78bc73b31019cf41a138cc61766268","unresolved":false,"context_lines":[{"line_number":572,"context_line":"            # server side"},{"line_number":573,"context_line":"            for i in range(0, len(router_ids), SYNC_ROUTERS_CHUNK_SIZE):"},{"line_number":574,"context_line":"                routers +\u003d self.plugin_rpc.get_routers("},{"line_number":575,"context_line":"                    context, router_ids[i:i + SYNC_ROUTERS_CHUNK_SIZE])"},{"line_number":576,"context_line":"        except oslo_messaging.MessagingException:"},{"line_number":577,"context_line":"            LOG.exception(_LE(\"Failed synchronizing routers due to RPC error\"))"},{"line_number":578,"context_line":"            raise n_exc.AbortSyncRouters()"}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_866598e2","line":575,"updated":"2015-11-07 01:48:42.000000000","message":"This is most interesting part of this patch, thanks. But I think we can make this patch better.\n\nI\u0027m curious on why we need to fetch all routers then go to process them, but not fetch some, process them and fetch another.\n\nIf a MessagingException happened during fetching, all routers we have already fetched will lose, and we have to fetch them again in next periodic task. That\u0027s wasting time and rpc. At least, maybe we can use some catch to store routers(or just their ids with timestamp) to prevent repeated fetching in nearby full resync task.","commit_id":"f007cf7468ce9bd74bd5be193fe801b65776a2c5"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"27e670d28779f761a9704bb8b19d96e5d1c51ad5","unresolved":false,"context_lines":[{"line_number":572,"context_line":"            # server side"},{"line_number":573,"context_line":"            for i in range(0, len(router_ids), SYNC_ROUTERS_CHUNK_SIZE):"},{"line_number":574,"context_line":"                routers +\u003d self.plugin_rpc.get_routers("},{"line_number":575,"context_line":"                    context, router_ids[i:i + SYNC_ROUTERS_CHUNK_SIZE])"},{"line_number":576,"context_line":"        except oslo_messaging.MessagingException:"},{"line_number":577,"context_line":"            LOG.exception(_LE(\"Failed synchronizing routers due to RPC error\"))"},{"line_number":578,"context_line":"            raise n_exc.AbortSyncRouters()"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba8a016a_73e4fa0c","line":575,"in_reply_to":"da85f559_866598e2","updated":"2015-11-18 16:56:21.000000000","message":"This is planned as the next step, current patch tries to preserve original agent behavior as much as possible for better catching regressions","commit_id":"f007cf7468ce9bd74bd5be193fe801b65776a2c5"},{"author":{"_account_id":7448,"name":"Carl Baldwin","email":"carl@ecbaldwin.net","username":"carl-baldwin"},"change_message_id":"a9d9c27553ff630a03d92cf05ab423c40e54accc","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"# Number of routers to fetch from server at a time on resync."},{"line_number":68,"context_line":"# Needed to prevent fatal memory consumption on server side."},{"line_number":69,"context_line":"SYNC_ROUTERS_CHUNK_SIZE \u003d 50"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"class L3PluginApi(object):"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba8a016a_9f8cea80","line":69,"updated":"2015-11-20 18:00:59.000000000","message":"What are your current thoughts on making this dynamic in a follow-on?","commit_id":"5b89681d8435dc397473ad63a3fdfa80ee736ffc"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"ee2168115927006d39034b258b7f82bb349704d1","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"# Number of routers to fetch from server at a time on resync."},{"line_number":68,"context_line":"# Needed to prevent fatal memory consumption on server side."},{"line_number":69,"context_line":"SYNC_ROUTERS_CHUNK_SIZE \u003d 50"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"class L3PluginApi(object):"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba8a016a_6faa8a1b","line":69,"in_reply_to":"ba8a016a_9f8cea80","updated":"2015-11-23 10:30:04.000000000","message":"What I\u0027m thinking now is that memory consumption may not longer be a problem after fix https://review.openstack.org/#/c/214974/ (will verify this soon)\n\nHowever timeouts might still happen as described in the bug filed recently: https://bugs.launchpad.net/neutron/+bug/1516260\n\nSo my thoughts on dynamic approach is that we can set some initial value (say 100) and decrease it dynamically in case timeouts happen on sync_routers() call. This should make l3 agent more responsive comparing to increasing timeout approach (assuming we start processing after getting the first chunk). Does that sound reasonable? If yes, it should be better done in scope of this patch IMO","commit_id":"5b89681d8435dc397473ad63a3fdfa80ee736ffc"},{"author":{"_account_id":7448,"name":"Carl Baldwin","email":"carl@ecbaldwin.net","username":"carl-baldwin"},"change_message_id":"a9d9c27553ff630a03d92cf05ab423c40e54accc","unresolved":false,"context_lines":[{"line_number":545,"context_line":"            routers \u003d []"},{"line_number":546,"context_line":"            # fetch routers by chunks to prevent fatal memory consumption on"},{"line_number":547,"context_line":"            # server side"},{"line_number":548,"context_line":"            for i in range(0, len(router_ids), SYNC_ROUTERS_CHUNK_SIZE):"},{"line_number":549,"context_line":"                routers +\u003d self.plugin_rpc.get_routers("},{"line_number":550,"context_line":"                    context, router_ids[i:i + SYNC_ROUTERS_CHUNK_SIZE])"},{"line_number":551,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba8a016a_9fb5ca32","line":548,"updated":"2015-11-20 18:00:59.000000000","message":"Any thoughts on queueing up the first batch (L556) before trying to fetch the second batch from the server?","commit_id":"5b89681d8435dc397473ad63a3fdfa80ee736ffc"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"ee2168115927006d39034b258b7f82bb349704d1","unresolved":false,"context_lines":[{"line_number":545,"context_line":"            routers \u003d []"},{"line_number":546,"context_line":"            # fetch routers by chunks to prevent fatal memory consumption on"},{"line_number":547,"context_line":"            # server side"},{"line_number":548,"context_line":"            for i in range(0, len(router_ids), SYNC_ROUTERS_CHUNK_SIZE):"},{"line_number":549,"context_line":"                routers +\u003d self.plugin_rpc.get_routers("},{"line_number":550,"context_line":"                    context, router_ids[i:i + SYNC_ROUTERS_CHUNK_SIZE])"},{"line_number":551,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":6,"id":"ba8a016a_f21f3d59","line":548,"in_reply_to":"ba8a016a_9fb5ca32","updated":"2015-11-23 10:30:04.000000000","message":"This is definitely planned, as stated in the commit message. Do you think it makes sense to add it in scope of this patch?","commit_id":"5b89681d8435dc397473ad63a3fdfa80ee736ffc"},{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"2fb2d3c644fc5ee6f97ded9ad72fafb938765e8d","unresolved":false,"context_lines":[{"line_number":567,"context_line":"                        timestamp\u003dtimestamp)"},{"line_number":568,"context_line":"                    self._queue.add(update)"},{"line_number":569,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":570,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":571,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":572,"context_line":"                raise"},{"line_number":573,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":11,"id":"3a7e1126_07a1a5a9","line":570,"range":{"start_line":570,"start_character":19,"end_line":570,"end_character":20},"updated":"2015-12-17 02:37:55.000000000","message":"In case sync_routers_chunks_size became less than the min chunk size, then we just eat the exception and assume the operation is a success? I must be missing something. Shouldn\u0027t we raise an abortSyncRouters and signal via LOG.error that the server is grinding to a halt?","commit_id":"d72106c6ada7d760138fc6acd139ada63b9686da"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"f6acb43ec68f42efbbb4eb1c5d36796fb5606fd4","unresolved":false,"context_lines":[{"line_number":567,"context_line":"                        timestamp\u003dtimestamp)"},{"line_number":568,"context_line":"                    self._queue.add(update)"},{"line_number":569,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":570,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":571,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":572,"context_line":"                raise"},{"line_number":573,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":11,"id":"3a7e1126_a430b657","line":570,"range":{"start_line":570,"start_character":19,"end_line":570,"end_character":20},"in_reply_to":"3a7e1126_07a1a5a9","updated":"2015-12-17 08:47:09.000000000","message":"Good catch!","commit_id":"d72106c6ada7d760138fc6acd139ada63b9686da"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"9cb0d27e9a79800ee88744bf53ae1917b39f675b","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                    self._queue.add(update)"},{"line_number":572,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":573,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":574,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":575,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":576,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"},{"line_number":577,"context_line":"                          self.sync_routers_chunk_size)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_968342cc","line":574,"range":{"start_line":574,"start_character":16,"end_line":574,"end_character":49},"updated":"2015-12-17 17:03:56.000000000","message":"Where will sync_routers_chunk_size grown up, do I miss something?\n\nWhat if there is a MessagingTimeout happened by accident, then the chunk size will be cut by half, and agent will have to use half traffic to get routers, it may lose efficiency here.","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":11682,"name":"Ryan Moats","email":"rmoats@us.ibm.com","username":"regXboi"},"change_message_id":"c6dfa6881c97d2df20a433ebfe943b6214abffab","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                    self._queue.add(update)"},{"line_number":572,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":573,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":574,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":575,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":576,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"},{"line_number":577,"context_line":"                          self.sync_routers_chunk_size)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_d4a2e2fc","line":574,"range":{"start_line":574,"start_character":16,"end_line":574,"end_character":49},"in_reply_to":"3a7e1126_34ad8931","updated":"2015-12-18 17:15:08.000000000","message":"Armando, wouldn\u0027t we want to do something like with the TCP window here (i.e. /\u003d 2 on a failure and *\u003d 1.1 on a success?)","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"58328d634a8c694afea0500331f3c5e9e1b63081","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                    self._queue.add(update)"},{"line_number":572,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":573,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":574,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":575,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":576,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"},{"line_number":577,"context_line":"                          self.sync_routers_chunk_size)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_a25b862d","line":574,"range":{"start_line":574,"start_character":16,"end_line":574,"end_character":49},"in_reply_to":"3a7e1126_968342cc","updated":"2015-12-18 00:40:54.000000000","message":"I considered doubling the size of the chunk in case a streak of full sync(s) without Timeouts. It\u0027s doable, but I wonder if it\u0027s overkill for this patch. On the other end, you might just end up shooting yourself in the foot and make the algorithm unstable (e.g. flipping between two chunk sizes - the bad and the good one).","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"be450dc56d2deb8f6c4cd282d65f69943fc31273","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                    self._queue.add(update)"},{"line_number":572,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":573,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":574,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":575,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":576,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"},{"line_number":577,"context_line":"                          self.sync_routers_chunk_size)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_34ad8931","line":574,"range":{"start_line":574,"start_character":16,"end_line":574,"end_character":49},"in_reply_to":"3a7e1126_a25b862d","updated":"2015-12-18 06:47:32.000000000","message":"OK, I will consider a better solution(if exist) should be another patch\u0027s job.","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"394f968b5a1d0f00912511fdc5b3ce34bc53c59c","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                    self._queue.add(update)"},{"line_number":572,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":573,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":574,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":575,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":576,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"},{"line_number":577,"context_line":"                          self.sync_routers_chunk_size)"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a7b0d38_1b13fe0e","line":574,"range":{"start_line":574,"start_character":16,"end_line":574,"end_character":49},"in_reply_to":"3a7e1126_c68f5173","updated":"2015-12-24 09:22:21.000000000","message":"Will add in the next patch set","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"5e217b0cc62aced4be0967db207c3558e6755e97","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                    self._queue.add(update)"},{"line_number":572,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":573,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":574,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":575,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":576,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"},{"line_number":577,"context_line":"                          self.sync_routers_chunk_size)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_e40ba58c","line":574,"range":{"start_line":574,"start_character":16,"end_line":574,"end_character":49},"in_reply_to":"3a7e1126_d4a2e2fc","updated":"2015-12-18 22:56:45.000000000","message":"We could, but we gotta cap at MAX size, no?","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":11682,"name":"Ryan Moats","email":"rmoats@us.ibm.com","username":"regXboi"},"change_message_id":"f8bc0b67e696761deab9e33e5245cc3601a1a23c","unresolved":false,"context_lines":[{"line_number":571,"context_line":"                    self._queue.add(update)"},{"line_number":572,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":573,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":574,"context_line":"                self.sync_routers_chunk_size /\u003d 2"},{"line_number":575,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":576,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"},{"line_number":577,"context_line":"                          self.sync_routers_chunk_size)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_c68f5173","line":574,"range":{"start_line":574,"start_character":16,"end_line":574,"end_character":49},"in_reply_to":"3a7e1126_e40ba58c","updated":"2015-12-21 17:59:00.000000000","message":"Exactly - we scale up with some slow geometric scale factor and then cap at MAX size...","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"9cb0d27e9a79800ee88744bf53ae1917b39f675b","unresolved":false,"context_lines":[{"line_number":578,"context_line":"            else:"},{"line_number":579,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":580,"context_line":"                              \u0027required time even with min chunk size: %s. \u0027"},{"line_number":581,"context_line":"                              \u0027It might be under very high load or \u0027"},{"line_number":582,"context_line":"                              \u0027just inoperable\u0027),"},{"line_number":583,"context_line":"                          self.sync_routers_chunk_size)"},{"line_number":584,"context_line":"            raise"},{"line_number":585,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_b912f107","line":582,"range":{"start_line":581,"start_character":49,"end_line":582,"end_character":47},"updated":"2015-12-17 17:03:56.000000000","message":"Question: If that case happened for quite a while, will this agent be quite for a while to leave chances for other agents? Or we should just let it be noise?","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"356d785a3f8a8946ec9e77577948ecdd26856304","unresolved":false,"context_lines":[{"line_number":578,"context_line":"            else:"},{"line_number":579,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":580,"context_line":"                              \u0027required time even with min chunk size: %s. \u0027"},{"line_number":581,"context_line":"                              \u0027It might be under very high load or \u0027"},{"line_number":582,"context_line":"                              \u0027just inoperable\u0027),"},{"line_number":583,"context_line":"                          self.sync_routers_chunk_size)"},{"line_number":584,"context_line":"            raise"},{"line_number":585,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_66ec96f3","line":582,"range":{"start_line":581,"start_character":49,"end_line":582,"end_character":47},"in_reply_to":"3a7e1126_20ac64ec","updated":"2015-12-19 08:12:17.000000000","message":"My previous point is, if there are multiple l3 agents resync and there are just few rpc workers on server side, will timeout agents hold too many retries to interrupt those low load agents resync.\n\nI\u0027m not sure whether will that happen, and should we concern that, so it\u0027s just a question, not the reason for my previous -1.","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"394f968b5a1d0f00912511fdc5b3ce34bc53c59c","unresolved":false,"context_lines":[{"line_number":578,"context_line":"            else:"},{"line_number":579,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":580,"context_line":"                              \u0027required time even with min chunk size: %s. \u0027"},{"line_number":581,"context_line":"                              \u0027It might be under very high load or \u0027"},{"line_number":582,"context_line":"                              \u0027just inoperable\u0027),"},{"line_number":583,"context_line":"                          self.sync_routers_chunk_size)"},{"line_number":584,"context_line":"            raise"},{"line_number":585,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a7b0d38_1b5c5eec","line":582,"range":{"start_line":581,"start_character":49,"end_line":582,"end_character":47},"in_reply_to":"3a7e1126_66ec96f3","updated":"2015-12-24 09:22:21.000000000","message":"An l3 agent will retry to sync its state until success and this patch does not change this behavior. However this patch improves the chances of successful sync","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"3b77105ec31d0e71e0f7454f22ec3c0a6e61ced2","unresolved":false,"context_lines":[{"line_number":578,"context_line":"            else:"},{"line_number":579,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":580,"context_line":"                              \u0027required time even with min chunk size: %s. \u0027"},{"line_number":581,"context_line":"                              \u0027It might be under very high load or \u0027"},{"line_number":582,"context_line":"                              \u0027just inoperable\u0027),"},{"line_number":583,"context_line":"                          self.sync_routers_chunk_size)"},{"line_number":584,"context_line":"            raise"},{"line_number":585,"context_line":"        except oslo_messaging.MessagingException:"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a7e1126_20ac64ec","line":582,"range":{"start_line":581,"start_character":49,"end_line":582,"end_character":47},"in_reply_to":"3a7e1126_b912f107","updated":"2015-12-18 07:57:57.000000000","message":"Not sure I got your point, which chances should it leave for other agents?","commit_id":"75ec308e4b56df2a8765811d9344bffad451b0c8"},{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"03f53dace8f863ff720f3d111c7eaca1556d1ee3","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"    def get_router_ids(self, context):"},{"line_number":106,"context_line":"        \"\"\"Make a remote process call to retrieve scheduled routers ids.\"\"\""},{"line_number":107,"context_line":"        cctxt \u003d self.client.prepare(version\u003d\u00271.8\u0027)"},{"line_number":108,"context_line":"        return cctxt.call(context, \u0027get_router_ids\u0027, host\u003dself.host)"},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"    def get_external_network_id(self, context):"}],"source_content_type":"text/x-python","patch_set":13,"id":"fa69d971_7159eefd","line":107,"updated":"2016-01-04 10:13:58.000000000","message":"this should be 1.9","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"9801366869ebc913eb1a47c2afb6baaffb5c02be","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"    def get_router_ids(self, context):"},{"line_number":106,"context_line":"        \"\"\"Make a remote process call to retrieve scheduled routers ids.\"\"\""},{"line_number":107,"context_line":"        cctxt \u003d self.client.prepare(version\u003d\u00271.8\u0027)"},{"line_number":108,"context_line":"        return cctxt.call(context, \u0027get_router_ids\u0027, host\u003dself.host)"},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"    def get_external_network_id(self, context):"}],"source_content_type":"text/x-python","patch_set":13,"id":"fa69d971_78130a0e","line":107,"in_reply_to":"fa69d971_7159eefd","updated":"2016-01-04 13:26:17.000000000","message":"right, thanks","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"e7e6b61f404a5997509472b74e837c0b6d9c4cb2","unresolved":false,"context_lines":[{"line_number":574,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":575,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":576,"context_line":"                self.sync_routers_chunk_size \u003d max("},{"line_number":577,"context_line":"                    self.sync_routers_chunk_size / 2,"},{"line_number":578,"context_line":"                    SYNC_ROUTERS_MIN_CHUNK_SIZE)"},{"line_number":579,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":580,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a7b0d38_6898dec0","line":577,"updated":"2015-12-29 03:41:52.000000000","message":"If we are not going to make SYNC_ROUTERS_MAX_CHUNK_SIZE and SYNC_ROUTERS_MIN_CHUNK_SIZE configurable to hardcode, why we still need max(..) here? \n\nIt will be always that:\n  sysc_routers_chunk_size \u003e\u003d SYNC_ROUTERS_MIN_CHUNK_SIZE","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"30bb9401c806898f9ccfdb641f0712ca42a69d47","unresolved":false,"context_lines":[{"line_number":574,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":575,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":576,"context_line":"                self.sync_routers_chunk_size \u003d max("},{"line_number":577,"context_line":"                    self.sync_routers_chunk_size / 2,"},{"line_number":578,"context_line":"                    SYNC_ROUTERS_MIN_CHUNK_SIZE)"},{"line_number":579,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":580,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a7b0d38_7743d742","line":577,"in_reply_to":"1a7b0d38_6898dec0","updated":"2015-12-29 09:03:15.000000000","message":"Not always. If we started to increase chunk size from SYNC_ROUTERS_MIN_CHUNK_SIZE (at #598) we may at some point have 96. Then if have 2 consecutive timeouts: 1) 96 / 2 \u003d 48; 2) 48 / 2 \u003d 24. But we don\u0027t want chunk size to be \u003c SYNC_ROUTERS_MIN_CHUNK_SIZE so we use max(). Does it make sense?","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"bd85fc9497911483ee4b3afef4041adfbeb86f8e","unresolved":false,"context_lines":[{"line_number":574,"context_line":"        except oslo_messaging.MessagingTimeout:"},{"line_number":575,"context_line":"            if self.sync_routers_chunk_size \u003e SYNC_ROUTERS_MIN_CHUNK_SIZE:"},{"line_number":576,"context_line":"                self.sync_routers_chunk_size \u003d max("},{"line_number":577,"context_line":"                    self.sync_routers_chunk_size / 2,"},{"line_number":578,"context_line":"                    SYNC_ROUTERS_MIN_CHUNK_SIZE)"},{"line_number":579,"context_line":"                LOG.error(_LE(\u0027Server failed to return info for routers in \u0027"},{"line_number":580,"context_line":"                              \u0027required time, decreasing chunk size to: %s\u0027),"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a7b0d38_0d06041d","line":577,"in_reply_to":"1a7b0d38_7743d742","updated":"2015-12-29 10:05:03.000000000","message":"I get your point, it\u0027s clear enough now, thanks :)","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"},{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"e7e6b61f404a5997509472b74e837c0b6d9c4cb2","unresolved":false,"context_lines":[{"line_number":596,"context_line":"        if self.sync_routers_chunk_size \u003c SYNC_ROUTERS_MAX_CHUNK_SIZE:"},{"line_number":597,"context_line":"            self.sync_routers_chunk_size \u003d min("},{"line_number":598,"context_line":"                self.sync_routers_chunk_size + SYNC_ROUTERS_MIN_CHUNK_SIZE,"},{"line_number":599,"context_line":"                SYNC_ROUTERS_MAX_CHUNK_SIZE)"},{"line_number":600,"context_line":""},{"line_number":601,"context_line":"        # Delete routers that have disappeared since the last sync"},{"line_number":602,"context_line":"        for router_id in prev_router_ids - curr_router_ids:"}],"source_content_type":"text/x-python","patch_set":13,"id":"1a7b0d38_68df7e69","line":599,"updated":"2015-12-29 03:41:52.000000000","message":"This is cool to me, thanks.","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"e30d508e8e07c98b4a7eb0fa6bd0e614f5d93206","unresolved":false,"context_lines":[{"line_number":592,"context_line":""},{"line_number":593,"context_line":"        self.fullsync \u003d False"},{"line_number":594,"context_line":"        LOG.debug(\"periodic_sync_routers_task successfully completed\")"},{"line_number":595,"context_line":"        # adjust chunk size after successful sync"},{"line_number":596,"context_line":"        if self.sync_routers_chunk_size \u003c SYNC_ROUTERS_MAX_CHUNK_SIZE:"},{"line_number":597,"context_line":"            self.sync_routers_chunk_size \u003d min("},{"line_number":598,"context_line":"                self.sync_routers_chunk_size + SYNC_ROUTERS_MIN_CHUNK_SIZE,"}],"source_content_type":"text/x-python","patch_set":14,"id":"da6ed579_24cb2274","line":595,"updated":"2016-01-13 17:08:41.000000000","message":"So my only thought on adjustment is that I don\u0027t know if doing it after every success is a good thing.  Perhaps after some number of syncs so it stays mostly stable.  But I think it can be done in a follow-on since the heuristic on how to best do the success/failure adjustment could be done in it\u0027s own class and shared between this and the DHCP change.","commit_id":"0e97feb0f30bc0ef6f4fe041cb41b7aa81042263"}],"neutron/agent/l3/config.py":[{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"8b94930919f6b678b6d17632730d22843a0b9343","unresolved":false,"context_lines":[{"line_number":102,"context_line":"    cfg.IntOpt(\u0027sync_routers_chunk_size\u0027,"},{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a2ca52d_7248485c","line":105,"updated":"2015-10-13 15:43:23.000000000","message":"I wonder if by paginating we\u0027re just masking the lack of proper garbage collection. Besides, what if 200 is \u0027big\u0027 enough for a system to experience the same \u0027fatal memory consumption\u0027?\n\nI wonder if there\u0027s some circular dependency that prevents GC to kick in. I remember we had similar issues on memory ballooning during unit tests. Those were successfully fixed by switching to weak references.","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"9c6ab316629b4d31b6927cbce3871c74c0c74e6b","unresolved":false,"context_lines":[{"line_number":102,"context_line":"    cfg.IntOpt(\u0027sync_routers_chunk_size\u0027,"},{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a2ca52d_d0c7b810","line":105,"in_reply_to":"5a2ca52d_7248485c","updated":"2015-10-14 07:54:22.000000000","message":"\u003e\u003e I wonder if by paginating we\u0027re just masking the lack of proper garbage collection.\n\nI suspect the problem might lie at messaging level: when server is ready with a (giant) response the agent already timeouted and sent another request for routers info, so we have kind of cascading effect. The server might do several attempts to send the response (with some timout) with no success before throwing it off, so memory is used all that time. Anyway even if we fix garbage colection at messaging (or any other) level we\u0027ll still have a problem with heavy requests which server fails to process in required time. And I see it as the main problem.\n\n\u003e\u003e Besides, what if 200 is \u0027big\u0027 enough for a system to experience the same \u0027fatal memory consumption\u0027\n\nso that\u0027s why this is configured parameter ;)\n\n\u003e\u003e I wonder if there\u0027s some circular dependency that prevents GC to kick in. I remember we had similar issues on memory ballooning during unit tests. Those were successfully fixed by switching to weak references.\n\nI see the problem comes from the db request: \n\nSELECT routerports.router_id AS routerports_router_id, routerports.port_id AS routerports_port_id, routerports.port_type AS routerports_port_type, ipallocations_1.port_id AS ipallo\ncations_1_port_id, ipallocations_1.ip_address AS ipallocations_1_ip_address, ipallocations_1.subnet_id AS ipallocations_1_subnet_id, ipallocations_1.network_id AS ipallocations_1_network_id, ports_1.tenant\n_id AS por....\n\nwhich has list of parameters with ~1000 items when agent requests info for all router it handles. The response size is about 2G, after reading the response process starts to call mmap() retrieving more memory.\noverall the processing takes several minutes.","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"b1966e4a679b1291fd9678344824f313a0b9ea9d","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a2ca52d_124a3484","line":106,"updated":"2015-10-13 15:52:33.000000000","message":"I am personally skeptical of these types of changes (like [1, 2]). I think paginating in this context is a bad idea. Sprinkling such fixes all over the place to address \u0027scale\u0027 concerns is problematic and it sounds to me that Timeout issues on RPC calls should be addressed differently, for instance by splitting the RPC call into two RPC casts, totally decoupling the parties in need of communicating.\n\nThis is the 3rd fix using this approach. It looks like I am the only one seeing that these are bad :(\n\n [1] https://review.openstack.org/#/c/163594/\n [2] https://review.openstack.org/#/c/219573/","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"66d1682da6626be7c8415ae7665d0031227f800d","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a29b11f_23bfa2fc","line":106,"in_reply_to":"3a29b11f_22bb32e7","updated":"2015-10-20 08:34:09.000000000","message":"\u003e\u003e You don\u0027t start processing the routers until you fetched the all list, so how is the agent starting the processing any sooner?\n\nYep, not with this patch to make it as simple as possible, this optimization can still be applied once tested.\n\n\u003e\u003e Pagination is bad because it can add chattiness on the bus/db, which may as well have side effects.\n\nDVR increased chattiness greatly and one of the most noticable side effects was increased load on the server. This patch tries to smooth the load at scale and only in case agent is resyncing which normally should not happen very often. I\u0027d say that chattiness added by this patch is negligible comparing to the one added by DVR.\n\n\u003e\u003e Besides the increased delay may lead the agent to deal with routers that may have well disappear by the time it gets to process them. The problem exists even without pagination, but pagination exacerbates it. I don\u0027t see how you address it here though.\n\nTests show that server handles chunks faster than it handles one big request so I\u0027m not sure about the increased delay. Anyway AFAIK RouterProcessingQueue should handle the case when router is deleted by the time agent starts to process it after fullsync. Also we have a functional test for this case: test_periodic_sync_routers_task_routers_deleted_while_agent_sync.\n\n\u003e\u003e Besides, the choice of the magic constant has a negative effect on usability and operability of the system. Should the value need to change or be tuned, it requires a server bump, and most admins would have no idea what number to come up with.\n\n200 was tested and proved working. I don\u0027t think admins should worry about it until they face the problem with memory consumption, and once they face it - they have a built-in mechanism to fix the problem by descreasing this config parameter. IOW I see 200 better than nothing - It doesn\u0027t affect agent workflow much and should be transparent for admins/users.","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"7d551570c273e5ea987dc3392a2d34427d723598","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a29b11f_435cf7ec","line":106,"in_reply_to":"3a29b11f_23bfa2fc","updated":"2015-10-20 20:52:33.000000000","message":"It feels like your last argument is the one that says it all: it reads to me like so \u003d\u003e \u0027let\u0027s sweep the issue under the rug and we\u0027ll worry about side effects later, if and when they come\u0027. Besides how can you really claim that 200 was tested and proved working? Sure it works in your environment, but that may be such a constrained testbed...\n\nI appreciate the need for your incremental fix, but I honestly believe that albeit easier to handle and understand, this approach you propose (and has been proposed) is a symptom of a design issue that we fail to acknowledge and solve it in a more holistic fashion. Sprinkling pagination patches all over the server/agent communication framework is not the right answer to this scale problem, especially if we hard-code the chunk size and defer it to the admin to figure out.\n\nMy suggestion in the past has been: a) let\u0027s figure out at least a way to make the chunk size dynamic so that the admin doesn\u0027t have to worry about this, should his/her system hits the ceiling, and b) (alternatively) let\u0027s solve the problem by decoupling agent and server entirely by moving away from the rpc.call pattern.\n\nEither way, you can continue to ignore me :)","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"2733c80e2e6d1101bf4302ee87581dd854623f59","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a29b11f_b7406935","line":106,"in_reply_to":"3a29b11f_2ad179fa","updated":"2015-10-22 16:33:56.000000000","message":"well but you are...you are dismissing all my points because apparently they are too difficult to tackle...and I am forcing myself very hard not to give you a -2...but my patience has a limit :)","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"e82a0c015ebb5081fdb92f1f96e3f378f9405a92","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a29b11f_2ad179fa","line":106,"in_reply_to":"3a29b11f_435cf7ec","updated":"2015-10-21 09:57:39.000000000","message":"I\u0027m not ignoring, really, and I\u0027m sorry if I made that impression :-(\n\nI\u0027m not a big fan of magic numbers and magic config parameters either. In this case however I failed to find a sensible way to get chunk size dynamically and appreciate any ideas on this.\n\nP.S. I\u0027ve just checket all rugs at home and can say that sweeping under the rug is not among my habbits! ;)","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"1341e44fe513c385f8b52664103e757c5104891c","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a29b11f_a2d9d4f4","line":106,"in_reply_to":"3a29b11f_b7406935","updated":"2015-10-23 08:20:02.000000000","message":"We are discussing how to better solve the issue, right? I am trying to not miss any of your comments, if that means \"ignoring\" to you then I\u0027m wondering what is not? You can use your -2 if you feel it\u0027s a completely wrong approach, this is your right as core reviewer and PTL ;)","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"9c6ab316629b4d31b6927cbce3871c74c0c74e6b","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a2ca52d_bbb69dda","line":106,"in_reply_to":"5a2ca52d_124a3484","updated":"2015-10-14 07:54:22.000000000","message":"Well, I see paginating as a cheap and simple way to smooth the load on neutron server and agent, to avoid heavy db requests and big RPC messages. You can see the patch is very straightforward. Of course we can go with a refactoring of server-agent RPC communications (like two casts instead of call, though I prefer the way where agent starts processing as early as possible, which is another benefit of paginating) and some tweaks on server side to avoid big db requests but I\u0027m afraid that will be much more changes. Can you please give more details on cons of paginating that you see?","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"},{"author":{"_account_id":748,"name":"Armando Migliaccio","email":"armamig@gmail.com","username":"armando-migliaccio"},"change_message_id":"42aa54d1545ac1c058c28bc54cc72e30b72ed3f2","unresolved":false,"context_lines":[{"line_number":103,"context_line":"               default\u003d200,"},{"line_number":104,"context_line":"               help\u003d_(\u0027Number of routers to fetch from server at a time on \u0027"},{"line_number":105,"context_line":"                      \u0027resync. Needed to prevent fatal memory consumption on \u0027"},{"line_number":106,"context_line":"                      \u0027server side.\u0027)),"},{"line_number":107,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a29b11f_22bb32e7","line":106,"in_reply_to":"5a2ca52d_bbb69dda","updated":"2015-10-19 20:16:49.000000000","message":"You don\u0027t start processing the routers until you fetched the all list, so how is the agent starting the processing any sooner?\n\nPagination is bad because it can add chattiness on the bus/db, which may as well have side effects.\n\nBesides the increased delay may lead the agent to deal with routers that may have well disappear by the time it gets to process them. The problem exists even without pagination, but pagination exacerbates it. I don\u0027t see how you address it here though.\n\nBesides, the choice of the magic constant has a negative effect on usability and operability of the system. Should the value need to change or be tuned, it requires a server bump, and most admins would have no idea what number to come up with.","commit_id":"172df45135566356e38cba76fe59c77ddd0c781f"}],"neutron/api/rpc/handlers/l3_rpc.py":[{"author":{"_account_id":1653,"name":"garyk","email":"gkotton@vmware.com","username":"garyk"},"change_message_id":"03f53dace8f863ff720f3d111c7eaca1556d1ee3","unresolved":false,"context_lines":[{"line_number":64,"context_line":""},{"line_number":65,"context_line":"    def get_router_ids(self, context, host):"},{"line_number":66,"context_line":"        \"\"\"Returns IDs of routers scheduled to l3 agent on \u003chost\u003e\"\"\""},{"line_number":67,"context_line":"        return self.l3plugin.list_router_ids_on_host(context, host)"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"    @db_api.retry_db_errors"},{"line_number":70,"context_line":"    def sync_routers(self, context, **kwargs):"}],"source_content_type":"text/x-python","patch_set":13,"id":"fa69d971_51e4ea0b","line":67,"updated":"2016-01-04 10:13:58.000000000","message":"say you have 1024 routers defined on the host. how do ensure that all of the routers are returned? should you not have a last accessed?","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"9801366869ebc913eb1a47c2afb6baaffb5c02be","unresolved":false,"context_lines":[{"line_number":64,"context_line":""},{"line_number":65,"context_line":"    def get_router_ids(self, context, host):"},{"line_number":66,"context_line":"        \"\"\"Returns IDs of routers scheduled to l3 agent on \u003chost\u003e\"\"\""},{"line_number":67,"context_line":"        return self.l3plugin.list_router_ids_on_host(context, host)"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"    @db_api.retry_db_errors"},{"line_number":70,"context_line":"    def sync_routers(self, context, **kwargs):"}],"source_content_type":"text/x-python","patch_set":13,"id":"fa69d971_9806fecc","line":67,"in_reply_to":"fa69d971_51e4ea0b","updated":"2016-01-04 13:26:17.000000000","message":"Sorry, didn\u0027t quite get the question. This call should just return a list of ids of all routers hosted on \u003chost\u003e. Can you please clarify your concern?","commit_id":"df2db5d0c174e5542cb8a027be19c204ca2819fd"}],"neutron/db/l3_agentschedulers_db.py":[{"author":{"_account_id":13667,"name":"ZongKai LI","email":"onionpiece@163.com","username":"lzklibj"},"change_message_id":"edc96b211d78bc73b31019cf41a138cc61766268","unresolved":false,"context_lines":[{"line_number":331,"context_line":"            query \u003d query.filter("},{"line_number":332,"context_line":"                RouterL3AgentBinding.router_id.in_(router_ids))"},{"line_number":333,"context_line":""},{"line_number":334,"context_line":"        return [item[0] for item in query]"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"    def list_active_sync_routers_on_active_l3_agent("},{"line_number":337,"context_line":"            self, context, host, router_ids):"}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_4bf9f94c","line":334,"updated":"2015-11-07 01:48:42.000000000","message":"If list_router_ids_on_host returns ([router_ids], agent), we will no need to call _get_agent_by_type_and_host again in L340-341 again.","commit_id":"f007cf7468ce9bd74bd5be193fe801b65776a2c5"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"27e670d28779f761a9704bb8b19d96e5d1c51ad5","unresolved":false,"context_lines":[{"line_number":331,"context_line":"            query \u003d query.filter("},{"line_number":332,"context_line":"                RouterL3AgentBinding.router_id.in_(router_ids))"},{"line_number":333,"context_line":""},{"line_number":334,"context_line":"        return [item[0] for item in query]"},{"line_number":335,"context_line":""},{"line_number":336,"context_line":"    def list_active_sync_routers_on_active_l3_agent("},{"line_number":337,"context_line":"            self, context, host, router_ids):"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba8a016a_5970620b","line":334,"in_reply_to":"da85f559_4bf9f94c","updated":"2015-11-18 16:56:21.000000000","message":"will fix to not get agent from db twice","commit_id":"f007cf7468ce9bd74bd5be193fe801b65776a2c5"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"9ed30eaa74b0b648df75400ae950132d99a87c38","unresolved":false,"context_lines":[{"line_number":335,"context_line":""},{"line_number":336,"context_line":"    def list_active_sync_routers_on_active_l3_agent("},{"line_number":337,"context_line":"            self, context, host, router_ids):"},{"line_number":338,"context_line":"        router_ids \u003d self.list_router_ids_on_host(context, host, router_ids)"},{"line_number":339,"context_line":"        if router_ids:"},{"line_number":340,"context_line":"            agent \u003d self._get_agent_by_type_and_host("},{"line_number":341,"context_line":"                context, constants.AGENT_TYPE_L3, host)"}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_45f31c74","line":338,"updated":"2015-11-08 07:29:42.000000000","message":"I am concerning the time increase here. It is always compromise between memory consumption and cpu consumption. \n\nFrom my point of view, I can see some improvement here.\n\na) \"_get_active_l3_agent_routers_sync_data\" don\u0027t need \"agent\" to work. So no need to query agent at line 340-341\n\nb) considering using \"router_ids \u003d router_ids or self.list_router_ids_on_host(context, host, router_ids)\"\nat line 338.\n\"list_active_sync_routers_on_active_l3_agent\" is only be called by rpc method. If we can count on that l3 agent will always get its own routers, we don\u0027t need to filter the router_ids again.","commit_id":"f007cf7468ce9bd74bd5be193fe801b65776a2c5"},{"author":{"_account_id":11159,"name":"Hong Hui Xiao","email":"honghui_xiao@yeah.net","username":"HongHuiXiao"},"change_message_id":"50b25ae6f4121c98ca16ed62622237309332ddbe","unresolved":false,"context_lines":[{"line_number":335,"context_line":""},{"line_number":336,"context_line":"    def list_active_sync_routers_on_active_l3_agent("},{"line_number":337,"context_line":"            self, context, host, router_ids):"},{"line_number":338,"context_line":"        router_ids \u003d self.list_router_ids_on_host(context, host, router_ids)"},{"line_number":339,"context_line":"        if router_ids:"},{"line_number":340,"context_line":"            agent \u003d self._get_agent_by_type_and_host("},{"line_number":341,"context_line":"                context, constants.AGENT_TYPE_L3, host)"}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_81bcd4bb","line":338,"in_reply_to":"da85f559_45f31c74","updated":"2015-11-12 07:55:33.000000000","message":"My review is immature, \"_get_active_l3_agent_routers_sync_data\" do need agent at \"L3_DVRsch_db_mixin\". Maybe Zong Kai\u0027s comment is good here. Revert my score....","commit_id":"f007cf7468ce9bd74bd5be193fe801b65776a2c5"}]}
