)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"02728d3ef463dffaae7c2f3fb52b6f2bc66d23e9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"d40c578d_96e670a9","updated":"2023-01-12 20:09:51.000000000","message":"Added some WIP notes for discussion.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"22a0d59128e4dc292288962eda7d57f0623b3f6e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"09a4468b_cdb5339c","updated":"2023-01-13 07:07:49.000000000","message":"Great stuff Dmitrii! I added a few comments in-line spurred by immediate thoughts that appeared while reading it.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b6546da3456ea2cc92ee784760fdcaed5dee0fb2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"bb57c097_abcb4f86","updated":"2023-01-18 08:23:32.000000000","message":"Just linking the lib change:\n\nhttps://review.opendev.org/c/openstack/neutron-lib/+/870887\n\nI\u0027ll submit an update to the spec as well soon.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"345dc5fe804e525d347b9da6dabaa6ff6b352a11","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"158a1547_f3e6b247","updated":"2023-02-14 16:20:01.000000000","message":"-W just to block this feature during this release. Please propose it for 2023.2 cycle. Thanks!","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"0b286004a9c94da07eb7f86d5d86e414f046f88e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"986b30c1_0a31af76","updated":"2023-03-09 15:06:41.000000000","message":"Please, move this spec to 2023.2 directory","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"dd179ff9080d3f3463b8d2693e62ffcffcd6eab4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"9876b904_1fd9901a","updated":"2023-03-02 09:44:12.000000000","message":"Please, propose this spec on top of https://review.opendev.org/c/openstack/neutron-specs/+/876044","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"05cd0597f5b6d37d78ef2f85b9c65def4fdf2fa3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"739e1925_15d7ea60","updated":"2023-02-28 13:19:15.000000000","message":"Sorry for the slow response.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"7556dca4_38af45d7","updated":"2023-02-21 15:12:26.000000000","message":"Thanks for the proposal! Left a few questions.\n\nAlso I\u0027m happy to see that our previous plans - even if we did not get to implement them - are of some use to you.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"003281c8_5819a129","in_reply_to":"7556dca4_38af45d7","updated":"2023-02-22 14:13:16.000000000","message":"Thanks a lot for the prior art and the review - it was really helpful.\n\nI left some answers inline and will make some changes as a result.\n\nhttps://review.opendev.org/c/openstack/neutron/+/873593 - the WIP changes (there are some others besides that one in the same topic).","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"82bac311b360913d22f91af6993795879d0ecce0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"34d86f44_8e89b2c4","updated":"2023-03-21 07:49:51.000000000","message":"++","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"5fc4ce191236d4946be47469d5dc9ba953d700a5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"676bbc5f_21f8969d","updated":"2023-03-20 18:33:12.000000000","message":"Good work, very thorough, only had one nit.","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"ec8cd7191e92913e0cffb207d787516789f60ec4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"ac902d06_5d3f2e8c","updated":"2023-03-27 11:12:48.000000000","message":"Apart from small nits, the spec is almost ready. Ping other reviewers","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"36e629a99d523aec22196d96f823b26c2c85d129","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"90c713fa_4e356a41","updated":"2023-03-29 07:35:46.000000000","message":"Looks ok, thanks for the spec","commit_id":"9763752c73644be92f9a2597c198968d88c1d220"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"fe49971b560729300fc2bf2872cdfcb95f0c2fef","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"784c257f_1b8043ba","updated":"2023-03-30 11:40:52.000000000","message":"Sorry, let\u0027s wait for Slawek","commit_id":"9763752c73644be92f9a2597c198968d88c1d220"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"425e77202399c8eda59101993aaaf4dc6737ce13","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"8fd42ed7_2fe229a2","in_reply_to":"784c257f_1b8043ba","updated":"2023-03-30 12:47:28.000000000","message":"I\u0027m ok with this","commit_id":"9763752c73644be92f9a2597c198968d88c1d220"}],"specs/2023.1/active-active-l3-gateway-with-multihoming.rst":[{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"22a0d59128e4dc292288962eda7d57f0623b3f6e","unresolved":true,"context_lines":[{"line_number":21,"context_line":"Problem Description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"}],"source_content_type":"text/x-rst","patch_set":1,"id":"f093a66f_189e32cd","line":24,"range":{"start_line":24,"start_character":29,"end_line":24,"end_character":52},"updated":"2023-01-13 07:07:49.000000000","message":"Add a bullet for L3 based resilience?\n\nI.e. this is also useful for making use of multiple upstream gateways within a DC network without the need for a shared L2 between the gateways. (L3+BFD replacing L2+VRRP)","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":35499,"name":"alisafari","email":"alisafar1212@gmail.com","username":"alisafari","status":"Software Engineer"},"change_message_id":"00517578c60bf668a0d5063e7b647bc687a7cb4b","unresolved":true,"context_lines":[{"line_number":21,"context_line":"Problem Description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"}],"source_content_type":"text/x-rst","patch_set":1,"id":"13583b59_623f5c17","line":24,"range":{"start_line":24,"start_character":0,"end_line":24,"end_character":4},"updated":"2023-01-18 14:48:01.000000000","message":"What is your solution for gateways which only provides connectivity to a small private subnet outside of neutron network (inside of datacenter network).\n\nThis would require us to provide an API to overwrite default route (0.0.0.0/0) on routers gateway port.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"38b295d465d4044329447e91cd57f8e3986f0ef6","unresolved":false,"context_lines":[{"line_number":21,"context_line":"Problem Description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3e855754_34c47f4d","line":24,"range":{"start_line":24,"start_character":29,"end_line":24,"end_character":52},"in_reply_to":"11704456_d97bf60c","updated":"2023-01-20 13:56:00.000000000","message":"Done","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"46a00745e0167b703a92b6168c78a99cfb09cb62","unresolved":true,"context_lines":[{"line_number":21,"context_line":"Problem Description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"}],"source_content_type":"text/x-rst","patch_set":1,"id":"46db8420_5348cd96","line":24,"range":{"start_line":24,"start_character":0,"end_line":24,"end_character":4},"in_reply_to":"13583b59_623f5c17","updated":"2023-01-20 12:12:00.000000000","message":"Are you talking about something like this?\n\nphysical DC network (private) \u003d\u003e special GW port (one of the router LRPs) \u003d\u003e OVN router \u003d\u003e normal GW port \u003d\u003e external DC network\n\nI think you could avoid specifying a default gateway on a subnet for the private DC network in Neutron to avoid having an inferred default route on this subnet from being added to the OVN router\u0027s routing table.\n\nCould you explain the need to override the default route a little bit more?","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":35499,"name":"alisafari","email":"alisafar1212@gmail.com","username":"alisafari","status":"Software Engineer"},"change_message_id":"5649f984a8925c0a1e2459e5e64e172a8129cf51","unresolved":true,"context_lines":[{"line_number":21,"context_line":"Problem Description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"}],"source_content_type":"text/x-rst","patch_set":1,"id":"b9c92e91_341f382d","line":24,"range":{"start_line":24,"start_character":0,"end_line":24,"end_character":4},"in_reply_to":"46db8420_5348cd96","updated":"2023-01-26 15:50:43.000000000","message":"Well our special private DC networks gateway routes traffic to multiple other private network in/out of our DC. This will require us to be able to overwrite default routes.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":21,"context_line":"Problem Description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"}],"source_content_type":"text/x-rst","patch_set":1,"id":"129bf31d_cb4bbea5","line":24,"range":{"start_line":24,"start_character":0,"end_line":24,"end_character":4},"in_reply_to":"b9c92e91_341f382d","updated":"2023-02-21 15:12:26.000000000","message":"These past (unimplemented) plans may give an idea if we ever need to overwrite implicitly added routes:\nhttps://opendev.org/openstack/neutron-specs/src/branch/master/specs/xena/explicit-default-routes.rst","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"a1e94036f99a98a3dc9e2f4c1cd8084a8617ff8a","unresolved":true,"context_lines":[{"line_number":21,"context_line":"Problem Description"},{"line_number":22,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"}],"source_content_type":"text/x-rst","patch_set":1,"id":"11704456_d97bf60c","line":24,"range":{"start_line":24,"start_character":29,"end_line":24,"end_character":52},"in_reply_to":"f093a66f_189e32cd","updated":"2023-01-13 13:00:47.000000000","message":"Yes, I suppose that deserves a separate bullet.\n\nI thought the \"independent network paths\" part covers it but I could expand it as you suggest.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"22a0d59128e4dc292288962eda7d57f0623b3f6e","unresolved":true,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"},{"line_number":28,"context_line":"  different ISPs)."},{"line_number":29,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f41f8bd_cd9992a4","line":26,"updated":"2023-01-13 07:07:49.000000000","message":"Perhaps elaborate this point a bit because it is not obvious that this proposal allows load sharing across both network components outside the cloud and gateway chassis in the cloud.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"a1e94036f99a98a3dc9e2f4c1cd8084a8617ff8a","unresolved":true,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"},{"line_number":28,"context_line":"  different ISPs)."},{"line_number":29,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"d4d75c70_c1d7fc6f","line":26,"in_reply_to":"1f41f8bd_cd9992a4","updated":"2023-01-13 13:00:47.000000000","message":"Indeed, we could have multiple OVN-side gateways sharing the load while pointing to the same next-hop or we could have both multiple OVN GWs and multiple next-hops so that the load is shared at multiple levels.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"38b295d465d4044329447e91cd57f8e3986f0ef6","unresolved":false,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Some network designs include multiple L3 gateways to:"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"* Share the load across different gateways;"},{"line_number":27,"context_line":"* Provide independent network paths for the north-south direction (e.g. via"},{"line_number":28,"context_line":"  different ISPs)."},{"line_number":29,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"63986e6c_c4e93050","line":26,"in_reply_to":"d4d75c70_c1d7fc6f","updated":"2023-01-20 13:56:00.000000000","message":"Done","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"22a0d59128e4dc292288962eda7d57f0623b3f6e","unresolved":true,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Conntrack can be utilized to avoid responses generated by instances to go via"},{"line_number":65,"context_line":"the route different from the one the request came in on in presence of ECMP"},{"line_number":66,"context_line":"routes. OVN has support for making the reply traffic take the symmetric path."},{"line_number":67,"context_line":"This can be configured by utilizing the `options column`_ in the logical router"},{"line_number":68,"context_line":"static routes table in OVN which allows configuring `ECMP symmetric reply`_ by"},{"line_number":69,"context_line":"setting ``ecmp_symmetric_reply`` option to ``true``."}],"source_content_type":"text/x-rst","patch_set":1,"id":"b4e5df7a_47f3eb56","line":66,"range":{"start_line":66,"start_character":16,"end_line":66,"end_character":23},"updated":"2023-01-13 07:07:49.000000000","message":"There are some restrictions [0] for the use of this option which may not fit with the way Neutron currently lay out logical routers.\n\nBut we should definitively plan for how we would expose/consume the option given the right topology.\n\n0: https://github.com/ovn-org/ovn/blob/c61a0e1d03da93ed47daf31892e9b0f69982efee/northd/northd.c#L9955-L9964","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"a1e94036f99a98a3dc9e2f4c1cd8084a8617ff8a","unresolved":true,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Conntrack can be utilized to avoid responses generated by instances to go via"},{"line_number":65,"context_line":"the route different from the one the request came in on in presence of ECMP"},{"line_number":66,"context_line":"routes. OVN has support for making the reply traffic take the symmetric path."},{"line_number":67,"context_line":"This can be configured by utilizing the `options column`_ in the logical router"},{"line_number":68,"context_line":"static routes table in OVN which allows configuring `ECMP symmetric reply`_ by"},{"line_number":69,"context_line":"setting ``ecmp_symmetric_reply`` option to ``true``."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bd87dad9_a2fa84be","line":66,"range":{"start_line":66,"start_character":16,"end_line":66,"end_character":23},"in_reply_to":"b4e5df7a_47f3eb56","updated":"2023-01-13 13:00:47.000000000","message":"Thanks for pointing this one out - I missed this one.\n\nI wonder what the restriction is at the OVN level as it would be nice to have this feature enabled in the distributed case as well. Though the load-sharing benefit would have less effect as it would only be relevant for instances that don\u0027t have floating IPs (for which load-sharing would happen naturally). For the FIP + distributed case just using ECMP default routes would be enough to get some load-sharing.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"6d55b5c7ae4f53ac52d7e0441e28fe7941711bf5","unresolved":true,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Conntrack can be utilized to avoid responses generated by instances to go via"},{"line_number":65,"context_line":"the route different from the one the request came in on in presence of ECMP"},{"line_number":66,"context_line":"routes. OVN has support for making the reply traffic take the symmetric path."},{"line_number":67,"context_line":"This can be configured by utilizing the `options column`_ in the logical router"},{"line_number":68,"context_line":"static routes table in OVN which allows configuring `ECMP symmetric reply`_ by"},{"line_number":69,"context_line":"setting ``ecmp_symmetric_reply`` option to ``true``."}],"source_content_type":"text/x-rst","patch_set":1,"id":"e217957c_9cd432b3","line":66,"range":{"start_line":66,"start_character":16,"end_line":66,"end_character":23},"in_reply_to":"bd87dad9_a2fa84be","updated":"2023-01-13 13:24:42.000000000","message":"The load sharing also has effect for the unNATted project network case.\n\nWe know about some large production clusters that do that today.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"38b295d465d4044329447e91cd57f8e3986f0ef6","unresolved":false,"context_lines":[{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Conntrack can be utilized to avoid responses generated by instances to go via"},{"line_number":65,"context_line":"the route different from the one the request came in on in presence of ECMP"},{"line_number":66,"context_line":"routes. OVN has support for making the reply traffic take the symmetric path."},{"line_number":67,"context_line":"This can be configured by utilizing the `options column`_ in the logical router"},{"line_number":68,"context_line":"static routes table in OVN which allows configuring `ECMP symmetric reply`_ by"},{"line_number":69,"context_line":"setting ``ecmp_symmetric_reply`` option to ``true``."}],"source_content_type":"text/x-rst","patch_set":1,"id":"2cb559da_641b47af","line":66,"range":{"start_line":66,"start_character":16,"end_line":66,"end_character":23},"in_reply_to":"e217957c_9cd432b3","updated":"2023-01-20 13:56:00.000000000","message":"Added a reference that this spec is not going to include the distributed routing + ECMP symmetric reply support.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"02728d3ef463dffaae7c2f3fb52b6f2bc66d23e9","unresolved":true,"context_lines":[{"line_number":118,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":119,"context_line":"routes should be used or not (if the L3 service plugin supports them). In the"},{"line_number":120,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":121,"context_line":"can be a reasonable default."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":124,"context_line":"used for checking whether the next-hop of a default route is reachable (if"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ea2f042d_b1652524","line":121,"range":{"start_line":121,"start_character":0,"end_line":121,"end_character":27},"updated":"2023-01-12 20:09:51.000000000","message":"We can assume that or it could also be configurable at the router level with an extra option like `enable_default_route_ecmp_symmetric_reply`.\n\nThe problem here is if we add to many options that won\u0027t frequently be used we will just confuse the operator.\n\nI\u0027d be curious to hear any feedback about this since hard-coding this default is a trade-off.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":118,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":119,"context_line":"routes should be used or not (if the L3 service plugin supports them). In the"},{"line_number":120,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":121,"context_line":"can be a reasonable default."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":124,"context_line":"used for checking whether the next-hop of a default route is reachable (if"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7b6c8dcf_290e465b","line":121,"range":{"start_line":121,"start_character":0,"end_line":121,"end_character":27},"in_reply_to":"3b2bd1ec_1300bc8a","updated":"2023-02-10 14:04:48.000000000","message":"Agreed, I think having a separate option for it could be a reasonable approach.\n\nI didn\u0027t want to overload operators with various options but since it\u0027s support is limited to some use-cases I think it\u0027s necessary to have an option.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"0b286004a9c94da07eb7f86d5d86e414f046f88e","unresolved":true,"context_lines":[{"line_number":118,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":119,"context_line":"routes should be used or not (if the L3 service plugin supports them). In the"},{"line_number":120,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":121,"context_line":"can be a reasonable default."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":124,"context_line":"used for checking whether the next-hop of a default route is reachable (if"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5839c58f_9e1cac24","line":121,"range":{"start_line":121,"start_character":0,"end_line":121,"end_character":27},"in_reply_to":"7b6c8dcf_290e465b","updated":"2023-03-09 15:06:41.000000000","message":"And why not to enable this when more than one GW is added to the router?","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":118,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":119,"context_line":"routes should be used or not (if the L3 service plugin supports them). In the"},{"line_number":120,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":121,"context_line":"can be a reasonable default."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":124,"context_line":"used for checking whether the next-hop of a default route is reachable (if"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3b2bd1ec_1300bc8a","line":121,"range":{"start_line":121,"start_character":0,"end_line":121,"end_character":27},"in_reply_to":"94854109_5dae0873","updated":"2023-02-09 14:59:46.000000000","message":"I vote for some optional behaviour for this even from API, or from config in worst case, but perhaps I am too cautious, and want to control everything","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"a1e94036f99a98a3dc9e2f4c1cd8084a8617ff8a","unresolved":true,"context_lines":[{"line_number":118,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":119,"context_line":"routes should be used or not (if the L3 service plugin supports them). In the"},{"line_number":120,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":121,"context_line":"can be a reasonable default."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":124,"context_line":"used for checking whether the next-hop of a default route is reachable (if"}],"source_content_type":"text/x-rst","patch_set":1,"id":"94854109_5dae0873","line":121,"range":{"start_line":121,"start_character":0,"end_line":121,"end_character":27},"in_reply_to":"ea2f042d_b1652524","updated":"2023-01-13 13:00:47.000000000","message":"Considering it only works for non-distributed OVN routers we should also make sure to have proper logic Neutron to account for that.\n\nhttps://review.opendev.org/c/openstack/neutron-specs/+/870030/comments/b4e5df7a_47f3eb56\n\nOr fix OVN itself to handle this option for distributed routers too.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"02728d3ef463dffaae7c2f3fb52b6f2bc66d23e9","unresolved":true,"context_lines":[{"line_number":144,"context_line":"associations, Neutron\u0027s behavior should be to use the next hop of the default"},{"line_number":145,"context_line":"route as a ``dst_ip``."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"The BFD table would thus include the following fields (see the description from"},{"line_number":148,"context_line":"`ovn-nb docs`_ for the BFD table for more information):"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* ``dst_ip`` - a BFD peer IP address;"}],"source_content_type":"text/x-rst","patch_set":1,"id":"77b3af73_f42f0430","line":147,"range":{"start_line":147,"start_character":37,"end_line":147,"end_character":53},"updated":"2023-01-12 20:09:51.000000000","message":"The API for managing BFD needs more thought in this spec.\n\nManually associating those sessions with each route or router can be tedious for an operator.\n\nAt the same time, creating too many BFD sessions could be avoided as well if the gateway device could specify a single BFD peer IP for multiple subnets: that way multiple routes could reuse the same BFD record and its settings.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"38b295d465d4044329447e91cd57f8e3986f0ef6","unresolved":true,"context_lines":[{"line_number":144,"context_line":"associations, Neutron\u0027s behavior should be to use the next hop of the default"},{"line_number":145,"context_line":"route as a ``dst_ip``."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"The BFD table would thus include the following fields (see the description from"},{"line_number":148,"context_line":"`ovn-nb docs`_ for the BFD table for more information):"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* ``dst_ip`` - a BFD peer IP address;"}],"source_content_type":"text/x-rst","patch_set":1,"id":"437ea3ed_7430d2c2","line":147,"range":{"start_line":147,"start_character":37,"end_line":147,"end_character":53},"in_reply_to":"77b3af73_f42f0430","updated":"2023-01-20 13:56:00.000000000","message":"We\u0027ll have a minimal way to specify the need to use BFD with default settings in this spec for now.\n\nFollow-on work might include implementing a full BFD spec:\nhttps://specs.openstack.org/openstack/neutron-specs/specs/xena/bfd_support.html","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"02728d3ef463dffaae7c2f3fb52b6f2bc66d23e9","unresolved":true,"context_lines":[{"line_number":233,"context_line":"The order of the rest of the list is ignored."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Duplicates in the list (that is multiple external gateways with the same"},{"line_number":236,"context_line":"``network_id``) are allowed: in that case multiple gateway ports will be"},{"line_number":237,"context_line":"attached to the same network (this can be used to have the active-active setup"},{"line_number":238,"context_line":"when external gateways are available on the same network)."},{"line_number":239,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"de0e543c_8b9c7826","line":236,"range":{"start_line":236,"start_character":16,"end_line":236,"end_character":27},"updated":"2023-01-12 20:09:51.000000000","message":"This is an important difference from the previous multiple-external-gateways spec.\n\nBringing an analogy with the L3-agent based implementation, this is much like having multiple snat-\u003crouter-id\u003e namespaces across multiple nodes with multiple gateway IPs present in them at the same time (compared to the typical L3HA behavior of one VIP jumping around those namespaces on failover).","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"fb186f57fb708d7d220a9912918e8aef34c3eae4","unresolved":true,"context_lines":[{"line_number":233,"context_line":"The order of the rest of the list is ignored."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Duplicates in the list (that is multiple external gateways with the same"},{"line_number":236,"context_line":"``network_id``) are allowed: in that case multiple gateway ports will be"},{"line_number":237,"context_line":"attached to the same network (this can be used to have the active-active setup"},{"line_number":238,"context_line":"when external gateways are available on the same network)."},{"line_number":239,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"8ca338a3_a90ac8d1","line":236,"range":{"start_line":236,"start_character":16,"end_line":236,"end_character":27},"in_reply_to":"26c6f126_22f80186","updated":"2023-02-02 15:20:30.000000000","message":"There is an option of matching on a combination of a network_id and external_fixed_ips assigned after gateway ports get attached.\n\nThis will require an operator to specify the concrete IPs in the request but:\n\n1) will be explicit\n2) will not require an API change from what is proposed.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":true,"context_lines":[{"line_number":233,"context_line":"The order of the rest of the list is ignored."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Duplicates in the list (that is multiple external gateways with the same"},{"line_number":236,"context_line":"``network_id``) are allowed: in that case multiple gateway ports will be"},{"line_number":237,"context_line":"attached to the same network (this can be used to have the active-active setup"},{"line_number":238,"context_line":"when external gateways are available on the same network)."},{"line_number":239,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"787be3c9_eaf54281","line":236,"range":{"start_line":236,"start_character":16,"end_line":236,"end_character":27},"in_reply_to":"2c11a7e5_b8b9e9ae","updated":"2023-02-22 14:13:16.000000000","message":"I ended up going after the `network_id + external_fixed_ip` matching route.\n\nFor updates it wasn\u0027t easy but with some matching it\u0027s possible:\n\nhttps://review.opendev.org/c/openstack/neutron/+/873593/16/neutron/db/l3_extra_gws_db.py#207\nhttps://review.opendev.org/c/openstack/neutron/+/873593/16/neutron/db/l3_extra_gws_db.py#409\n\n\nThe 3-rd option above is problematic for policy reasons as the gateway port does not have a tenant ID specified and so working with it requires an elevated context:\n\nhttps://github.com/openstack/neutron/blob/24152f2019784fe684bb7b439b78e7265c6ffa08/neutron/db/l3_db.py#L345\n\nhttps://github.com/openstack/neutron-lib/blob/23fa6b3c2f46be6de8523355c786a682fd4419ef/neutron_lib/context.py#L116-L124","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"05cd0597f5b6d37d78ef2f85b9c65def4fdf2fa3","unresolved":true,"context_lines":[{"line_number":233,"context_line":"The order of the rest of the list is ignored."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Duplicates in the list (that is multiple external gateways with the same"},{"line_number":236,"context_line":"``network_id``) are allowed: in that case multiple gateway ports will be"},{"line_number":237,"context_line":"attached to the same network (this can be used to have the active-active setup"},{"line_number":238,"context_line":"when external gateways are available on the same network)."},{"line_number":239,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"2f0f0dcc_40e6145b","line":236,"range":{"start_line":236,"start_character":16,"end_line":236,"end_character":27},"in_reply_to":"787be3c9_eaf54281","updated":"2023-02-28 13:19:15.000000000","message":"\u003e I ended up going after the `network_id + external_fixed_ip` matching route.\n\u003e \n\u003e For updates it wasn\u0027t easy but with some matching it\u0027s possible:\n\u003e \n\u003e https://review.opendev.org/c/openstack/neutron/+/873593/16/neutron/db/l3_extra_gws_db.py#207\n\u003e https://review.opendev.org/c/openstack/neutron/+/873593/16/neutron/db/l3_extra_gws_db.py#409\n\nI don\u0027t get this. Don\u0027t we need two sets of external_fixed_ips for this to work? One to match the already existing external gws. And a second to express what should be the new updated value. It seems to me that the referred code assumes we never update external_fixed_ip itself. Am I misunderstanding something?\n\nIf we no longer allow the update of external_fixed_ips, then we only superficially keep the same API - syntax is the same, but the semantics are different.\n\n\u003e The 3-rd option above is problematic for policy reasons as the gateway port does not have a tenant ID specified and so working with it requires an elevated context:\n\u003e \n\u003e https://github.com/openstack/neutron/blob/24152f2019784fe684bb7b439b78e7265c6ffa08/neutron/db/l3_db.py#L345\n\u003e \nhttps://github.com/openstack/neutron-lib/blob/23fa6b3c2f46be6de8523355c786a682fd4419ef/neutron_lib/context.py#L116-L124\n\nI\u0027m not saying we should do this, but I guess making the external_gw API port centric would involve making the gw port visible and therefore setting a proper tenant_id (/project_id).","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":233,"context_line":"The order of the rest of the list is ignored."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Duplicates in the list (that is multiple external gateways with the same"},{"line_number":236,"context_line":"``network_id``) are allowed: in that case multiple gateway ports will be"},{"line_number":237,"context_line":"attached to the same network (this can be used to have the active-active setup"},{"line_number":238,"context_line":"when external gateways are available on the same network)."},{"line_number":239,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"2c11a7e5_b8b9e9ae","line":236,"range":{"start_line":236,"start_character":16,"end_line":236,"end_character":27},"in_reply_to":"8ca338a3_a90ac8d1","updated":"2023-02-21 15:12:26.000000000","message":"I believe this is not the only place in the neutron API where the port should have been the primary object. To me option (3) has the most clarity. But at the moment I did not think through the depth of API changes we would need for this. For example adding an external_gw today does not return the gw_port\u0027s ID in the API, so without changing that we already have some client side guesswork...\n\n\u003e a combination of a network_id and external_fixed_ips\n\nThis could work for delete, but not for update.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"42fd97fb33004f69ba050d01456d183de0ee75a7","unresolved":true,"context_lines":[{"line_number":233,"context_line":"The order of the rest of the list is ignored."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Duplicates in the list (that is multiple external gateways with the same"},{"line_number":236,"context_line":"``network_id``) are allowed: in that case multiple gateway ports will be"},{"line_number":237,"context_line":"attached to the same network (this can be used to have the active-active setup"},{"line_number":238,"context_line":"when external gateways are available on the same network)."},{"line_number":239,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"26c6f126_22f80186","line":236,"range":{"start_line":236,"start_character":16,"end_line":236,"end_character":27},"in_reply_to":"de0e543c_8b9c7826","updated":"2023-01-31 12:59:52.000000000","message":"Just want to make an observation after staring enough time at the existing code base.\n\nThere is currently a design issue to overcome which can be phrased as: \"how can we identify individual gateway instances?\". The current API for associating a gateway with a router is network-centric, not port-centric (Neutron is responsible for creating the network:router_gateway port and an operator is not involved). This creates an issue for the extra gateways API since in this proposal the aim is to allow multiple external gateways associated with the same network_id. For example:\n\n* When updating external gateways provided that there are multiple gateways attached to the same network an operator needs to know which ones will be updated consistently across API calls (Neutron will want to know too if we hope to have deterministic tests);\n* When deleting external gateways an operator might want to be able to delete just some (not all) gateways associated with the same network_id. Moreover, we may want to specify which ones in particular. However, since there is no way to identify them besides external_gateway_info \"content + order in the request body\" this becomes a problem.\n\nAs there\u0027s currently no identification mechanism for \"gateways\" - this is a collection of state that\u0027s not just a particular port_id or a network_id. The existing design always assumes there is only one gateway and does not bother with ID-ing individual instances. With multi-homing we need some form of enumiration.\n\nThis also becomes important for the existing codebase reuse as methods like _validate_gw_info make assumptions based on a singular gw_port_id retrieved from a router. We need to have a context that clearly identifies a gateway instance we are working with.\n\nSome thoughts on directions that could be taken:\n\n1) Relying on the external_gateway_info dict order in the request body:\n\n* may work for updates but seems brittle/difficult to maintain;\n* does not help when in for the \"delete\" API.\n\n2) Add an extra ID for each instance of external gateway info (treat it as an entity).\n\n3) Make the extra gateway API more port-centric by allowing gateway port ID to be specified in an update or delete request so that there is no ambiguity.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"02728d3ef463dffaae7c2f3fb52b6f2bc66d23e9","unresolved":true,"context_lines":[{"line_number":284,"context_line":"Extra routes API: BFD"},{"line_number":285,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":286,"context_line":""},{"line_number":287,"context_line":"When it comes to BFD and extra routes, we can:"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"* either pass BFD settings to the extra routes API (by extending it) and let"},{"line_number":290,"context_line":"  the implementation manage the relevant BFD records and associate them with"}],"source_content_type":"text/x-rst","patch_set":1,"id":"56df9795_d29ab582","line":287,"range":{"start_line":287,"start_character":39,"end_line":287,"end_character":45},"updated":"2023-01-12 20:09:51.000000000","message":"To be revisited since since a choice actually needs to be made here.\n\nUsability for an operator is a concern but I also do not want to clutter the extra routes API.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"38b295d465d4044329447e91cd57f8e3986f0ef6","unresolved":false,"context_lines":[{"line_number":284,"context_line":"Extra routes API: BFD"},{"line_number":285,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":286,"context_line":""},{"line_number":287,"context_line":"When it comes to BFD and extra routes, we can:"},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"* either pass BFD settings to the extra routes API (by extending it) and let"},{"line_number":290,"context_line":"  the implementation manage the relevant BFD records and associate them with"}],"source_content_type":"text/x-rst","patch_set":1,"id":"f533eb0d_5d2e1042","line":287,"range":{"start_line":287,"start_character":39,"end_line":287,"end_character":45},"in_reply_to":"56df9795_d29ab582","updated":"2023-01-20 13:56:00.000000000","message":"Done","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"22a0d59128e4dc292288962eda7d57f0623b3f6e","unresolved":true,"context_lines":[{"line_number":307,"context_line":"  locally to the instance but there are downsides to that as well which needs"},{"line_number":308,"context_line":"  to be considered more carefully"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"* Accepting ECMP routes via dynamic routing protocols. The current aim is to"},{"line_number":311,"context_line":"  utilize the default gateway information available in Neutron subnets to"},{"line_number":312,"context_line":"  configure default gateway ECMP routes or to use the extra routes extension;"},{"line_number":313,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"fc7e358a_fac12728","line":310,"range":{"start_line":310,"start_character":2,"end_line":310,"end_character":11},"updated":"2023-01-13 07:07:49.000000000","message":"Should we mention somewhere that we view the implementation of this spec as a building block for use by a possible future implementation of dynamic routing protocol support?","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"38b295d465d4044329447e91cd57f8e3986f0ef6","unresolved":false,"context_lines":[{"line_number":307,"context_line":"  locally to the instance but there are downsides to that as well which needs"},{"line_number":308,"context_line":"  to be considered more carefully"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"* Accepting ECMP routes via dynamic routing protocols. The current aim is to"},{"line_number":311,"context_line":"  utilize the default gateway information available in Neutron subnets to"},{"line_number":312,"context_line":"  configure default gateway ECMP routes or to use the extra routes extension;"},{"line_number":313,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"a6b11c30_9e41229a","line":310,"range":{"start_line":310,"start_character":2,"end_line":310,"end_character":11},"in_reply_to":"b0d96cd6_5d3b18f5","updated":"2023-01-20 13:56:00.000000000","message":"Ack","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"a1e94036f99a98a3dc9e2f4c1cd8084a8617ff8a","unresolved":true,"context_lines":[{"line_number":307,"context_line":"  locally to the instance but there are downsides to that as well which needs"},{"line_number":308,"context_line":"  to be considered more carefully"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"* Accepting ECMP routes via dynamic routing protocols. The current aim is to"},{"line_number":311,"context_line":"  utilize the default gateway information available in Neutron subnets to"},{"line_number":312,"context_line":"  configure default gateway ECMP routes or to use the extra routes extension;"},{"line_number":313,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"b0d96cd6_5d3b18f5","line":310,"range":{"start_line":310,"start_character":2,"end_line":310,"end_character":11},"in_reply_to":"fc7e358a_fac12728","updated":"2023-01-13 13:00:47.000000000","message":"I can extend this paragraph to say it - I think it\u0027s a useful thing to note.\n\nI think Neutron would need to get modelling for the \"administrative distance\" (or sometimes \"preference\") concept to resolve tie-breakers when it comes to routes to the same next-hop that come from different sources.\n\nhttps://en.wikipedia.org/wiki/Administrative_distance\n\nI\u0027ve seen different modes implemented where:\n\n* The lowest administrative distance assigned to the protocol by the system determines which route to the same destination wins;\n* \"Inter-protocol ECMP\" - when load-sharing is done across routes that come from different sources (static, different dynamic routing protocols) provided they have the same destination and metric.\n\nSo, there\u0027s a bit more modelling to do from what I guess.","commit_id":"24268d016484e0227f61221942d87193812d8e2f"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":46,"context_line":"addressed by implementing an extra conditional behavior when adding more than"},{"line_number":47,"context_line":"one gateway port to a router."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"ECMP routes can result in black-holing of traffic should the next-hop of a"},{"line_number":50,"context_line":"route becomes unreachable. `BFD`_ is a standard protocol adopted by IETF"},{"line_number":51,"context_line":"for next-hop failure detection which can be used for route eviction. OVN"},{"line_number":52,"context_line":"supports BFD `as of v21.03.0`_ with a data model that allows enabling"},{"line_number":53,"context_line":"BFD on a per next-hop basis by associating BFD session information with routes,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"304a229b_11e5bc65","line":50,"range":{"start_line":49,"start_character":0,"end_line":50,"end_character":26},"updated":"2023-02-09 14:59:46.000000000","message":"Perhaps easier to understand: \"ECMP routes can result in black-holing of traffic if the next hop of a route becomes unreachable\"\n(I hope I understand well what your intention is here)","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":46,"context_line":"addressed by implementing an extra conditional behavior when adding more than"},{"line_number":47,"context_line":"one gateway port to a router."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"ECMP routes can result in black-holing of traffic should the next-hop of a"},{"line_number":50,"context_line":"route becomes unreachable. `BFD`_ is a standard protocol adopted by IETF"},{"line_number":51,"context_line":"for next-hop failure detection which can be used for route eviction. OVN"},{"line_number":52,"context_line":"supports BFD `as of v21.03.0`_ with a data model that allows enabling"},{"line_number":53,"context_line":"BFD on a per next-hop basis by associating BFD session information with routes,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"2924bb73_19409c94","line":50,"range":{"start_line":49,"start_character":0,"end_line":50,"end_character":26},"in_reply_to":"304a229b_11e5bc65","updated":"2023-02-10 14:04:48.000000000","message":"Yes, you got it right - I think your version is better. I need to have less clericalism in the text :^)","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":50,"context_line":"route becomes unreachable. `BFD`_ is a standard protocol adopted by IETF"},{"line_number":51,"context_line":"for next-hop failure detection which can be used for route eviction. OVN"},{"line_number":52,"context_line":"supports BFD `as of v21.03.0`_ with a data model that allows enabling"},{"line_number":53,"context_line":"BFD on a per next-hop basis by associating BFD session information with routes,"},{"line_number":54,"context_line":"however, it is not modeled at the Neutron level even if a backend supports it."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"From the Neutron data model perspective, ECMP for routes is already a supported"}],"source_content_type":"text/x-rst","patch_set":2,"id":"63d8587c_ec4b4165","line":53,"range":{"start_line":53,"start_character":0,"end_line":53,"end_character":78},"updated":"2023-02-09 14:59:46.000000000","message":"Perhaps a word at least should be here about the possible performance impact, I checked BFD for OVS (I suppose OVN also sets OVSdb fields finally) and OVS sends out BFD messages, so it can have performance impact finally","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":50,"context_line":"route becomes unreachable. `BFD`_ is a standard protocol adopted by IETF"},{"line_number":51,"context_line":"for next-hop failure detection which can be used for route eviction. OVN"},{"line_number":52,"context_line":"supports BFD `as of v21.03.0`_ with a data model that allows enabling"},{"line_number":53,"context_line":"BFD on a per next-hop basis by associating BFD session information with routes,"},{"line_number":54,"context_line":"however, it is not modeled at the Neutron level even if a backend supports it."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"From the Neutron data model perspective, ECMP for routes is already a supported"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9799e8f8_8a6688ad","line":53,"range":{"start_line":53,"start_character":0,"end_line":53,"end_character":78},"in_reply_to":"63d8587c_ec4b4165","updated":"2023-02-10 14:04:48.000000000","message":"Yes, having too many sessions is a concern.\n\nWe could approach it in various to reduce the number of sessions, e.g. use a \"shared\" router in an admin project and an intermediate virtual network through which other other routers would connect to it or something else.\n\nThis also raises an RBAC-related concern: letting random tenants configure BFD probably isn\u0027t a good idea so we may require changing BFD parameters to be a cloud admin only operation.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"22e84054292b94f7fd488e507f68a2d141451dc4","unresolved":true,"context_lines":[{"line_number":50,"context_line":"route becomes unreachable. `BFD`_ is a standard protocol adopted by IETF"},{"line_number":51,"context_line":"for next-hop failure detection which can be used for route eviction. OVN"},{"line_number":52,"context_line":"supports BFD `as of v21.03.0`_ with a data model that allows enabling"},{"line_number":53,"context_line":"BFD on a per next-hop basis by associating BFD session information with routes,"},{"line_number":54,"context_line":"however, it is not modeled at the Neutron level even if a backend supports it."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"From the Neutron data model perspective, ECMP for routes is already a supported"}],"source_content_type":"text/x-rst","patch_set":2,"id":"eb830915_777dfe10","line":53,"range":{"start_line":53,"start_character":0,"end_line":53,"end_character":78},"in_reply_to":"9799e8f8_8a6688ad","updated":"2023-02-10 14:31:39.000000000","message":"ftr; the OVN BFD implementation for routes is actually handled as packet-ins to the ovn-controller, so they do not use the OVS BFD implementation for this part. Performance impact still applies though.\n\nThe OVS BFD implementation is still used on tunnel endpoints for liveness detection between chassis.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":50,"context_line":"route becomes unreachable. `BFD`_ is a standard protocol adopted by IETF"},{"line_number":51,"context_line":"for next-hop failure detection which can be used for route eviction. OVN"},{"line_number":52,"context_line":"supports BFD `as of v21.03.0`_ with a data model that allows enabling"},{"line_number":53,"context_line":"BFD on a per next-hop basis by associating BFD session information with routes,"},{"line_number":54,"context_line":"however, it is not modeled at the Neutron level even if a backend supports it."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"From the Neutron data model perspective, ECMP for routes is already a supported"}],"source_content_type":"text/x-rst","patch_set":2,"id":"2c438010_33dfda0b","line":53,"range":{"start_line":53,"start_character":0,"end_line":53,"end_character":78},"in_reply_to":"eb830915_777dfe10","updated":"2023-02-21 15:12:26.000000000","message":"+1 for reusing BFD sessions between tenants.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":83,"context_line":"In terms of the representation of multiple gateway ports in the Neutron DB the"},{"line_number":84,"context_line":"proposal follows the `multiple external gateways spec`_:"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"* Keep the ``routers``, ``routerports`` and ``ports`` tables as they are now"},{"line_number":87,"context_line":"  but start storing multiple ``network:router_gateway`` ports per router in the"},{"line_number":88,"context_line":"  ``routerports`` table;"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"* For backwards-compatibility store a single gateway port id in"},{"line_number":91,"context_line":"  ``routers.gw_port_id`` which will be present both in this field and also in"},{"line_number":92,"context_line":"  the ``routerports`` table and in ``routers.gw_port_id``)."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"* Extend the ``neutron.db.models.l3.Router`` class with a new attribute"},{"line_number":95,"context_line":"  ``gw_ports`` that will map to all relevant ``network:router_gateway`` ports"}],"source_content_type":"text/x-rst","patch_set":2,"id":"d7b99645_e43d6c6c","line":92,"range":{"start_line":86,"start_character":0,"end_line":92,"end_character":59},"updated":"2023-02-09 14:59:46.000000000","message":"+1, this still seems to be a good way to keep the old behaviour and extend it","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":false,"context_lines":[{"line_number":83,"context_line":"In terms of the representation of multiple gateway ports in the Neutron DB the"},{"line_number":84,"context_line":"proposal follows the `multiple external gateways spec`_:"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"* Keep the ``routers``, ``routerports`` and ``ports`` tables as they are now"},{"line_number":87,"context_line":"  but start storing multiple ``network:router_gateway`` ports per router in the"},{"line_number":88,"context_line":"  ``routerports`` table;"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"* For backwards-compatibility store a single gateway port id in"},{"line_number":91,"context_line":"  ``routers.gw_port_id`` which will be present both in this field and also in"},{"line_number":92,"context_line":"  the ``routerports`` table and in ``routers.gw_port_id``)."},{"line_number":93,"context_line":""},{"line_number":94,"context_line":"* Extend the ``neutron.db.models.l3.Router`` class with a new attribute"},{"line_number":95,"context_line":"  ``gw_ports`` that will map to all relevant ``network:router_gateway`` ports"}],"source_content_type":"text/x-rst","patch_set":2,"id":"558e2ae7_ee1a10d1","line":92,"range":{"start_line":86,"start_character":0,"end_line":92,"end_character":59},"in_reply_to":"d7b99645_e43d6c6c","updated":"2023-02-10 14:04:48.000000000","message":"Ack","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":113,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":114,"context_line":"can be a reasonable default."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":117,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":118,"context_line":"the L3 service plugin supports BFD)."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Besides the handling of default routes derived from subnets, the BFD"},{"line_number":121,"context_line":"information needs to be associated with every route object similar to how"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ab75e3db_73cd523d","line":118,"range":{"start_line":116,"start_character":0,"end_line":118,"end_character":36},"updated":"2023-02-09 14:59:46.000000000","message":"ditto, I vote for add API of cfg option for it, even as a \"merged\" option together with enable_default_route_ecmp","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"87a18643a607dbed11ed0999ce27070300d9edb5","unresolved":true,"context_lines":[{"line_number":113,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":114,"context_line":"can be a reasonable default."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":117,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":118,"context_line":"the L3 service plugin supports BFD)."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Besides the handling of default routes derived from subnets, the BFD"},{"line_number":121,"context_line":"information needs to be associated with every route object similar to how"}],"source_content_type":"text/x-rst","patch_set":2,"id":"a6dc839b_2a51eade","line":118,"range":{"start_line":116,"start_character":0,"end_line":118,"end_character":36},"in_reply_to":"8de66160_23084825","updated":"2023-03-14 10:42:19.000000000","message":"thanks","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b400b0c19f80edb9b927573671d65d4f8fbf3ec9","unresolved":true,"context_lines":[{"line_number":113,"context_line":"OVN case implying `ECMP symmetric reply`_ to be enabled when adding ECMP routes"},{"line_number":114,"context_line":"can be a reasonable default."},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":117,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":118,"context_line":"the L3 service plugin supports BFD)."},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Besides the handling of default routes derived from subnets, the BFD"},{"line_number":121,"context_line":"information needs to be associated with every route object similar to how"}],"source_content_type":"text/x-rst","patch_set":2,"id":"8de66160_23084825","line":118,"range":{"start_line":116,"start_character":0,"end_line":118,"end_character":36},"in_reply_to":"ab75e3db_73cd523d","updated":"2023-03-10 15:42:17.000000000","message":"OK, that could be a way to specify a default value for newly created routers.\n\nI can look at the enable_snat_by_default option and see if it\u0027s going to be easy to adapt.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"0b286004a9c94da07eb7f86d5d86e414f046f88e","unresolved":true,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Besides the handling of default routes derived from subnets, the BFD"},{"line_number":121,"context_line":"information needs to be associated with every route object similar to how"},{"line_number":122,"context_line":"it is done in the OVN data model itself. The reason behind this is that BFD"},{"line_number":123,"context_line":"can be enabled or disabled on a per next-hop basis even if the destination"},{"line_number":124,"context_line":"of a route is the same."},{"line_number":125,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"756065c9_6f97af1c","line":122,"range":{"start_line":122,"start_character":19,"end_line":122,"end_character":39},"updated":"2023-03-09 15:06:41.000000000","message":"Maybe I\u0027m missing something but the BFD register is associated to a port, nothing else.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"81ab23a6d6e207a349f26e436c713503fef3e805","unresolved":true,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"Besides the handling of default routes derived from subnets, the BFD"},{"line_number":121,"context_line":"information needs to be associated with every route object similar to how"},{"line_number":122,"context_line":"it is done in the OVN data model itself. The reason behind this is that BFD"},{"line_number":123,"context_line":"can be enabled or disabled on a per next-hop basis even if the destination"},{"line_number":124,"context_line":"of a route is the same."},{"line_number":125,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"a61c75e1_e8fe746e","line":122,"range":{"start_line":122,"start_character":19,"end_line":122,"end_character":39},"in_reply_to":"756065c9_6f97af1c","updated":"2023-03-09 17:03:08.000000000","message":"The BFD table itself has an index across logical port and BFD peer IP [0], and there is a reference from the `Logical_Router_Static_Route` table to the BFD table for BFD-enabled routes [1].\n\n0: https://github.com/ovn-org/ovn/blob/69692c6cb90ee88525dcbc356733c4b6d3870816/ovn-nb.ovsschema#L635\n1: https://manpages.ubuntu.com/manpages/kinetic/man5/ovn-nb.5.html#logical_router_static_route%20table","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":128,"context_line":"``routerroutes`` table storing static routes much like the association of"},{"line_number":129,"context_line":"`logical router routes with BFD records`_ is done."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"OVN allows making a BFD session for a particular static route use a different"},{"line_number":132,"context_line":"destination IP for checking reachability rather than the next hop of the static"},{"line_number":133,"context_line":"route itself (by storing the BFD peer IP address in the ``dst_ip`` field in"},{"line_number":134,"context_line":"the ``BFD`` table). This is a useful semantic for separating the control plane"}],"source_content_type":"text/x-rst","patch_set":2,"id":"e32e247c_87286462","line":131,"range":{"start_line":131,"start_character":56,"end_line":131,"end_character":65},"updated":"2023-02-09 14:59:46.000000000","message":"...route to use....?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"923612d99854742ab977a346681bfb9d0b0c2911","unresolved":false,"context_lines":[{"line_number":128,"context_line":"``routerroutes`` table storing static routes much like the association of"},{"line_number":129,"context_line":"`logical router routes with BFD records`_ is done."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"OVN allows making a BFD session for a particular static route use a different"},{"line_number":132,"context_line":"destination IP for checking reachability rather than the next hop of the static"},{"line_number":133,"context_line":"route itself (by storing the BFD peer IP address in the ``dst_ip`` field in"},{"line_number":134,"context_line":"the ``BFD`` table). This is a useful semantic for separating the control plane"}],"source_content_type":"text/x-rst","patch_set":2,"id":"cff5633c_be48692d","line":131,"range":{"start_line":131,"start_character":56,"end_line":131,"end_character":65},"in_reply_to":"ba6d8854_a172fc2f","updated":"2023-03-27 13:31:25.000000000","message":"Done","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":128,"context_line":"``routerroutes`` table storing static routes much like the association of"},{"line_number":129,"context_line":"`logical router routes with BFD records`_ is done."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"OVN allows making a BFD session for a particular static route use a different"},{"line_number":132,"context_line":"destination IP for checking reachability rather than the next hop of the static"},{"line_number":133,"context_line":"route itself (by storing the BFD peer IP address in the ``dst_ip`` field in"},{"line_number":134,"context_line":"the ``BFD`` table). This is a useful semantic for separating the control plane"}],"source_content_type":"text/x-rst","patch_set":2,"id":"ba6d8854_a172fc2f","line":131,"range":{"start_line":131,"start_character":56,"end_line":131,"end_character":65},"in_reply_to":"e32e247c_87286462","updated":"2023-02-10 14:04:48.000000000","message":"Thanks for spotting, I\u0027ll fix it.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":135,"context_line":"and data plane and can be used for static routes of Neutron routers too."},{"line_number":136,"context_line":"However, when it comes to default routes inferred from gateway port to subnet"},{"line_number":137,"context_line":"associations, Neutron\u0027s behavior should be to use the next hop of the default"},{"line_number":138,"context_line":"route as a ``dst_ip``."},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"OVN models the BFD table as follows (see `ovn-nb docs`_ for more information):"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"e5562b2a_f57475f4","line":138,"updated":"2023-02-21 15:12:26.000000000","message":"What will ovn use as the source IP? Do we need to allocate IPs for that purpose beyond what the router already has? If yes, will that be discoverable through the API?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"923612d99854742ab977a346681bfb9d0b0c2911","unresolved":false,"context_lines":[{"line_number":135,"context_line":"and data plane and can be used for static routes of Neutron routers too."},{"line_number":136,"context_line":"However, when it comes to default routes inferred from gateway port to subnet"},{"line_number":137,"context_line":"associations, Neutron\u0027s behavior should be to use the next hop of the default"},{"line_number":138,"context_line":"route as a ``dst_ip``."},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"OVN models the BFD table as follows (see `ovn-nb docs`_ for more information):"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"daf9494e_654b8ef2","line":138,"in_reply_to":"e5562b2a_f57475f4","updated":"2023-03-27 13:31:25.000000000","message":"The ovn-controller will use the IP address associated with the LRP provided in the `logical_port` field of the `BFD` table.\n\nRef: https://manpages.ubuntu.com/manpages/jammy/man5/ovn-nb.5.html#bfd%20table","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":141,"context_line":""},{"line_number":142,"context_line":"* ``dst_ip`` - a BFD peer IP address;"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"* ``mix_tx`` - an integer specifying the minimum interval (milliseconds, \u003e\u003d1)"},{"line_number":145,"context_line":"  that OVN would use when transmitting the BFD control packet (minus jitter);"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"* ``min_rx`` - an integer specifying the minimum interval (milliseconds)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"e4c826d7_281520bd","line":144,"range":{"start_line":144,"start_character":4,"end_line":144,"end_character":10},"updated":"2023-02-09 14:59:46.000000000","message":"min_tx","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"923612d99854742ab977a346681bfb9d0b0c2911","unresolved":false,"context_lines":[{"line_number":141,"context_line":""},{"line_number":142,"context_line":"* ``dst_ip`` - a BFD peer IP address;"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"* ``mix_tx`` - an integer specifying the minimum interval (milliseconds, \u003e\u003d1)"},{"line_number":145,"context_line":"  that OVN would use when transmitting the BFD control packet (minus jitter);"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"* ``min_rx`` - an integer specifying the minimum interval (milliseconds)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"94075f39_c3d76b80","line":144,"range":{"start_line":144,"start_character":4,"end_line":144,"end_character":10},"in_reply_to":"183f365e_7c807010","updated":"2023-03-27 13:31:25.000000000","message":"Done","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":141,"context_line":""},{"line_number":142,"context_line":"* ``dst_ip`` - a BFD peer IP address;"},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"* ``mix_tx`` - an integer specifying the minimum interval (milliseconds, \u003e\u003d1)"},{"line_number":145,"context_line":"  that OVN would use when transmitting the BFD control packet (minus jitter);"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"* ``min_rx`` - an integer specifying the minimum interval (milliseconds)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"183f365e_7c807010","line":144,"range":{"start_line":144,"start_character":4,"end_line":144,"end_character":10},"in_reply_to":"e4c826d7_281520bd","updated":"2023-02-10 14:04:48.000000000","message":"ty, definitely a typo.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":160,"context_line":"In Neutron each static route could optionally have a BFD record associated with"},{"line_number":161,"context_line":"it with a BFD record ID acting as a foreign key in the ``routerroutes`` table."},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"The proposed approach for this spec is limited to only adding a small portion"},{"line_number":164,"context_line":"of the columns in the database and without exposing a full API to BFD (a future"},{"line_number":165,"context_line":"revision might implement the `BFD support spec`_ to provide a full API) with"},{"line_number":166,"context_line":"the aim of allowing operators to specify the need to use BFD with default"},{"line_number":167,"context_line":"settings of a backend."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"The proposed columns for the ``bfd_monitor`` table for in this spec are:"},{"line_number":170,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"b1acda2c_49e58f5f","line":167,"range":{"start_line":163,"start_character":0,"end_line":167,"end_character":22},"updated":"2023-02-09 14:59:46.000000000","message":"agree, I think these things (like min_tx and min_rx) can be defaulted or in worst case we can add some cfg option for it","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"d4afb23fc778471b9b7f8f5df53efca234bad655","unresolved":true,"context_lines":[{"line_number":189,"context_line":"| status            | String  | N/A   | R    | Shows if the BFD monitor was          |"},{"line_number":190,"context_line":"|                   |         |       |      | succesfully created in the backend,   |"},{"line_number":191,"context_line":"|                   |         |       |      | but nothing about the session status, |"},{"line_number":192,"context_line":"|                   |         |       |      | for that the session_status API       |"},{"line_number":193,"context_line":"|                   |         |       |      | endpoint can be used.                 |"},{"line_number":194,"context_line":"+-------------------+---------+-------+------+---------------------------------------+"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"d10aacbc_4a886ed6","line":192,"range":{"start_line":192,"start_character":60,"end_line":192,"end_character":74},"updated":"2023-02-09 14:59:46.000000000","message":"Is this part of this spec?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"923612d99854742ab977a346681bfb9d0b0c2911","unresolved":false,"context_lines":[{"line_number":189,"context_line":"| status            | String  | N/A   | R    | Shows if the BFD monitor was          |"},{"line_number":190,"context_line":"|                   |         |       |      | succesfully created in the backend,   |"},{"line_number":191,"context_line":"|                   |         |       |      | but nothing about the session status, |"},{"line_number":192,"context_line":"|                   |         |       |      | for that the session_status API       |"},{"line_number":193,"context_line":"|                   |         |       |      | endpoint can be used.                 |"},{"line_number":194,"context_line":"+-------------------+---------+-------+------+---------------------------------------+"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bfce661c_69272371","line":192,"range":{"start_line":192,"start_character":60,"end_line":192,"end_character":74},"in_reply_to":"b7e6b7cd_d1d8d49f","updated":"2023-03-27 13:31:25.000000000","message":"Done","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":189,"context_line":"| status            | String  | N/A   | R    | Shows if the BFD monitor was          |"},{"line_number":190,"context_line":"|                   |         |       |      | succesfully created in the backend,   |"},{"line_number":191,"context_line":"|                   |         |       |      | but nothing about the session status, |"},{"line_number":192,"context_line":"|                   |         |       |      | for that the session_status API       |"},{"line_number":193,"context_line":"|                   |         |       |      | endpoint can be used.                 |"},{"line_number":194,"context_line":"+-------------------+---------+-------+------+---------------------------------------+"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"b7e6b7cd_d1d8d49f","line":192,"range":{"start_line":192,"start_character":60,"end_line":192,"end_character":74},"in_reply_to":"d10aacbc_4a886ed6","updated":"2023-02-10 14:04:48.000000000","message":"Sorry, I took a subset of definitions from the BFD support spec for compatibility with it:\n\nhttps://specs.openstack.org/openstack/neutron-specs/specs/xena/bfd_support.html\n\nBut I missed editing this part of the description - thanks for spotting it.\n\nI\u0027ll change this part.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"9d7d663aa47446980e3762879a75e8a21ac58ed9","unresolved":true,"context_lines":[{"line_number":212,"context_line":"  {"},{"line_number":213,"context_line":"    \u0027network_id\u0027: {"},{"line_number":214,"context_line":"      \u0027type:uuid\u0027: None, \u0027required\u0027: True},"},{"line_number":215,"context_line":"    \u0027external_fixed_ips\u0027: {"},{"line_number":216,"context_line":"      \u0027type:fixed_ips\u0027: None,"},{"line_number":217,"context_line":"      \u0027required\u0027: False"},{"line_number":218,"context_line":"    },"}],"source_content_type":"text/x-rst","patch_set":2,"id":"f0cd214f_30c6400e","line":215,"range":{"start_line":215,"start_character":26,"end_line":215,"end_character":27},"updated":"2023-02-16 14:21:46.000000000","message":"This should be a list","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"d32134ac76b6fe41757ce0163c0fb184b02dbd76","unresolved":false,"context_lines":[{"line_number":212,"context_line":"  {"},{"line_number":213,"context_line":"    \u0027network_id\u0027: {"},{"line_number":214,"context_line":"      \u0027type:uuid\u0027: None, \u0027required\u0027: True},"},{"line_number":215,"context_line":"    \u0027external_fixed_ips\u0027: {"},{"line_number":216,"context_line":"      \u0027type:fixed_ips\u0027: None,"},{"line_number":217,"context_line":"      \u0027required\u0027: False"},{"line_number":218,"context_line":"    },"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a990e07_785826f2","line":215,"range":{"start_line":215,"start_character":26,"end_line":215,"end_character":27},"in_reply_to":"f0cd214f_30c6400e","updated":"2023-02-17 07:28:34.000000000","message":"ah, that is embedded in \u0027type:fixed_ips\u0027, never mind me :)","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":231,"context_line":"  }"},{"line_number":232,"context_line":""},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":235,"context_line":"routes should be used or not (if the L3 service plugin supports them)."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"}],"source_content_type":"text/x-rst","patch_set":2,"id":"e50d8af9_56d5a91c","line":234,"range":{"start_line":234,"start_character":35,"end_line":234,"end_character":47},"updated":"2023-02-21 15:12:26.000000000","message":"If it\u0027s router level, then why don\u0027t we have it as a router attribute? If it\u0027s in the gw_info dict, then we can have multiple contradictory values.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"05cd0597f5b6d37d78ef2f85b9c65def4fdf2fa3","unresolved":false,"context_lines":[{"line_number":231,"context_line":"  }"},{"line_number":232,"context_line":""},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":235,"context_line":"routes should be used or not (if the L3 service plugin supports them)."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1ca2a382_eba820b7","line":234,"range":{"start_line":234,"start_character":35,"end_line":234,"end_character":47},"in_reply_to":"54d0763a_d4fce1ee","updated":"2023-02-28 13:19:15.000000000","message":"I see your point to keep the enable_default_route_* values in the external_gw_info. I would not suggest to change it. Thanks for the explanation.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":true,"context_lines":[{"line_number":231,"context_line":"  }"},{"line_number":232,"context_line":""},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":235,"context_line":"routes should be used or not (if the L3 service plugin supports them)."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"}],"source_content_type":"text/x-rst","patch_set":2,"id":"54d0763a_d4fce1ee","line":234,"range":{"start_line":234,"start_character":35,"end_line":234,"end_character":47},"in_reply_to":"e50d8af9_56d5a91c","updated":"2023-02-22 14:13:16.000000000","message":"I was going after how it\u0027s done with `enable_snat` (which is not ideal, considering it\u0027s a router-level attribute too in reality and it cannot be set on a per-gateway basis).\n\nhttps://github.com/openstack/neutron/blob/24152f2019784fe684bb7b439b78e7265c6ffa08/neutron/db/l3_gwmode_db.py#L35-L37\n\nBut it could be modeled as an extra attribute similar to those which would make the neutron-lib change look slightly differently:\n\nhttps://github.com/openstack/neutron/blob/24152f2019784fe684bb7b439b78e7265c6ffa08/neutron/db/models/l3_attrs.py#L18-L42\n\nI think enable_snat was originally stored at the router level because of the assumption of a single gateway. However, semantically it\u0027s a per-gateway thing. Enabling SNAT on a per-gateway basis has it\u0027s own challenges with asymmetric routing but that\u0027s separate to it being a per-gateway attribute.\n\nWith the ECMP and BFD policies I thought of the same approach as there isn\u0027t a separate table for the \"gateway attachment\" entity which could store this. The `routerports` table is an interesting candidate for this if we wanted to do it at the per-gateway level.\n\nTo sum up, so far I planned for it to be at the router level to be consistent with `enable_snat` in terms of the implementation. But semantically, the granularity of ECMP and BFD options could be extended to the gateway level so they fit into the external_gateway_info.\n\nA compromise could be to keep the new options at the router level while allowing for mixed scenarios to be implemented using extra routes with per-route BFD or manually entered ECMP routes. After all, the whole point of this feature is to make it easy to enable the ECMP case with BFD and mixed options are edge cases.\n\nI can alter the spec to how what you suggest.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":234,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":235,"context_line":"routes should be used or not (if the L3 service plugin supports them)."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":238,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":239,"context_line":"the L3 service plugin supports BFD)."},{"line_number":240,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"ce71496b_c1666b96","line":237,"range":{"start_line":237,"start_character":34,"end_line":237,"end_character":46},"updated":"2023-02-21 15:12:26.000000000","message":"The same question here.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"05cd0597f5b6d37d78ef2f85b9c65def4fdf2fa3","unresolved":false,"context_lines":[{"line_number":234,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":235,"context_line":"routes should be used or not (if the L3 service plugin supports them)."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":238,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":239,"context_line":"the L3 service plugin supports BFD)."},{"line_number":240,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"844c2889_c7a43768","line":237,"range":{"start_line":237,"start_character":34,"end_line":237,"end_character":46},"in_reply_to":"76470ad7_063af2c0","updated":"2023-02-28 13:19:15.000000000","message":"Done","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":true,"context_lines":[{"line_number":234,"context_line":"``enable_default_route_ecmp`` is a router-level policy on whether ECMP default"},{"line_number":235,"context_line":"routes should be used or not (if the L3 service plugin supports them)."},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"``enable_default_route_bfd`` is a router-level policy on whether BFD should be"},{"line_number":238,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":239,"context_line":"the L3 service plugin supports BFD)."},{"line_number":240,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"76470ad7_063af2c0","line":237,"range":{"start_line":237,"start_character":34,"end_line":237,"end_character":46},"in_reply_to":"ce71496b_c1666b96","updated":"2023-02-22 14:13:16.000000000","message":"Answered above.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"68abbe4eb5144828ae99946146ffedd2e6c524e1","unresolved":true,"context_lines":[{"line_number":245,"context_line":"compatibility purposes as it contains the same information as the"},{"line_number":246,"context_line":"``external_gateway_info`` does. When ``enable_default_route_ecmp`` is set to"},{"line_number":247,"context_line":"``false`` it also defines the default gateway placed into the routers routing"},{"line_number":248,"context_line":"table (in the multi-segment network case, the placement of a gateway port"},{"line_number":249,"context_line":"matters for inferring the default gateway based on the subnet used on a network"},{"line_number":250,"context_line":"segment)."},{"line_number":251,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"ac964aee_093d09fe","line":248,"range":{"start_line":248,"start_character":14,"end_line":248,"end_character":40},"updated":"2023-01-25 08:24:11.000000000","message":"Note that there is a bug related to default routes not being added to OVN NB if a segment_id of a subnet is not set to `None`:\n\nhttps://bugs.launchpad.net/neutron/+bug/2003842\n\nThis bug needs to be fixed to properly support the multiple gateways on a multi-segment network case but it affects Neutron as it is even without this feature.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"0b286004a9c94da07eb7f86d5d86e414f046f88e","unresolved":true,"context_lines":[{"line_number":245,"context_line":"compatibility purposes as it contains the same information as the"},{"line_number":246,"context_line":"``external_gateway_info`` does. When ``enable_default_route_ecmp`` is set to"},{"line_number":247,"context_line":"``false`` it also defines the default gateway placed into the routers routing"},{"line_number":248,"context_line":"table (in the multi-segment network case, the placement of a gateway port"},{"line_number":249,"context_line":"matters for inferring the default gateway based on the subnet used on a network"},{"line_number":250,"context_line":"segment)."},{"line_number":251,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"4cd1bb64_79fd5d0d","line":248,"range":{"start_line":248,"start_character":14,"end_line":248,"end_character":40},"in_reply_to":"0dfa14cc_a4753aea","updated":"2023-03-09 15:06:41.000000000","message":"Please make clear in this section that we don\u0027t support OVN router multisegment support.\n\nAbout the GW subnets, make that clear too here, stating that overlapping subnets cannot be configured as GWs of the same router.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":245,"context_line":"compatibility purposes as it contains the same information as the"},{"line_number":246,"context_line":"``external_gateway_info`` does. When ``enable_default_route_ecmp`` is set to"},{"line_number":247,"context_line":"``false`` it also defines the default gateway placed into the routers routing"},{"line_number":248,"context_line":"table (in the multi-segment network case, the placement of a gateway port"},{"line_number":249,"context_line":"matters for inferring the default gateway based on the subnet used on a network"},{"line_number":250,"context_line":"segment)."},{"line_number":251,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"95dda7de_447f503e","line":248,"range":{"start_line":248,"start_character":14,"end_line":248,"end_character":40},"in_reply_to":"3d2e3115_907474f6","updated":"2023-02-21 15:12:26.000000000","message":"One option is to disallow overlapping subnets by mandating the use of an address scope. However it\u0027s already very limited how much overlap we allow on the same router:\n\nhttps://opendev.org/openstack/neutron/src/commit/8455edda46b4a465e2f184b59ad31476ce79c6c4/neutron/db/l3_db.py#L572-L587","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"923612d99854742ab977a346681bfb9d0b0c2911","unresolved":false,"context_lines":[{"line_number":245,"context_line":"compatibility purposes as it contains the same information as the"},{"line_number":246,"context_line":"``external_gateway_info`` does. When ``enable_default_route_ecmp`` is set to"},{"line_number":247,"context_line":"``false`` it also defines the default gateway placed into the routers routing"},{"line_number":248,"context_line":"table (in the multi-segment network case, the placement of a gateway port"},{"line_number":249,"context_line":"matters for inferring the default gateway based on the subnet used on a network"},{"line_number":250,"context_line":"segment)."},{"line_number":251,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"a6e4177f_ce34d881","line":248,"range":{"start_line":248,"start_character":14,"end_line":248,"end_character":40},"in_reply_to":"4cd1bb64_79fd5d0d","updated":"2023-03-27 13:31:25.000000000","message":"Done","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":true,"context_lines":[{"line_number":245,"context_line":"compatibility purposes as it contains the same information as the"},{"line_number":246,"context_line":"``external_gateway_info`` does. When ``enable_default_route_ecmp`` is set to"},{"line_number":247,"context_line":"``false`` it also defines the default gateway placed into the routers routing"},{"line_number":248,"context_line":"table (in the multi-segment network case, the placement of a gateway port"},{"line_number":249,"context_line":"matters for inferring the default gateway based on the subnet used on a network"},{"line_number":250,"context_line":"segment)."},{"line_number":251,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"0dfa14cc_a4753aea","line":248,"range":{"start_line":248,"start_character":14,"end_line":248,"end_character":40},"in_reply_to":"95dda7de_447f503e","updated":"2023-02-22 14:13:16.000000000","message":"In the WIP patches I have this:\n\nhttps://review.opendev.org/c/openstack/neutron/+/873593/16/neutron/db/l3_extra_gws_db.py#170\n\nSo there is a way to do it without mandating address scopes just by checking overlapping ranges. I am reusing some of the previously written code but the key difference here is that if multiple ports are attached to the same network (causing the overlap) then it\u0027s ok. If the overlap is there on subnets of different networks then it\u0027s definitely a problem.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"901a2fe8f460f80c9068051abbbf3bececa84887","unresolved":true,"context_lines":[{"line_number":245,"context_line":"compatibility purposes as it contains the same information as the"},{"line_number":246,"context_line":"``external_gateway_info`` does. When ``enable_default_route_ecmp`` is set to"},{"line_number":247,"context_line":"``false`` it also defines the default gateway placed into the routers routing"},{"line_number":248,"context_line":"table (in the multi-segment network case, the placement of a gateway port"},{"line_number":249,"context_line":"matters for inferring the default gateway based on the subnet used on a network"},{"line_number":250,"context_line":"segment)."},{"line_number":251,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3d2e3115_907474f6","line":248,"range":{"start_line":248,"start_character":14,"end_line":248,"end_character":40},"in_reply_to":"ac964aee_093d09fe","updated":"2023-02-02 19:19:54.000000000","message":"Discussed that further with Rodolfo\n\nhttps://meetings.opendev.org/irclogs/%23openstack-neutron/%23openstack-neutron.2023-01-27.log.html#t2023-01-27T14:30:42\n\nand the case where a gateway is created on a multi-segment network will not be supported.\n\nHowever, having gateways on different single-segment networks raises a different question: subnets on those networks may potentially overlap unless they have subnets allocated from the same subnet pool (i.e. they are in the same address scope).\n\nOverlapping subnets lead to a situation where a next hop of a default route is reachable via directly connected routes added for either of the gateways and can inadvertently create a black hole (on a different segment that next-hop IP might not be a router or may not be assigned but we\u0027d still get a logical flow for it). BFD would limit the impact of that to a degree. We could also get into a situation where next hops themselves would match in which case there would be a conflict during a route addition (while routes have a matching 0.0.0.0/0 destination and next-hop they have a different address scope/VRF).\n\nI am going to explore how to guard an operator against that.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":267,"context_line":""},{"line_number":268,"context_line":"* ``PUT /v2.0/routers/{router_id}/add_external_gateways``"},{"line_number":269,"context_line":""},{"line_number":270,"context_line":"  Accepts a list of ``external_gateway_info`` structures.  Adding an"},{"line_number":271,"context_line":"  external gateway to a network that already has one raises an error."},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"* ``PUT /v2.0/routers/{router_id}/update_external_gateways``"},{"line_number":274,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"8fd1d763_02c75396","line":271,"range":{"start_line":270,"start_character":59,"end_line":271,"end_character":69},"updated":"2023-02-21 15:12:26.000000000","message":"Maybe forgot to modify? But as I understand the comment in L255, this all still maye in flight.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":true,"context_lines":[{"line_number":267,"context_line":""},{"line_number":268,"context_line":"* ``PUT /v2.0/routers/{router_id}/add_external_gateways``"},{"line_number":269,"context_line":""},{"line_number":270,"context_line":"  Accepts a list of ``external_gateway_info`` structures.  Adding an"},{"line_number":271,"context_line":"  external gateway to a network that already has one raises an error."},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"* ``PUT /v2.0/routers/{router_id}/update_external_gateways``"},{"line_number":274,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9435ccaa_238a83b1","line":271,"range":{"start_line":270,"start_character":59,"end_line":271,"end_character":69},"in_reply_to":"8fd1d763_02c75396","updated":"2023-02-22 14:13:16.000000000","message":"Yes, thanks for spotting it!","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":297,"context_line":"spec is to check whether the same is true for the OVN-based L3 implementation."},{"line_number":298,"context_line":""},{"line_number":299,"context_line":"One parameter that is proposed to be added here is a boolean"},{"line_number":300,"context_line":"``ecmp_symmetric_reply`` which will act a way to populate the relevant new"},{"line_number":301,"context_line":"column for each static route (it is modeled at the route level in OVN as well)."},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"Extra routes API: BFD"}],"source_content_type":"text/x-rst","patch_set":2,"id":"6f417302_e02cb989","line":300,"range":{"start_line":300,"start_character":2,"end_line":300,"end_character":22},"updated":"2023-02-21 15:12:26.000000000","message":"Just to see if I understand the scope: This spec does not include advertising the neutron subnets availability over multiple external gateways. Unless someone manually configures that, traffic destined to the cloud will not use ECMP and because of ecmp_symmetric_reply its response will not either. So we will use ECMP only for traffic originating from the cloud. Do I get it right?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":true,"context_lines":[{"line_number":297,"context_line":"spec is to check whether the same is true for the OVN-based L3 implementation."},{"line_number":298,"context_line":""},{"line_number":299,"context_line":"One parameter that is proposed to be added here is a boolean"},{"line_number":300,"context_line":"``ecmp_symmetric_reply`` which will act a way to populate the relevant new"},{"line_number":301,"context_line":"column for each static route (it is modeled at the route level in OVN as well)."},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"Extra routes API: BFD"}],"source_content_type":"text/x-rst","patch_set":2,"id":"edf03f5a_df31a0a7","line":300,"range":{"start_line":300,"start_character":2,"end_line":300,"end_character":22},"in_reply_to":"6f417302_e02cb989","updated":"2023-02-22 14:13:16.000000000","message":"Yes, that option is just for making sure that replies go the same route as the requests by the means of utilizing conntrack.\n\nIn order for us to properly advertise things we need to make the BGP service plugin work with OVN and also make it aware of multiple external gateways.\n\nWe tried to keep the scope of this limited and be a building block for the future work in that direction.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"05cd0597f5b6d37d78ef2f85b9c65def4fdf2fa3","unresolved":false,"context_lines":[{"line_number":297,"context_line":"spec is to check whether the same is true for the OVN-based L3 implementation."},{"line_number":298,"context_line":""},{"line_number":299,"context_line":"One parameter that is proposed to be added here is a boolean"},{"line_number":300,"context_line":"``ecmp_symmetric_reply`` which will act a way to populate the relevant new"},{"line_number":301,"context_line":"column for each static route (it is modeled at the route level in OVN as well)."},{"line_number":302,"context_line":""},{"line_number":303,"context_line":"Extra routes API: BFD"}],"source_content_type":"text/x-rst","patch_set":2,"id":"8d15ba8b_72473a35","line":300,"range":{"start_line":300,"start_character":2,"end_line":300,"end_character":22},"in_reply_to":"edf03f5a_df31a0a7","updated":"2023-02-28 13:19:15.000000000","message":"Makes sense to me.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"8ed49a818932b98cf88276a8539546863aada39e","unresolved":true,"context_lines":[{"line_number":303,"context_line":"Extra routes API: BFD"},{"line_number":304,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":305,"context_line":""},{"line_number":306,"context_line":"In the absence of a full BFD API, users will have an option to specify a policy"},{"line_number":307,"context_line":"on the routers (``enable_default_route_bfd``) or routes (by supplying an"},{"line_number":308,"context_line":"additional option to the ``add_extraroutes`` API invocation called"},{"line_number":309,"context_line":"``enable_bfd``). Neutron will manage ``bfd_monitor`` records related to those"},{"line_number":310,"context_line":"options internally for now using route next hops as ``dst_ip`` fields."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Out of Scope"},{"line_number":313,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"23739657_cb257ff6","line":310,"range":{"start_line":306,"start_character":0,"end_line":310,"end_character":70},"updated":"2023-02-09 15:03:46.000000000","message":"You mentioned a status field, but that is only for the db record, am I understand well? How the user can check that on the API?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":15554,"name":"Bence Romsics","email":"bence.romsics@gmail.com","username":"ebenrom","status":"working for Ericsson, UTC+1 (+DST)"},"change_message_id":"f4361922d734c1b6f52a34d9101bdceab3b19dbc","unresolved":true,"context_lines":[{"line_number":303,"context_line":"Extra routes API: BFD"},{"line_number":304,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":305,"context_line":""},{"line_number":306,"context_line":"In the absence of a full BFD API, users will have an option to specify a policy"},{"line_number":307,"context_line":"on the routers (``enable_default_route_bfd``) or routes (by supplying an"},{"line_number":308,"context_line":"additional option to the ``add_extraroutes`` API invocation called"},{"line_number":309,"context_line":"``enable_bfd``). Neutron will manage ``bfd_monitor`` records related to those"},{"line_number":310,"context_line":"options internally for now using route next hops as ``dst_ip`` fields."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Out of Scope"},{"line_number":313,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"beec59e9_0beb32d1","line":310,"range":{"start_line":306,"start_character":0,"end_line":310,"end_character":70},"in_reply_to":"12ea247b_9cc56859","updated":"2023-02-21 15:12:26.000000000","message":"I think we need some kind of visibility into the BFD session status, mostly for network debugging reasons. In the first step I could imagine not having that in the API, but the accessing it maybe some admin-only, ovn-specific way. As long as it\u0027s documented, that could work.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":303,"context_line":"Extra routes API: BFD"},{"line_number":304,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":305,"context_line":""},{"line_number":306,"context_line":"In the absence of a full BFD API, users will have an option to specify a policy"},{"line_number":307,"context_line":"on the routers (``enable_default_route_bfd``) or routes (by supplying an"},{"line_number":308,"context_line":"additional option to the ``add_extraroutes`` API invocation called"},{"line_number":309,"context_line":"``enable_bfd``). Neutron will manage ``bfd_monitor`` records related to those"},{"line_number":310,"context_line":"options internally for now using route next hops as ``dst_ip`` fields."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Out of Scope"},{"line_number":313,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"12ea247b_9cc56859","line":310,"range":{"start_line":306,"start_character":0,"end_line":310,"end_character":70},"in_reply_to":"23739657_cb257ff6","updated":"2023-02-10 14:04:48.000000000","message":"I think we\u0027ll need an API for this that goes with this spec to have that visibility:\n\nhttps://specs.openstack.org/openstack/neutron-specs/specs/xena/bfd_support.html","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"b804cad80af7494617c6294b1e01e67205331b2e","unresolved":true,"context_lines":[{"line_number":303,"context_line":"Extra routes API: BFD"},{"line_number":304,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":305,"context_line":""},{"line_number":306,"context_line":"In the absence of a full BFD API, users will have an option to specify a policy"},{"line_number":307,"context_line":"on the routers (``enable_default_route_bfd``) or routes (by supplying an"},{"line_number":308,"context_line":"additional option to the ``add_extraroutes`` API invocation called"},{"line_number":309,"context_line":"``enable_bfd``). Neutron will manage ``bfd_monitor`` records related to those"},{"line_number":310,"context_line":"options internally for now using route next hops as ``dst_ip`` fields."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Out of Scope"},{"line_number":313,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"e4aabb75_1a909220","line":310,"range":{"start_line":306,"start_character":0,"end_line":310,"end_character":70},"in_reply_to":"beec59e9_0beb32d1","updated":"2023-02-22 14:13:16.000000000","message":"I agree that not having it exposed completely is a bit challenging for the operators.\n\nAlbeit they can look at the OVN NB records to see some of the state in OVN:\n\nhttps://github.com/ovn-org/ovn/blob/b027731ca445c9f2d841fcab7f791e70ee19a2ac/ovn-nb.ovsschema#L449-L458\nhttps://github.com/ovn-org/ovn/blob/main/ovn-nb.ovsschema#L612-L635","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"0b286004a9c94da07eb7f86d5d86e414f046f88e","unresolved":true,"context_lines":[{"line_number":316,"context_line":"  while it is present in the protocol RFC itself. Therefore, the data model"},{"line_number":317,"context_line":"  should be extensible to support this in the future;"},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"* Solving the distributed SNAT problem."},{"line_number":320,"context_line":"  One direction is to use conntrack state synchronization between the gateway"},{"line_number":321,"context_line":"  ports. Other ideas involve making smarter control plane choices about where"},{"line_number":322,"context_line":"  this conntrack state should exist instead of distributing it"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9236a520_f9e72703","line":319,"range":{"start_line":319,"start_character":2,"end_line":319,"end_character":39},"updated":"2023-03-09 15:06:41.000000000","message":"Sorry, can you describe better this issue? When that happens?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"cde62e5999a7e6d675767ae9d1981c7495ea8e09","unresolved":true,"context_lines":[{"line_number":316,"context_line":"  while it is present in the protocol RFC itself. Therefore, the data model"},{"line_number":317,"context_line":"  should be extensible to support this in the future;"},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"* Solving the distributed SNAT problem."},{"line_number":320,"context_line":"  One direction is to use conntrack state synchronization between the gateway"},{"line_number":321,"context_line":"  ports. Other ideas involve making smarter control plane choices about where"},{"line_number":322,"context_line":"  this conntrack state should exist instead of distributing it"}],"source_content_type":"text/x-rst","patch_set":2,"id":"eb631529_e61003e8","line":319,"range":{"start_line":319,"start_character":2,"end_line":319,"end_character":39},"in_reply_to":"9236a520_f9e72703","updated":"2023-03-09 17:37:55.000000000","message":"Let\u0027s assume a network with one router with `--enable-snat` set and a single gateway port/default route, in that network there is an instance that is located on a hypervisor which is not any of the active chassis for the LRP.\n\nWhen that instance without a FIP, originates traffic to an endpoint outside the cloud, its traffic will be tunneled to the current active GW. On the active GW the packet will be SNAT\u0027ed and committed too conntrack.\n\nThis state in conntrack is required for the return traffic to work.\n\nWhen introducing multiple gateway ports and multiple default routes, we may have a situation where the return traffic comes to a different chassis, which does not have the conntrack state and as a consequence will drop the packet on the floor.\n\nThe immediate path forward for this is to document or forcefully not allow the use of `--enable-snat` together with this feature.\n\nPossible future (out of scope for this spec) solutions could be:\n* Shard the port used for egress SNATted traffic across participating gateways, move the unSNAT operation to the chassis hosting the instance, and make the chassis receiving the traffic tunnel it to the correct chassis based on the port sharding information. This requires work in OVN.\n* Allocate an external IP per hypervisor and distribute the shared SNAT operation. This would be extremely wasteful wrt. IP space and is not desirable.\n* Perform some sort of synchronization of conntrack state between participating gateways. This approach can have scale implications and potential timing/race condition issues.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"0b286004a9c94da07eb7f86d5d86e414f046f88e","unresolved":true,"context_lines":[{"line_number":351,"context_line":"policy on whether to enable `ECMP symmetric reply`_ depending on whether the L3"},{"line_number":352,"context_line":"service plugin supports it or not."},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"However, the `commit introducing the feature`_ in OVN notes a limitation on its"},{"line_number":355,"context_line":"use: it can only be used on gateway routers, not distributed routers that have"},{"line_number":356,"context_line":"a gateway port due to the dependency of the ingress pipeline logic of the"},{"line_number":357,"context_line":"logical router on the hypervisor-local CT state."}],"source_content_type":"text/x-rst","patch_set":2,"id":"637ee55d_079f4de8","line":354,"range":{"start_line":354,"start_character":0,"end_line":354,"end_character":47},"updated":"2023-03-09 15:06:41.000000000","message":"So will this feature be limited to HA and non-DVR routers?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"81ab23a6d6e207a349f26e436c713503fef3e805","unresolved":true,"context_lines":[{"line_number":351,"context_line":"policy on whether to enable `ECMP symmetric reply`_ depending on whether the L3"},{"line_number":352,"context_line":"service plugin supports it or not."},{"line_number":353,"context_line":""},{"line_number":354,"context_line":"However, the `commit introducing the feature`_ in OVN notes a limitation on its"},{"line_number":355,"context_line":"use: it can only be used on gateway routers, not distributed routers that have"},{"line_number":356,"context_line":"a gateway port due to the dependency of the ingress pipeline logic of the"},{"line_number":357,"context_line":"logical router on the hypervisor-local CT state."}],"source_content_type":"text/x-rst","patch_set":2,"id":"3615ba54_b04ab58c","line":354,"range":{"start_line":354,"start_character":0,"end_line":354,"end_character":47},"in_reply_to":"637ee55d_079f4de8","updated":"2023-03-09 17:03:08.000000000","message":"I think this is mentioned in the spec mostly for reference to possible future improvement of the feature.\n\nIn order for OpenStack Neutron OVN to make use of it, we would need to change how the router topology is laid out in OVN. Specifically we would need to use Gateway router as opposed to Logical Routers with Distributed Gateway Ports as we do today.\n\nRefer to the `chassis` option in the `Logical_Router` table for more detail [0].\n\n0: https://manpages.ubuntu.com/manpages/kinetic/man5/ovn-nb.5.html#logical_router%20table","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"8ed49a818932b98cf88276a8539546863aada39e","unresolved":true,"context_lines":[{"line_number":355,"context_line":"use: it can only be used on gateway routers, not distributed routers that have"},{"line_number":356,"context_line":"a gateway port due to the dependency of the ingress pipeline logic of the"},{"line_number":357,"context_line":"logical router on the hypervisor-local CT state."},{"line_number":358,"context_line":""},{"line_number":359,"context_line":".. _multiple external gateways spec: https://specs.openstack.org/openstack/neutron-specs/specs/xena/multiple-external-gateways.html"},{"line_number":360,"context_line":".. _BFD: https://www.rfc-editor.org/rfc/rfc5880"},{"line_number":361,"context_line":".. _as of v21.03.0: https://github.com/ovn-org/ovn/commit/6e0a69ad4bcdf9e4cace5c73ef48ab06065e8519"}],"source_content_type":"text/x-rst","patch_set":2,"id":"cbeb5ee2_0255c76e","line":358,"updated":"2023-02-09 15:03:46.000000000","message":"Just for formality could you please add fields with assignee(s), work items, some words about testing and documentation?","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"e5b415932886cd9dea52402ec7c5eedc273997ed","unresolved":true,"context_lines":[{"line_number":355,"context_line":"use: it can only be used on gateway routers, not distributed routers that have"},{"line_number":356,"context_line":"a gateway port due to the dependency of the ingress pipeline logic of the"},{"line_number":357,"context_line":"logical router on the hypervisor-local CT state."},{"line_number":358,"context_line":""},{"line_number":359,"context_line":".. _multiple external gateways spec: https://specs.openstack.org/openstack/neutron-specs/specs/xena/multiple-external-gateways.html"},{"line_number":360,"context_line":".. _BFD: https://www.rfc-editor.org/rfc/rfc5880"},{"line_number":361,"context_line":".. _as of v21.03.0: https://github.com/ovn-org/ovn/commit/6e0a69ad4bcdf9e4cace5c73ef48ab06065e8519"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bc9e5af4_01f4f533","line":358,"in_reply_to":"cbeb5ee2_0255c76e","updated":"2023-02-10 14:04:48.000000000","message":"Ack, will do.","commit_id":"8ce21d40eda0f8392fb6d4cd5068298e16f3b7ca"}],"specs/2023.2/active-active-l3-gateway-with-multihoming.rst":[{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"82869193e69a963dd17bd7c69cacaa9c5f4988ef","unresolved":true,"context_lines":[{"line_number":125,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":126,"context_line":"the L3 service plugin supports BFD)."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Besides the handling of default routes derived from subnets, the BFD"},{"line_number":129,"context_line":"information needs to be associated with every route object similar to how"},{"line_number":130,"context_line":"it is done in the OVN data model itself. The reason behind this is that BFD"},{"line_number":131,"context_line":"can be enabled or disabled on a per next-hop basis even if the destination"}],"source_content_type":"text/x-rst","patch_set":3,"id":"04d48d9e_9275148c","line":128,"updated":"2023-03-16 07:59:13.000000000","message":"While the contents of this section is very informative, I\u0027d remove the text from line 128 to 196 and save for a future spec where the work is actually planned for implementation within the cycle.\n\nWe\u0027ve agreed to implement ECMP+BFD for default gateways and will not have time to complete the work required for extra routes nor modeling of BFD options.","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"2e69c800a025496299dd632880583c482c553077","unresolved":false,"context_lines":[{"line_number":125,"context_line":"used for checking whether the next-hop of a default route is reachable (if"},{"line_number":126,"context_line":"the L3 service plugin supports BFD)."},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"Besides the handling of default routes derived from subnets, the BFD"},{"line_number":129,"context_line":"information needs to be associated with every route object similar to how"},{"line_number":130,"context_line":"it is done in the OVN data model itself. The reason behind this is that BFD"},{"line_number":131,"context_line":"can be enabled or disabled on a per next-hop basis even if the destination"}],"source_content_type":"text/x-rst","patch_set":3,"id":"f31bbe5c_f998e0aa","line":128,"in_reply_to":"04d48d9e_9275148c","updated":"2023-03-20 09:29:04.000000000","message":"Done","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"82869193e69a963dd17bd7c69cacaa9c5f4988ef","unresolved":true,"context_lines":[{"line_number":210,"context_line":"extended to handle those in the ``router`` API resource (a separate extension"},{"line_number":211,"context_line":"is added per attribute):"},{"line_number":212,"context_line":""},{"line_number":213,"context_line":"* ``enable_default_route_ecmp`` is a router-level policy on whether ECMP "},{"line_number":214,"context_line":"  default routes should be used or not (if the L3 service plugin supports"},{"line_number":215,"context_line":"  them)."},{"line_number":216,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"67db8fbc_3f976772","line":213,"updated":"2023-03-16 07:59:13.000000000","message":"nit: White space at end of line","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"2e69c800a025496299dd632880583c482c553077","unresolved":false,"context_lines":[{"line_number":210,"context_line":"extended to handle those in the ``router`` API resource (a separate extension"},{"line_number":211,"context_line":"is added per attribute):"},{"line_number":212,"context_line":""},{"line_number":213,"context_line":"* ``enable_default_route_ecmp`` is a router-level policy on whether ECMP "},{"line_number":214,"context_line":"  default routes should be used or not (if the L3 service plugin supports"},{"line_number":215,"context_line":"  them)."},{"line_number":216,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"f6821874_13bc8a5f","line":213,"in_reply_to":"67db8fbc_3f976772","updated":"2023-03-20 09:29:04.000000000","message":"Done","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"82869193e69a963dd17bd7c69cacaa9c5f4988ef","unresolved":true,"context_lines":[{"line_number":286,"context_line":"focused on the agent-based implementation - part of the work to implement this"},{"line_number":287,"context_line":"spec is to check whether the same is true for the OVN-based L3 implementation."},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"One parameter that is proposed to be added here is a boolean"},{"line_number":290,"context_line":"``ecmp_symmetric_reply`` which will act a way to populate the relevant new"},{"line_number":291,"context_line":"column for each static route (it is modeled at the route level in OVN as well)."},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"Extra routes API: BFD"},{"line_number":294,"context_line":"^^^^^^^^^^^^^^^^^^^^^"}],"source_content_type":"text/x-rst","patch_set":3,"id":"42902814_e6615f15","line":291,"range":{"start_line":289,"start_character":0,"end_line":291,"end_character":79},"updated":"2023-03-16 07:59:13.000000000","message":"I believe we agreed to drop this because the option is not available to Neutron OVN in its current implementation.","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"2e69c800a025496299dd632880583c482c553077","unresolved":false,"context_lines":[{"line_number":286,"context_line":"focused on the agent-based implementation - part of the work to implement this"},{"line_number":287,"context_line":"spec is to check whether the same is true for the OVN-based L3 implementation."},{"line_number":288,"context_line":""},{"line_number":289,"context_line":"One parameter that is proposed to be added here is a boolean"},{"line_number":290,"context_line":"``ecmp_symmetric_reply`` which will act a way to populate the relevant new"},{"line_number":291,"context_line":"column for each static route (it is modeled at the route level in OVN as well)."},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"Extra routes API: BFD"},{"line_number":294,"context_line":"^^^^^^^^^^^^^^^^^^^^^"}],"source_content_type":"text/x-rst","patch_set":3,"id":"c441930e_0ad9d33f","line":291,"range":{"start_line":289,"start_character":0,"end_line":291,"end_character":79},"in_reply_to":"42902814_e6615f15","updated":"2023-03-20 09:29:04.000000000","message":"Done","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"82869193e69a963dd17bd7c69cacaa9c5f4988ef","unresolved":true,"context_lines":[{"line_number":290,"context_line":"``ecmp_symmetric_reply`` which will act a way to populate the relevant new"},{"line_number":291,"context_line":"column for each static route (it is modeled at the route level in OVN as well)."},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"Extra routes API: BFD"},{"line_number":294,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"In the absence of a full BFD API, users will have an option to specify a policy"}],"source_content_type":"text/x-rst","patch_set":3,"id":"6639260c_a402fbb7","line":293,"updated":"2023-03-16 07:59:13.000000000","message":"As discussed, the target is to implement support for default gateways, please move extra routes to out of scope.","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"2e69c800a025496299dd632880583c482c553077","unresolved":false,"context_lines":[{"line_number":290,"context_line":"``ecmp_symmetric_reply`` which will act a way to populate the relevant new"},{"line_number":291,"context_line":"column for each static route (it is modeled at the route level in OVN as well)."},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"Extra routes API: BFD"},{"line_number":294,"context_line":"^^^^^^^^^^^^^^^^^^^^^"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"In the absence of a full BFD API, users will have an option to specify a policy"}],"source_content_type":"text/x-rst","patch_set":3,"id":"d2fc253c_4b1f5e70","line":293,"in_reply_to":"6639260c_a402fbb7","updated":"2023-03-20 09:29:04.000000000","message":"Done","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"82869193e69a963dd17bd7c69cacaa9c5f4988ef","unresolved":true,"context_lines":[{"line_number":327,"context_line":"* Implementation of BFD for the non-OVN L3 implementation based on Linux"},{"line_number":328,"context_line":"  namespaces."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Asymmetric Routing and Distributed Routers"},{"line_number":331,"context_line":"------------------------------------------"},{"line_number":332,"context_line":""},{"line_number":333,"context_line":"Conntrack can be utilized to avoid responses generated by instances to go via"}],"source_content_type":"text/x-rst","patch_set":3,"id":"e92aa728_bd594ecf","line":330,"updated":"2023-03-16 07:59:13.000000000","message":"This section is informative, but since the feature is currently not available for how Neutron utilizes OVN routers, and we have no intention on implementing it as part of this spec, should we just remove it and save it for a future spec?","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"2e69c800a025496299dd632880583c482c553077","unresolved":false,"context_lines":[{"line_number":327,"context_line":"* Implementation of BFD for the non-OVN L3 implementation based on Linux"},{"line_number":328,"context_line":"  namespaces."},{"line_number":329,"context_line":""},{"line_number":330,"context_line":"Asymmetric Routing and Distributed Routers"},{"line_number":331,"context_line":"------------------------------------------"},{"line_number":332,"context_line":""},{"line_number":333,"context_line":"Conntrack can be utilized to avoid responses generated by instances to go via"}],"source_content_type":"text/x-rst","patch_set":3,"id":"a4887f1a_dcf04895","line":330,"in_reply_to":"e92aa728_bd594ecf","updated":"2023-03-20 09:29:04.000000000","message":"Done","commit_id":"7c983c1a10a2ff57aa1fe9e2c5687821117995eb"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"82bac311b360913d22f91af6993795879d0ecce0","unresolved":true,"context_lines":[{"line_number":95,"context_line":"  but start storing multiple ``network:router_gateway`` ports per router in the"},{"line_number":96,"context_line":"  ``routerports`` table;"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"* For backwards-compatibility store a single gateway port id in"},{"line_number":99,"context_line":"  ``routers.gw_port_id`` which will be present both in this field and also in"},{"line_number":100,"context_line":"  the ``routerports`` table and in ``routers.gw_port_id``)."},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Extend the ``neutron.db.models.l3.Router`` class with a new attribute"},{"line_number":103,"context_line":"  ``gw_ports`` that will map to all relevant ``network:router_gateway`` ports"}],"source_content_type":"text/x-rst","patch_set":6,"id":"e9e17348_f303bae2","line":100,"range":{"start_line":98,"start_character":0,"end_line":100,"end_character":59},"updated":"2023-03-21 07:49:51.000000000","message":"I have difficulty to understand the phrasing but it\u0027s probably me :-)\n\nI guess this part will be the \"tricky one\" as we want to keep gw_port_id consistent regarding API and other agents","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"b36284f39effd04d17ca9c10b2f660b7e9eb4542","unresolved":false,"context_lines":[{"line_number":95,"context_line":"  but start storing multiple ``network:router_gateway`` ports per router in the"},{"line_number":96,"context_line":"  ``routerports`` table;"},{"line_number":97,"context_line":""},{"line_number":98,"context_line":"* For backwards-compatibility store a single gateway port id in"},{"line_number":99,"context_line":"  ``routers.gw_port_id`` which will be present both in this field and also in"},{"line_number":100,"context_line":"  the ``routerports`` table and in ``routers.gw_port_id``)."},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Extend the ``neutron.db.models.l3.Router`` class with a new attribute"},{"line_number":103,"context_line":"  ``gw_ports`` that will map to all relevant ``network:router_gateway`` ports"}],"source_content_type":"text/x-rst","patch_set":6,"id":"8fb68625_936bf1df","line":100,"range":{"start_line":98,"start_character":0,"end_line":100,"end_character":59},"in_reply_to":"e9e17348_f303bae2","updated":"2023-03-22 10:14:41.000000000","message":"Done","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"82bac311b360913d22f91af6993795879d0ecce0","unresolved":true,"context_lines":[{"line_number":99,"context_line":"  ``routers.gw_port_id`` which will be present both in this field and also in"},{"line_number":100,"context_line":"  the ``routerports`` table and in ``routers.gw_port_id``)."},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Extend the ``neutron.db.models.l3.Router`` class with a new attribute"},{"line_number":103,"context_line":"  ``gw_ports`` that will map to all relevant ``network:router_gateway`` ports"},{"line_number":104,"context_line":"  stored in the ``routerports`` table."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"BFD and ECMP Route Behavior Modeling"},{"line_number":107,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}],"source_content_type":"text/x-rst","patch_set":6,"id":"46d70965_a39607e2","line":104,"range":{"start_line":102,"start_character":2,"end_line":104,"end_character":38},"updated":"2023-03-21 07:49:51.000000000","message":"Is there a necessity of ordering the ports returned?","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"b36284f39effd04d17ca9c10b2f660b7e9eb4542","unresolved":true,"context_lines":[{"line_number":99,"context_line":"  ``routers.gw_port_id`` which will be present both in this field and also in"},{"line_number":100,"context_line":"  the ``routerports`` table and in ``routers.gw_port_id``)."},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Extend the ``neutron.db.models.l3.Router`` class with a new attribute"},{"line_number":103,"context_line":"  ``gw_ports`` that will map to all relevant ``network:router_gateway`` ports"},{"line_number":104,"context_line":"  stored in the ``routerports`` table."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"BFD and ECMP Route Behavior Modeling"},{"line_number":107,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}],"source_content_type":"text/x-rst","patch_set":6,"id":"ffc43fcb_07623d9e","line":104,"range":{"start_line":102,"start_character":2,"end_line":104,"end_character":38},"in_reply_to":"46d70965_a39607e2","updated":"2023-03-22 10:14:41.000000000","message":"As laid out in the Router API section below, the API response for the ``external_gateways`` element [0] will indeed be ordered to list the compatibility gateway first [1].\n\n0: https://review.opendev.org/c/openstack/neutron-specs/+/870030/7/specs/2023.2/active-active-l3-gateway-with-multihoming.rst#192\n1: https://review.opendev.org/c/openstack/neutron/+/873593/23/neutron/db/l3_extra_gws_db.py#83","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"34171d4c5a2064bf98b7e79c157c287dc3495881","unresolved":false,"context_lines":[{"line_number":99,"context_line":"  ``routers.gw_port_id`` which will be present both in this field and also in"},{"line_number":100,"context_line":"  the ``routerports`` table and in ``routers.gw_port_id``)."},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"* Extend the ``neutron.db.models.l3.Router`` class with a new attribute"},{"line_number":103,"context_line":"  ``gw_ports`` that will map to all relevant ``network:router_gateway`` ports"},{"line_number":104,"context_line":"  stored in the ``routerports`` table."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"BFD and ECMP Route Behavior Modeling"},{"line_number":107,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"}],"source_content_type":"text/x-rst","patch_set":6,"id":"ed8e7b5a_cb20db4a","line":104,"range":{"start_line":102,"start_character":2,"end_line":104,"end_character":38},"in_reply_to":"ffc43fcb_07623d9e","updated":"2023-03-29 08:52:34.000000000","message":"Done","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"5fc4ce191236d4946be47469d5dc9ba953d700a5","unresolved":true,"context_lines":[{"line_number":140,"context_line":""},{"line_number":141,"context_line":"* ``dst_ip`` - a BFD peer IP address;"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"* ``mix_tx`` - an integer specifying the minimum interval (milliseconds, \u003e\u003d1)"},{"line_number":144,"context_line":"  that OVN would use when transmitting the BFD control packet (minus jitter);"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"* ``min_rx`` - an integer specifying the minimum interval (milliseconds)"}],"source_content_type":"text/x-rst","patch_set":6,"id":"96c94991_63aa5749","line":143,"range":{"start_line":143,"start_character":4,"end_line":143,"end_character":10},"updated":"2023-03-20 18:33:12.000000000","message":"Small typo: min_tx","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"7092011acb257eac7605e802419d6d70a7071d09","unresolved":false,"context_lines":[{"line_number":140,"context_line":""},{"line_number":141,"context_line":"* ``dst_ip`` - a BFD peer IP address;"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"* ``mix_tx`` - an integer specifying the minimum interval (milliseconds, \u003e\u003d1)"},{"line_number":144,"context_line":"  that OVN would use when transmitting the BFD control packet (minus jitter);"},{"line_number":145,"context_line":""},{"line_number":146,"context_line":"* ``min_rx`` - an integer specifying the minimum interval (milliseconds)"}],"source_content_type":"text/x-rst","patch_set":6,"id":"0e56208f_bdee4b5b","line":143,"range":{"start_line":143,"start_character":4,"end_line":143,"end_character":10},"in_reply_to":"96c94991_63aa5749","updated":"2023-03-22 09:55:16.000000000","message":"Done","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"82bac311b360913d22f91af6993795879d0ecce0","unresolved":true,"context_lines":[{"line_number":211,"context_line":"chooses an existing extra gateway as a replacement for the compatibility"},{"line_number":212,"context_line":"gateway instead."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"The ``external_gateways`` attribute cannot be set in"},{"line_number":215,"context_line":"``POST /v2.0/routers`` or ``PUT /v2.0/routers/{router_id}`` requests,"},{"line_number":216,"context_line":"instead it can be managed via sub-methods:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"* ``PUT /v2.0/routers/{router_id}/add_external_gateways``"},{"line_number":219,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"5e39ef16_d4f38a39","line":216,"range":{"start_line":214,"start_character":0,"end_line":216,"end_character":42},"updated":"2023-03-21 07:49:51.000000000","message":"It\u0027s probably a noob question but why you can\u0027t?","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"b36284f39effd04d17ca9c10b2f660b7e9eb4542","unresolved":true,"context_lines":[{"line_number":211,"context_line":"chooses an existing extra gateway as a replacement for the compatibility"},{"line_number":212,"context_line":"gateway instead."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"The ``external_gateways`` attribute cannot be set in"},{"line_number":215,"context_line":"``POST /v2.0/routers`` or ``PUT /v2.0/routers/{router_id}`` requests,"},{"line_number":216,"context_line":"instead it can be managed via sub-methods:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"* ``PUT /v2.0/routers/{router_id}/add_external_gateways``"},{"line_number":219,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"f1d8e68b_50510a6b","line":216,"range":{"start_line":214,"start_character":0,"end_line":216,"end_character":42},"in_reply_to":"5e39ef16_d4f38a39","updated":"2023-03-22 10:14:41.000000000","message":"I think the reasoning is that we\u0027re adding this capability to the API while keeping compatibility with the existing API.\n\nWhat if someone were to PUT/POST a request that updates both the `external_gateway_info` and the `external_gateways`?","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"713c34ab45612ca1a8322e8f5001dd660611336f","unresolved":true,"context_lines":[{"line_number":211,"context_line":"chooses an existing extra gateway as a replacement for the compatibility"},{"line_number":212,"context_line":"gateway instead."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"The ``external_gateways`` attribute cannot be set in"},{"line_number":215,"context_line":"``POST /v2.0/routers`` or ``PUT /v2.0/routers/{router_id}`` requests,"},{"line_number":216,"context_line":"instead it can be managed via sub-methods:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"* ``PUT /v2.0/routers/{router_id}/add_external_gateways``"},{"line_number":219,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"f53be410_4d818b82","line":216,"range":{"start_line":214,"start_character":0,"end_line":216,"end_character":42},"in_reply_to":"f1d8e68b_50510a6b","updated":"2023-03-23 16:27:59.000000000","message":"I think it also could be race prone and if 2 clients would make requests updating this field it could be not updated properly. We have seen similar issue with extra routes field.","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"34171d4c5a2064bf98b7e79c157c287dc3495881","unresolved":false,"context_lines":[{"line_number":211,"context_line":"chooses an existing extra gateway as a replacement for the compatibility"},{"line_number":212,"context_line":"gateway instead."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"The ``external_gateways`` attribute cannot be set in"},{"line_number":215,"context_line":"``POST /v2.0/routers`` or ``PUT /v2.0/routers/{router_id}`` requests,"},{"line_number":216,"context_line":"instead it can be managed via sub-methods:"},{"line_number":217,"context_line":""},{"line_number":218,"context_line":"* ``PUT /v2.0/routers/{router_id}/add_external_gateways``"},{"line_number":219,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"4906872f_975c0b8e","line":216,"range":{"start_line":214,"start_character":0,"end_line":216,"end_character":42},"in_reply_to":"f53be410_4d818b82","updated":"2023-03-29 08:52:34.000000000","message":"Done","commit_id":"5d72a935a960bf1106b863af6a392162a5b4b212"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"713c34ab45612ca1a8322e8f5001dd660611336f","unresolved":true,"context_lines":[{"line_number":209,"context_line":"unchanged.  Setting ``external_gateway_info`` to an empty value removes a"},{"line_number":210,"context_line":"single (compatibility) gateway from the set of gateway ports of a router and"},{"line_number":211,"context_line":"chooses an existing extra gateway as a replacement for the compatibility"},{"line_number":212,"context_line":"gateway instead."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"The ``external_gateways`` attribute cannot be set in"},{"line_number":215,"context_line":"``POST /v2.0/routers`` or ``PUT /v2.0/routers/{router_id}`` requests,"}],"source_content_type":"text/x-rst","patch_set":8,"id":"6069b858_bcf0bb89","line":212,"updated":"2023-03-23 16:27:59.000000000","message":"I\u0027m not sure I understand correctly. So if user will update router and set current field \"external_gateway_info\" to be empty, Neutron will remove that gateway_info from the new \"external_gateways\" list (element 0 in that list) and set new element \"0\" as \"external_gateway_info\". Correct?","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"9ae7bfc1d4031270c3f08bec86b984ec118e3391","unresolved":true,"context_lines":[{"line_number":209,"context_line":"unchanged.  Setting ``external_gateway_info`` to an empty value removes a"},{"line_number":210,"context_line":"single (compatibility) gateway from the set of gateway ports of a router and"},{"line_number":211,"context_line":"chooses an existing extra gateway as a replacement for the compatibility"},{"line_number":212,"context_line":"gateway instead."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"The ``external_gateways`` attribute cannot be set in"},{"line_number":215,"context_line":"``POST /v2.0/routers`` or ``PUT /v2.0/routers/{router_id}`` requests,"}],"source_content_type":"text/x-rst","patch_set":8,"id":"669f5701_43cb1d9f","line":212,"in_reply_to":"27b78b50_f1e95158","updated":"2023-03-28 12:12:09.000000000","message":"ok, that sounds good to me","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"301a08e8feb31bd76effc0b944863df3c7a9d4b4","unresolved":true,"context_lines":[{"line_number":209,"context_line":"unchanged.  Setting ``external_gateway_info`` to an empty value removes a"},{"line_number":210,"context_line":"single (compatibility) gateway from the set of gateway ports of a router and"},{"line_number":211,"context_line":"chooses an existing extra gateway as a replacement for the compatibility"},{"line_number":212,"context_line":"gateway instead."},{"line_number":213,"context_line":""},{"line_number":214,"context_line":"The ``external_gateways`` attribute cannot be set in"},{"line_number":215,"context_line":"``POST /v2.0/routers`` or ``PUT /v2.0/routers/{router_id}`` requests,"}],"source_content_type":"text/x-rst","patch_set":8,"id":"27b78b50_f1e95158","line":212,"in_reply_to":"6069b858_bcf0bb89","updated":"2023-03-23 22:36:27.000000000","message":"Yes, the idea here is that so long as there are extra gateway ports they can take the place of the compatibility gw port in case it gets removed.\n\nSee https://review.opendev.org/c/openstack/neutron/+/873593/24/neutron/db/l3_extra_gws_db.py#485\n\nI don\u0027t see much use in having extra ports without a compatibility port to be honest - it just adds an odd state to the mix.\n\nAnd if a user somehow ended up with multiple gateway ports in the first place, it likely means that there\u0027s awareness of an operator or automation of having multiple ports gateway ports.\n\nThe compatibility API works with a single external_gateway_info dict so it\u0027s targeted at working with a single gateway port as far as updating or removing it.\n\nBut maybe I am missing something here - if there\u0027s any concern about this behavior I\u0027m open to discuss it.","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"301a08e8feb31bd76effc0b944863df3c7a9d4b4","unresolved":true,"context_lines":[{"line_number":228,"context_line":"  Accepts a list of ``external_gateway_info`` structures.  The external"},{"line_number":229,"context_line":"  gateways to be updated are identified by the ``network_id`` field and"},{"line_number":230,"context_line":"  ``external_fixed_ips`` found in the PUT request. Updating ``enable_snat`` is"},{"line_number":231,"context_line":"  only possible at the per-router basis on the first item specified. Updating"},{"line_number":232,"context_line":"  ``external_fixed_ips`` is possible without recreating a port only if there"},{"line_number":233,"context_line":"  is a single port attached to a given ``network_id`` - in other cases it will"},{"line_number":234,"context_line":"  cause a port to be recreated with the new set of IPs."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"* ``PUT /v2.0/routers/{router_id}/remove_external_gateways``"}],"source_content_type":"text/x-rst","patch_set":8,"id":"17f69cc5_c48207c1","line":233,"range":{"start_line":231,"start_character":69,"end_line":233,"end_character":53},"updated":"2023-03-23 22:36:27.000000000","message":"I actually found a way to avoid recreating ports even if there are multiple ports attached to the same network so I am going to drop this comment.","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"923612d99854742ab977a346681bfb9d0b0c2911","unresolved":false,"context_lines":[{"line_number":228,"context_line":"  Accepts a list of ``external_gateway_info`` structures.  The external"},{"line_number":229,"context_line":"  gateways to be updated are identified by the ``network_id`` field and"},{"line_number":230,"context_line":"  ``external_fixed_ips`` found in the PUT request. Updating ``enable_snat`` is"},{"line_number":231,"context_line":"  only possible at the per-router basis on the first item specified. Updating"},{"line_number":232,"context_line":"  ``external_fixed_ips`` is possible without recreating a port only if there"},{"line_number":233,"context_line":"  is a single port attached to a given ``network_id`` - in other cases it will"},{"line_number":234,"context_line":"  cause a port to be recreated with the new set of IPs."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"* ``PUT /v2.0/routers/{router_id}/remove_external_gateways``"}],"source_content_type":"text/x-rst","patch_set":8,"id":"4c0bdcc1_88a2750e","line":233,"range":{"start_line":231,"start_character":69,"end_line":233,"end_character":53},"in_reply_to":"17f69cc5_c48207c1","updated":"2023-03-27 13:31:25.000000000","message":"Done","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"713c34ab45612ca1a8322e8f5001dd660611336f","unresolved":true,"context_lines":[{"line_number":231,"context_line":"  only possible at the per-router basis on the first item specified. Updating"},{"line_number":232,"context_line":"  ``external_fixed_ips`` is possible without recreating a port only if there"},{"line_number":233,"context_line":"  is a single port attached to a given ``network_id`` - in other cases it will"},{"line_number":234,"context_line":"  cause a port to be recreated with the new set of IPs."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"* ``PUT /v2.0/routers/{router_id}/remove_external_gateways``"},{"line_number":237,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"397d085b_dee2f87b","line":234,"updated":"2023-03-23 16:27:59.000000000","message":"is this really necessary? Shouldn\u0027t add and remove be enough?","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"301a08e8feb31bd76effc0b944863df3c7a9d4b4","unresolved":true,"context_lines":[{"line_number":231,"context_line":"  only possible at the per-router basis on the first item specified. Updating"},{"line_number":232,"context_line":"  ``external_fixed_ips`` is possible without recreating a port only if there"},{"line_number":233,"context_line":"  is a single port attached to a given ``network_id`` - in other cases it will"},{"line_number":234,"context_line":"  cause a port to be recreated with the new set of IPs."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"* ``PUT /v2.0/routers/{router_id}/remove_external_gateways``"},{"line_number":237,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"c07013c5_5afd855f","line":234,"in_reply_to":"397d085b_dee2f87b","updated":"2023-03-23 22:36:27.000000000","message":"Generally its needed to have a way to specify a particular desired state regardless of the current number of gateway ports, for example: create 2 new gateway ports, remove one, leave the rest alone. It\u0027s probably easier for an operator to do that than having to assess the current state themselves and then execute a set of add and remove API calls.\n\nThere are a couple of special cases in that use-case:\n\n1) Removing all gateways: much like specifying an empty external_gateway_info in the compat API for a single gateway, we can specify an empty list and clear everything rather than having to read the current state and passing it to the remove API;\nhttps://review.opendev.org/c/openstack/neutron/+/873593/24/neutron/db/l3_extra_gws_db.py#404\n\n2) Updating the external_fixed_ips on a given gateway port.\nhttps://review.opendev.org/c/openstack/neutron/+/873593/24/neutron/db/l3_extra_gws_db.py#446\n\n3) Reordering gateway ports (making a different port to be a special compatibility port)\nhttps://review.opendev.org/c/openstack/neutron/+/873593/24/neutron/db/l3_extra_gws_db.py#420\n\nI took the idea for this method from a prior spec but the implementation made me think a bit due to the need to have quite an involved logic to actually allow for this:\n\nhttps://review.opendev.org/c/openstack/neutron/+/873593/24/neutron/db/l3_extra_gws_db.py#402\nhttps://review.opendev.org/c/openstack/neutron/+/873593/24/neutron/db/l3_extra_gws_db.py#190\n\nThe reason the prior spec didn\u0027t allow for multiple gateway ports attached to the same network was probably due to the complexity in mapping the request data to an actual gateway port ID (since the API is network-oriented) when doing updates.","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"9ae7bfc1d4031270c3f08bec86b984ec118e3391","unresolved":true,"context_lines":[{"line_number":231,"context_line":"  only possible at the per-router basis on the first item specified. Updating"},{"line_number":232,"context_line":"  ``external_fixed_ips`` is possible without recreating a port only if there"},{"line_number":233,"context_line":"  is a single port attached to a given ``network_id`` - in other cases it will"},{"line_number":234,"context_line":"  cause a port to be recreated with the new set of IPs."},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"* ``PUT /v2.0/routers/{router_id}/remove_external_gateways``"},{"line_number":237,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"8ac65d5f_3b5816b9","line":234,"in_reply_to":"c07013c5_5afd855f","updated":"2023-03-28 12:12:09.000000000","message":"ok","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"713c34ab45612ca1a8322e8f5001dd660611336f","unresolved":true,"context_lines":[{"line_number":260,"context_line":"on the routers (``enable_default_route_bfd``)."},{"line_number":261,"context_line":""},{"line_number":262,"context_line":""},{"line_number":263,"context_line":"OVN driver Changes"},{"line_number":264,"context_line":"------------------"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"In general, we will update the existing OVN driver to handle the presence of"}],"source_content_type":"text/x-rst","patch_set":8,"id":"52135f1f_9d1f3ca5","line":263,"range":{"start_line":263,"start_character":11,"end_line":263,"end_character":18},"updated":"2023-03-23 16:27:59.000000000","message":"nitty nit: should start with lowercase","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":24824,"name":"Dmitrii Shcherbakov","username":"dmitriis"},"change_message_id":"301a08e8feb31bd76effc0b944863df3c7a9d4b4","unresolved":true,"context_lines":[{"line_number":260,"context_line":"on the routers (``enable_default_route_bfd``)."},{"line_number":261,"context_line":""},{"line_number":262,"context_line":""},{"line_number":263,"context_line":"OVN driver Changes"},{"line_number":264,"context_line":"------------------"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"In general, we will update the existing OVN driver to handle the presence of"}],"source_content_type":"text/x-rst","patch_set":8,"id":"ed1672e6_f436e4cc","line":263,"range":{"start_line":263,"start_character":11,"end_line":263,"end_character":18},"in_reply_to":"52135f1f_9d1f3ca5","updated":"2023-03-23 22:36:27.000000000","message":"I mainly use the title case (https://en.wikipedia.org/wiki/Title_case) but probably it got mixed when one of us was making changes.\n\nI\u0027ll update it to be consistent across the spec.","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"},{"author":{"_account_id":13686,"name":"Frode Nordahl","email":"fnordahl@ubuntu.com","username":"fnordahl"},"change_message_id":"923612d99854742ab977a346681bfb9d0b0c2911","unresolved":false,"context_lines":[{"line_number":260,"context_line":"on the routers (``enable_default_route_bfd``)."},{"line_number":261,"context_line":""},{"line_number":262,"context_line":""},{"line_number":263,"context_line":"OVN driver Changes"},{"line_number":264,"context_line":"------------------"},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"In general, we will update the existing OVN driver to handle the presence of"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3ffe2357_da0b3888","line":263,"range":{"start_line":263,"start_character":11,"end_line":263,"end_character":18},"in_reply_to":"ed1672e6_f436e4cc","updated":"2023-03-27 13:31:25.000000000","message":"Done","commit_id":"c7600fbd8cb931a1ad04bf33431c3c133b603bda"}]}