)]}'
{"specs/version1.0/active-active-l3-distributor.rst":[{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"ef117d05715ba7c5bdb9411d9a1e03aad738c15a","unresolved":false,"context_lines":[{"line_number":60,"context_line":"* Each amphora will run a *bgp speaker* in the default namespace in order to"},{"line_number":61,"context_line":"  announce the anycast VIP into the L3 fabric. BGP peering and announcements"},{"line_number":62,"context_line":"  will occur over the lb-mgmt-net network. The anycast VIP will get advertised"},{"line_number":63,"context_line":"  as a /32 or /128 route with a next-hop of the back-end (tenant network) IP"},{"line_number":64,"context_line":"  assigned to the amphora instance. The back-end network IPs must be directly"},{"line_number":65,"context_line":"  routable from the L3 fabric, such as in the provider networking model."},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1f1a1f67_fe578c01","line":63,"range":{"start_line":63,"start_character":48,"end_line":63,"end_character":56},"updated":"2017-09-21 21:00:15.000000000","message":"Here we mean the frontend network of the load balancer, the VIP network right?  I think in other parts of the document (diagram) this is refereed to as the anycast network.","commit_id":"c3588573a15703a6535062282e247d5aa56d0271"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"ef117d05715ba7c5bdb9411d9a1e03aad738c15a","unresolved":false,"context_lines":[{"line_number":218,"context_line":"    ID of the distributor responsible for distributing traffic for the"},{"line_number":219,"context_line":"    corresponding VIP."},{"line_number":220,"context_line":""},{"line_number":221,"context_line":"Add table ``tenant_network`` with the following columns:"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"* id ``(String(36) , nullable\u003dFalse)``"},{"line_number":224,"context_line":"    ID representing an attached interface to an amphora instance."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1f1a1f67_a1cc5961","line":221,"range":{"start_line":221,"start_character":13,"end_line":221,"end_character":24},"updated":"2017-09-21 21:00:15.000000000","message":"frontend network","commit_id":"c3588573a15703a6535062282e247d5aa56d0271"},{"author":{"_account_id":10850,"name":"German Eichberger","email":"german.eichberger@gmail.com","username":"german"},"change_message_id":"1693cc1e488a8b15610192c8fec0a651dee3fc0e","unresolved":false,"context_lines":[{"line_number":19,"context_line":"This blueprint describes a *L3 active-active* distributor implementation to"},{"line_number":20,"context_line":"support the Octavia *active-active-topology*. The *L3 active-active*"},{"line_number":21,"context_line":"distributor will leverage the capabilities of a layer 3 clos network fabric in"},{"line_number":22,"context_line":"order to distribute traffic to an *Amphora Cluster* of 2 or more amphoras."},{"line_number":23,"context_line":"Specifically, the *L3 active-active* distributor design will leverage Equal"},{"line_number":24,"context_line":"Cost Multipath Load Sharing (ECMP) with anycast routing to achieve traffic"},{"line_number":25,"context_line":"distribution across the *Amphora Cluster*.  In this reference implementation,"}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f2577a7_da3dd4dd","line":22,"range":{"start_line":22,"start_character":55,"end_line":22,"end_character":56},"updated":"2017-09-14 19:46:51.000000000","message":"NIT: maybe allow 1 amphora for test reasons","commit_id":"abac5a3352d439405affe51beb86f744fff23fae"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":19,"context_line":"This blueprint describes a *L3 active-active* distributor implementation to"},{"line_number":20,"context_line":"support the Octavia *active-active-topology*. The *L3 active-active*"},{"line_number":21,"context_line":"distributor will leverage the capabilities of a layer 3 clos network fabric in"},{"line_number":22,"context_line":"order to distribute traffic to an *Amphora Cluster* of 2 or more amphoras."},{"line_number":23,"context_line":"Specifically, the *L3 active-active* distributor design will leverage Equal"},{"line_number":24,"context_line":"Cost Multipath Load Sharing (ECMP) with anycast routing to achieve traffic"},{"line_number":25,"context_line":"distribution across the *Amphora Cluster*.  In this reference implementation,"}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f2577a7_142b2730","line":22,"range":{"start_line":22,"start_character":55,"end_line":22,"end_character":56},"in_reply_to":"5f2577a7_da3dd4dd","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"abac5a3352d439405affe51beb86f744fff23fae"},{"author":{"_account_id":10850,"name":"German Eichberger","email":"german.eichberger@gmail.com","username":"german"},"change_message_id":"1693cc1e488a8b15610192c8fec0a651dee3fc0e","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* The operator shall be able to define a *bgp peer profile*, which includes the"},{"line_number":74,"context_line":"  required metadata for the amphora to establish a bgp peering session with"},{"line_number":75,"context_line":"  the L3 fabric. The bgp peering information will be passed into the"},{"line_number":76,"context_line":"  amphora-agent configuration file via config drive during boot. Once booted,"},{"line_number":77,"context_line":"  the amphora will use the bgp peering information to establish a BGP peer and"},{"line_number":78,"context_line":"  announce its anycast VIP."},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f2577a7_953f1fb3","line":76,"updated":"2017-09-14 19:46:51.000000000","message":"We discussed to maybe establish at a later time (e.g. listener creation) so make a bit more vague...","commit_id":"abac5a3352d439405affe51beb86f744fff23fae"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* The operator shall be able to define a *bgp peer profile*, which includes the"},{"line_number":74,"context_line":"  required metadata for the amphora to establish a bgp peering session with"},{"line_number":75,"context_line":"  the L3 fabric. The bgp peering information will be passed into the"},{"line_number":76,"context_line":"  amphora-agent configuration file via config drive during boot. Once booted,"},{"line_number":77,"context_line":"  the amphora will use the bgp peering information to establish a BGP peer and"},{"line_number":78,"context_line":"  announce its anycast VIP."},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f2577a7_f4e89be2","line":76,"in_reply_to":"5f2577a7_953f1fb3","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"abac5a3352d439405affe51beb86f744fff23fae"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"8deae39632c8fb8740b70bcd6e8089788e9fa36e","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* The operator shall be able to define a *bgp peer profile*, which includes the"},{"line_number":74,"context_line":"  required metadata for the amphora to establish a bgp peering session with"},{"line_number":75,"context_line":"  the L3 fabric. The bgp peering information will be passed into the"},{"line_number":76,"context_line":"  amphora-agent configuration file via config drive during boot. Once booted,"},{"line_number":77,"context_line":"  the amphora will use the bgp peering information to establish a BGP peer and"},{"line_number":78,"context_line":"  announce its anycast VIP."},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"5f2577a7_17cfa9b0","line":76,"in_reply_to":"5f2577a7_953f1fb3","updated":"2017-09-15 05:47:09.000000000","message":"Yes, good point.  We don\u0027t want it advertising as a good amp until it is able to serve content. Listener creation is the first time HAProxy is started, so probably appropriate, but may be early in some cases.","commit_id":"abac5a3352d439405affe51beb86f744fff23fae"}],"specs/version1.1/active-active-l3-distributor.rst":[{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":11,"context_line":"  Please review the active-active topology blueprint first ("},{"line_number":12,"context_line":"  :doc:`../version0.9/active-active-topology` )"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"https://blueprints.launchpad.net/octavia/+spec/l3-active-active"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Problem description"},{"line_number":17,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_eac9c258","line":14,"range":{"start_line":14,"start_character":0,"end_line":14,"end_character":63},"updated":"2017-09-15 22:44:35.000000000","message":"nit: We did not migrate blueprints to storyboard so we should open a storyboard \"story\" for this.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This blueprint describes a *L3 active-active* distributor implementation to"},{"line_number":20,"context_line":"support the Octavia *active-active-topology*. The *L3 active-active*"},{"line_number":21,"context_line":"distributor will leverage the capabilities of a layer 3 clos network fabric in"},{"line_number":22,"context_line":"order to distribute traffic to an *Amphora Cluster* of 1 or more amphoras."},{"line_number":23,"context_line":"Specifically, the *L3 active-active* distributor design will leverage Equal"},{"line_number":24,"context_line":"Cost Multipath Load Sharing (ECMP) with anycast routing to achieve traffic"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_aade1ad7","line":21,"range":{"start_line":21,"start_character":56,"end_line":21,"end_character":60},"updated":"2017-09-15 22:44:35.000000000","message":"nit: Clos\n(Charles Clos)","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This blueprint describes a *L3 active-active* distributor implementation to"},{"line_number":20,"context_line":"support the Octavia *active-active-topology*. The *L3 active-active*"},{"line_number":21,"context_line":"distributor will leverage the capabilities of a layer 3 clos network fabric in"},{"line_number":22,"context_line":"order to distribute traffic to an *Amphora Cluster* of 1 or more amphoras."},{"line_number":23,"context_line":"Specifically, the *L3 active-active* distributor design will leverage Equal"},{"line_number":24,"context_line":"Cost Multipath Load Sharing (ECMP) with anycast routing to achieve traffic"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_80069fcd","line":21,"range":{"start_line":21,"start_character":56,"end_line":21,"end_character":60},"in_reply_to":"5f2577a7_aade1ad7","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet and"},{"line_number":68,"context_line":"  front-end subnet (provider network) when creating a new load balancer. The"},{"line_number":69,"context_line":"  amphora will have ports on three networks (anycast, front-end, management)."},{"line_number":70,"context_line":"  The anycast VIP will get configured on the loopback interface inside the"},{"line_number":71,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":72,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_ad44f447","line":69,"range":{"start_line":69,"start_character":2,"end_line":69,"end_character":43},"updated":"2017-09-15 22:44:35.000000000","message":"nit: are these truly neutron ports, or is one really an allowed address pair on the front-end port?","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet and"},{"line_number":68,"context_line":"  front-end subnet (provider network) when creating a new load balancer. The"},{"line_number":69,"context_line":"  amphora will have ports on three networks (anycast, front-end, management)."},{"line_number":70,"context_line":"  The anycast VIP will get configured on the loopback interface inside the"},{"line_number":71,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":72,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_a0151be7","line":69,"range":{"start_line":69,"start_character":2,"end_line":69,"end_character":43},"in_reply_to":"5f2577a7_ad44f447","updated":"2017-09-20 01:20:28.000000000","message":"Current model creates three ports for the amp (VIP, VRRP, and management).  L3 Anycast model creates three ports as well (Anycast VIP, front-end (VRRP), and management.  The only difference is the networks.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":242,"context_line":"  +------------------------------------------------+"},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"  +------------------------------------------------+"},{"line_number":245,"context_line":"  |      Register Amphora with L3 Distributor      |"},{"line_number":246,"context_line":"  +------------------------------------------------+"},{"line_number":247,"context_line":"  |                                                |"},{"line_number":248,"context_line":"  | +---------------+            +---------------+ |"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_4dceb02d","line":245,"range":{"start_line":245,"start_character":9,"end_line":245,"end_character":46},"updated":"2017-09-15 22:44:35.000000000","message":"Listener start time?  or register amp","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":242,"context_line":"  +------------------------------------------------+"},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"  +------------------------------------------------+"},{"line_number":245,"context_line":"  |      Register Amphora with L3 Distributor      |"},{"line_number":246,"context_line":"  +------------------------------------------------+"},{"line_number":247,"context_line":"  |                                                |"},{"line_number":248,"context_line":"  | +---------------+            +---------------+ |"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_2086cb14","line":245,"range":{"start_line":245,"start_character":9,"end_line":245,"end_character":46},"in_reply_to":"5f2577a7_4dceb02d","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":267,"context_line":"  +------------------------------------------------+"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"  +------------------------------------------------+"},{"line_number":270,"context_line":"  |     Unregister Amphora with L3 Distributor     |"},{"line_number":271,"context_line":"  +------------------------------------------------+"},{"line_number":272,"context_line":"  |                                                |"},{"line_number":273,"context_line":"  | +---------------+            +---------------+ |"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_8000df02","line":270,"range":{"start_line":270,"start_character":8,"end_line":270,"end_character":46},"updated":"2017-09-15 22:44:35.000000000","message":"Or listener stop","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":267,"context_line":"  +------------------------------------------------+"},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"  +------------------------------------------------+"},{"line_number":270,"context_line":"  |     Unregister Amphora with L3 Distributor     |"},{"line_number":271,"context_line":"  +------------------------------------------------+"},{"line_number":272,"context_line":"  |                                                |"},{"line_number":273,"context_line":"  | +---------------+            +---------------+ |"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_a0719b46","line":270,"range":{"start_line":270,"start_character":8,"end_line":270,"end_character":46},"in_reply_to":"5f2577a7_8000df02","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":305,"context_line":"   its anycast VIP into the network with a next-hop of its front-end IP"},{"line_number":306,"context_line":"   address."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"5. A register amphora request will be sent to the amphora (BGP Speaker) in"},{"line_number":309,"context_line":"   order to announce its VIP route. The BGP speaker will communicate using the"},{"line_number":310,"context_line":"   BGP protocol and send a BGP \"announce\" message to the ToR switch."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"6. The ToR switch will learn this new route and advertise it into the L3"},{"line_number":313,"context_line":"   fabric. At this point the L3 fabric will know of the new VIP route and how"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_2d3da4dc","line":310,"range":{"start_line":308,"start_character":3,"end_line":310,"end_character":68},"updated":"2017-09-15 22:44:35.000000000","message":"This should also be updated to reflect the announce will start with listener create in one use case.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":305,"context_line":"   its anycast VIP into the network with a next-hop of its front-end IP"},{"line_number":306,"context_line":"   address."},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"5. A register amphora request will be sent to the amphora (BGP Speaker) in"},{"line_number":309,"context_line":"   order to announce its VIP route. The BGP speaker will communicate using the"},{"line_number":310,"context_line":"   BGP protocol and send a BGP \"announce\" message to the ToR switch."},{"line_number":311,"context_line":""},{"line_number":312,"context_line":"6. The ToR switch will learn this new route and advertise it into the L3"},{"line_number":313,"context_line":"   fabric. At this point the L3 fabric will know of the new VIP route and how"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_20b48bd0","line":310,"range":{"start_line":308,"start_character":3,"end_line":310,"end_character":68},"in_reply_to":"5f2577a7_2d3da4dc","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":320,"context_line":"8. Once the route is accepted by the L3 fabric, traffic will get distributed"},{"line_number":321,"context_line":"   to the recently registered amphora (HAProxy)."},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"9. In order to unregister an amphora (HAProxy), the BGP speaker needs to be"},{"line_number":324,"context_line":"   informed to withdraw the VIP route. The BGP speaker will then send a route"},{"line_number":325,"context_line":"   \"withdraw\" message over the BGP peering session to the ToR switch."},{"line_number":326,"context_line":""},{"line_number":327,"context_line":"10. The ToR switch will tell the L3 fabric over BGP that the anycast VIP route"},{"line_number":328,"context_line":"    for the amphora being unregistered is no longer valid."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_ed0b1c27","line":325,"range":{"start_line":323,"start_character":3,"end_line":325,"end_character":69},"updated":"2017-09-15 22:44:35.000000000","message":"Same here, stop announce may be at listener stop.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":320,"context_line":"8. Once the route is accepted by the L3 fabric, traffic will get distributed"},{"line_number":321,"context_line":"   to the recently registered amphora (HAProxy)."},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"9. In order to unregister an amphora (HAProxy), the BGP speaker needs to be"},{"line_number":324,"context_line":"   informed to withdraw the VIP route. The BGP speaker will then send a route"},{"line_number":325,"context_line":"   \"withdraw\" message over the BGP peering session to the ToR switch."},{"line_number":326,"context_line":""},{"line_number":327,"context_line":"10. The ToR switch will tell the L3 fabric over BGP that the anycast VIP route"},{"line_number":328,"context_line":"    for the amphora being unregistered is no longer valid."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_80de9f0e","line":325,"range":{"start_line":323,"start_character":3,"end_line":325,"end_character":69},"in_reply_to":"5f2577a7_ed0b1c27","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":347,"context_line":"    ID of the distributor responsible for distributing traffic for the"},{"line_number":348,"context_line":"    corresponding VIP."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Add table ``frontend_network`` with the following columns:"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"* id ``(String(36) , nullable\u003dFalse)``"},{"line_number":353,"context_line":"    ID representing an attached interface to an amphora instance."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_e0ac43ee","line":350,"range":{"start_line":350,"start_character":0,"end_line":350,"end_character":58},"updated":"2017-09-15 22:44:35.000000000","message":"I think we can use the same VIP table, I would even support renaming VRPP to frontend for clarity.  We can extend this table if there is an incorrect assumption that the VIP and VRRP share a subnet.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":347,"context_line":"    ID of the distributor responsible for distributing traffic for the"},{"line_number":348,"context_line":"    corresponding VIP."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Add table ``frontend_network`` with the following columns:"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"* id ``(String(36) , nullable\u003dFalse)``"},{"line_number":353,"context_line":"    ID representing an attached interface to an amphora instance."}],"source_content_type":"text/x-rst","patch_set":13,"id":"ff5d4b2f_810d1e92","line":350,"range":{"start_line":350,"start_character":0,"end_line":350,"end_character":58},"in_reply_to":"5f2577a7_e0ac43ee","updated":"2017-09-20 01:20:28.000000000","message":"I removed this and plan to rename the vrrp_* fields on the amphora table to represent the frontend network. As far as the subnet goes, we will need to modify the code so it is not assumed that vrrp subnet \u003d vip subnet. Does it make sense to store the frontend subnet in the amphora table or can we just query it from the port_id?","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"165508819bb7f70458e354ce0e5e45160fd6b1d1","unresolved":false,"context_lines":[{"line_number":347,"context_line":"    ID of the distributor responsible for distributing traffic for the"},{"line_number":348,"context_line":"    corresponding VIP."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Add table ``frontend_network`` with the following columns:"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"* id ``(String(36) , nullable\u003dFalse)``"},{"line_number":353,"context_line":"    ID representing an attached interface to an amphora instance."}],"source_content_type":"text/x-rst","patch_set":13,"id":"7f515b1d_f53dc22c","line":350,"range":{"start_line":350,"start_character":0,"end_line":350,"end_character":58},"in_reply_to":"ff5d4b2f_810d1e92","updated":"2017-09-21 21:56:50.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"ef117d05715ba7c5bdb9411d9a1e03aad738c15a","unresolved":false,"context_lines":[{"line_number":347,"context_line":"    ID of the distributor responsible for distributing traffic for the"},{"line_number":348,"context_line":"    corresponding VIP."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"Add table ``frontend_network`` with the following columns:"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"* id ``(String(36) , nullable\u003dFalse)``"},{"line_number":353,"context_line":"    ID representing an attached interface to an amphora instance."}],"source_content_type":"text/x-rst","patch_set":13,"id":"7f515b1d_898d5d7d","line":350,"range":{"start_line":350,"start_character":0,"end_line":350,"end_character":58},"in_reply_to":"ff5d4b2f_810d1e92","updated":"2017-09-21 21:00:15.000000000","message":"I would store it and not rely on querying the neutron API to look it up.  We should have it at create time.  These are things handy for an operator to query by.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":477,"context_line":"* amphora_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":478,"context_line":"    ID of distributor amphora instance."},{"line_number":479,"context_line":""},{"line_number":480,"context_line":"* vip_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":481,"context_line":"    The IP address of the anycast VIP."},{"line_number":482,"context_line":""},{"line_number":483,"context_line":"* frontend_network_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":484,"context_line":"    The front-end network IP used to handle VIP traffic. This is the next-hop"},{"line_number":485,"context_line":"    address that will be advertised for the VIP."},{"line_number":486,"context_line":""},{"line_number":487,"context_line":"* distributor_l3_bgp_peer_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":488,"context_line":"    The BGP peer we will announce the anycast VIP to."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_60ef331e","line":485,"range":{"start_line":480,"start_character":0,"end_line":485,"end_character":48},"updated":"2017-09-15 22:44:35.000000000","message":"would this just be a foreign key to the VIP table?","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":477,"context_line":"* amphora_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":478,"context_line":"    ID of distributor amphora instance."},{"line_number":479,"context_line":""},{"line_number":480,"context_line":"* vip_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":481,"context_line":"    The IP address of the anycast VIP."},{"line_number":482,"context_line":""},{"line_number":483,"context_line":"* frontend_network_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":484,"context_line":"    The front-end network IP used to handle VIP traffic. This is the next-hop"},{"line_number":485,"context_line":"    address that will be advertised for the VIP."},{"line_number":486,"context_line":""},{"line_number":487,"context_line":"* distributor_l3_bgp_peer_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":488,"context_line":"    The BGP peer we will announce the anycast VIP to."}],"source_content_type":"text/x-rst","patch_set":13,"id":"ff5d4b2f_ed53abe1","line":485,"range":{"start_line":480,"start_character":0,"end_line":485,"end_character":48},"in_reply_to":"5f2577a7_60ef331e","updated":"2017-09-20 01:20:28.000000000","message":"I changed vip_ip_address to loadbalancer_id, which is the PK in the VIP table. The question here is if we want this direct relationship or just store the VIP IP.  Thoughts? \n\nI also renamed frontend_network_ip_address to nexthop_ip, as the nexthop_ip might not belong to any amphora, such as in the UDP load balancing use case.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"165508819bb7f70458e354ce0e5e45160fd6b1d1","unresolved":false,"context_lines":[{"line_number":477,"context_line":"* amphora_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":478,"context_line":"    ID of distributor amphora instance."},{"line_number":479,"context_line":""},{"line_number":480,"context_line":"* vip_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":481,"context_line":"    The IP address of the anycast VIP."},{"line_number":482,"context_line":""},{"line_number":483,"context_line":"* frontend_network_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":484,"context_line":"    The front-end network IP used to handle VIP traffic. This is the next-hop"},{"line_number":485,"context_line":"    address that will be advertised for the VIP."},{"line_number":486,"context_line":""},{"line_number":487,"context_line":"* distributor_l3_bgp_peer_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":488,"context_line":"    The BGP peer we will announce the anycast VIP to."}],"source_content_type":"text/x-rst","patch_set":13,"id":"7f515b1d_f526e216","line":485,"range":{"start_line":480,"start_character":0,"end_line":485,"end_character":48},"in_reply_to":"7f515b1d_ec029f3e","updated":"2017-09-21 21:56:50.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"ef117d05715ba7c5bdb9411d9a1e03aad738c15a","unresolved":false,"context_lines":[{"line_number":477,"context_line":"* amphora_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":478,"context_line":"    ID of distributor amphora instance."},{"line_number":479,"context_line":""},{"line_number":480,"context_line":"* vip_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":481,"context_line":"    The IP address of the anycast VIP."},{"line_number":482,"context_line":""},{"line_number":483,"context_line":"* frontend_network_ip_address ``(String(64) , nullable\u003dFalse)``"},{"line_number":484,"context_line":"    The front-end network IP used to handle VIP traffic. This is the next-hop"},{"line_number":485,"context_line":"    address that will be advertised for the VIP."},{"line_number":486,"context_line":""},{"line_number":487,"context_line":"* distributor_l3_bgp_peer_id ``(String(36) , nullable\u003dFalse)``"},{"line_number":488,"context_line":"    The BGP peer we will announce the anycast VIP to."}],"source_content_type":"text/x-rst","patch_set":13,"id":"7f515b1d_ec029f3e","line":485,"range":{"start_line":480,"start_character":0,"end_line":485,"end_character":48},"in_reply_to":"ff5d4b2f_ed53abe1","updated":"2017-09-21 21:00:15.000000000","message":"Yeah, this is an interesting one.  I personally like the load_balancer_id idea as it maintains the relational integrity.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":490,"context_line":"REST API impact"},{"line_number":491,"context_line":"---------------"},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"* LBaaS API -- Allow the user to specify a specific VIP/subnet and front-end"},{"line_number":494,"context_line":"  subnet (provider network) when creating a new load balancer."},{"line_number":495,"context_line":"  Currently the user can only specify the VIP subnet, which results in both the"},{"line_number":496,"context_line":"  VIP and front-end network being on the same subnet."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_c0e38733","line":493,"range":{"start_line":493,"start_character":2,"end_line":493,"end_character":7},"updated":"2017-09-15 22:44:35.000000000","message":"Octavia","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":490,"context_line":"REST API impact"},{"line_number":491,"context_line":"---------------"},{"line_number":492,"context_line":""},{"line_number":493,"context_line":"* LBaaS API -- Allow the user to specify a specific VIP/subnet and front-end"},{"line_number":494,"context_line":"  subnet (provider network) when creating a new load balancer."},{"line_number":495,"context_line":"  Currently the user can only specify the VIP subnet, which results in both the"},{"line_number":496,"context_line":"  VIP and front-end network being on the same subnet."}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_a0de3bd7","line":493,"range":{"start_line":493,"start_character":2,"end_line":493,"end_character":7},"in_reply_to":"5f2577a7_c0e38733","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":499,"context_line":"  amphora API in order to implement the control plane (BGP) and advertise new"},{"line_number":500,"context_line":"  anycast VIP routes into the network."},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"The below extended amphora API calls will be implemented:"},{"line_number":503,"context_line":""},{"line_number":504,"context_line":"1. ``Register Amphora``"},{"line_number":505,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_a0b99b15","line":502,"range":{"start_line":502,"start_character":19,"end_line":502,"end_character":30},"updated":"2017-09-15 22:44:35.000000000","message":"For amphora acting as a distributor just to clarify.\nIn the other mode, this are not used and the data is included in the listener upload and start/stop calls.","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":499,"context_line":"  amphora API in order to implement the control plane (BGP) and advertise new"},{"line_number":500,"context_line":"  anycast VIP routes into the network."},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"The below extended amphora API calls will be implemented:"},{"line_number":503,"context_line":""},{"line_number":504,"context_line":"1. ``Register Amphora``"},{"line_number":505,"context_line":""}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_690f1337","line":502,"range":{"start_line":502,"start_character":19,"end_line":502,"end_character":30},"in_reply_to":"5f2577a7_a0b99b15","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"6509692a6bebcf0f27f0c9a865fe116511fae8e4","unresolved":false,"context_lines":[{"line_number":601,"context_line":""},{"line_number":602,"context_line":"Documentation Impact"},{"line_number":603,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"References"},{"line_number":606,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":607,"context_line":"* `Active-Active Topology"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_c05147db","line":604,"updated":"2017-09-15 22:44:35.000000000","message":"There is API-Ref doc impact.  The API reference will need to be updated for the extended load balancer API (frontend subnet ID)","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"2be05cfd2b68b2efaac297f92a7ec1e2590c6079","unresolved":false,"context_lines":[{"line_number":601,"context_line":""},{"line_number":602,"context_line":"Documentation Impact"},{"line_number":603,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":604,"context_line":""},{"line_number":605,"context_line":"References"},{"line_number":606,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":607,"context_line":"* `Active-Active Topology"}],"source_content_type":"text/x-rst","patch_set":13,"id":"5f2577a7_e429ea89","line":604,"in_reply_to":"5f2577a7_c05147db","updated":"2017-09-20 01:20:28.000000000","message":"Done","commit_id":"c4f6c6cdc7cd4902ce049a0f0f3e57670c983cad"},{"author":{"_account_id":12024,"name":"Tong Liu","email":"tongl@vmware.com","username":"tong101"},"change_message_id":"e342ffbbf57799238a20073b047ca8aeaf631f37","unresolved":false,"context_lines":[{"line_number":103,"context_line":"The below diagram shows the interaction between 2 .. n amphora instances from"},{"line_number":104,"context_line":"each tenant and how they interact with the L3 network distributor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"::"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"                   Management                                        Front-End"},{"line_number":109,"context_line":"  Internet         Network                                           Networks"}],"source_content_type":"text/x-rst","patch_set":14,"id":"bf57530f_cd08dd29","line":106,"updated":"2017-09-20 18:32:39.000000000","message":"Does Amphora of Tenant A connect to Anycast network?","commit_id":"fb56d401f25fb9b145e63d2d1015756b12aec224"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"b7d48278dcb598c7da99ea9accd054e4109adb3f","unresolved":false,"context_lines":[{"line_number":103,"context_line":"The below diagram shows the interaction between 2 .. n amphora instances from"},{"line_number":104,"context_line":"each tenant and how they interact with the L3 network distributor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"::"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"                   Management                                        Front-End"},{"line_number":109,"context_line":"  Internet         Network                                           Networks"}],"source_content_type":"text/x-rst","patch_set":14,"id":"bf57530f_582149fb","line":106,"in_reply_to":"bf57530f_cd08dd29","updated":"2017-09-20 19:10:46.000000000","message":"Yes, but traffic for the anycast IP (/32 route) goes over the front-end network.","commit_id":"fb56d401f25fb9b145e63d2d1015756b12aec224"},{"author":{"_account_id":26401,"name":"Wei Li","email":"liw@dtdream.com","username":"wei"},"change_message_id":"fcd237a1eddbbc4d620dcc4d915dcb8a1d11e1d0","unresolved":false,"context_lines":[{"line_number":125,"context_line":"  │             │        ║            ┌─────────────────────────╨───┐    ║"},{"line_number":126,"context_line":"  │             │        ║            │     Amphora of Tenant B     │    ║"},{"line_number":127,"context_line":"  │             │        ║      ┌────┬┴──────────┬──────────────────┴┬───╨┐"},{"line_number":128,"context_line":"  │             ╞════════╬══════╡MGMT│ns: default│ns: amphora-haproxy│f.e.│"},{"line_number":129,"context_line":"  │             │        ║      │ IP ├-----------┼-------------------┤ IP │"},{"line_number":130,"context_line":"  │             │        ║      └────┤    BGP    │    Anycast VIP    ├───╥┘"},{"line_number":131,"context_line":"  │             │        ║           │   Speaker │    (loopback)     │   ║"}],"source_content_type":"text/x-rst","patch_set":14,"id":"9f5c4f37_6d821fcb","line":128,"range":{"start_line":128,"start_character":16,"end_line":128,"end_character":21},"updated":"2017-09-21 12:39:19.000000000","message":"Why connect L3 network switch?","commit_id":"fb56d401f25fb9b145e63d2d1015756b12aec224"}],"specs/version1/active-active-l3-distributor.rst":[{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Distributor for L3 Active-Active, N+1 Amphora Setup"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":".. attention::"},{"line_number":11,"context_line":"  This blueprint is a work in progress (WIP)"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"  Please review the active-active topology blueprint first ("},{"line_number":14,"context_line":"  :doc:`active-active-topology` )"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_9bdbf1eb","line":11,"range":{"start_line":10,"start_character":0,"end_line":11,"end_character":44},"updated":"2017-04-07 22:55:22.000000000","message":"We do this via the workflow -1 on the patch.  That way if it is fine reviewers can just +2 the patch and let it merge.  Otherwise you have to commit another patch to remove this and then we have to re-review.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Distributor for L3 Active-Active, N+1 Amphora Setup"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":".. attention::"},{"line_number":11,"context_line":"  This blueprint is a work in progress (WIP)"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"  Please review the active-active topology blueprint first ("},{"line_number":14,"context_line":"  :doc:`active-active-topology` )"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_19131468","line":11,"range":{"start_line":10,"start_character":0,"end_line":11,"end_character":44},"in_reply_to":"bff0334d_9bdbf1eb","updated":"2017-04-28 21:29:46.000000000","message":"Done","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":55,"context_line":"* Octavia shall implement the *L3 active-active* distributor through a pluggable"},{"line_number":56,"context_line":"  driver."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"* Each amphora will run a *bgp speaker* in the default namespace in order to"},{"line_number":59,"context_line":"  announce the anycast VIP into the L3 fabric. The anycast VIP will get"},{"line_number":60,"context_line":"  advertised as a /32 or /128 route with a next-hop of either the floating IP"},{"line_number":61,"context_line":"  (FIP) or back-end (tenant network) IP assigned to the amphora instance."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_fb343dfd","line":58,"range":{"start_line":58,"start_character":47,"end_line":58,"end_character":64},"updated":"2017-04-07 22:55:22.000000000","message":"Just so we are clear, the default namespace only has access to the lb-mgmt-net.  Is that your intention?","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":55,"context_line":"* Octavia shall implement the *L3 active-active* distributor through a pluggable"},{"line_number":56,"context_line":"  driver."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"* Each amphora will run a *bgp speaker* in the default namespace in order to"},{"line_number":59,"context_line":"  announce the anycast VIP into the L3 fabric. The anycast VIP will get"},{"line_number":60,"context_line":"  advertised as a /32 or /128 route with a next-hop of either the floating IP"},{"line_number":61,"context_line":"  (FIP) or back-end (tenant network) IP assigned to the amphora instance."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_b4cd4d86","line":58,"range":{"start_line":58,"start_character":47,"end_line":58,"end_character":64},"in_reply_to":"bff0334d_fb343dfd","updated":"2017-04-28 21:29:46.000000000","message":"Explicitly specified lb-mgmt-net","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":57,"context_line":""},{"line_number":58,"context_line":"* Each amphora will run a *bgp speaker* in the default namespace in order to"},{"line_number":59,"context_line":"  announce the anycast VIP into the L3 fabric. The anycast VIP will get"},{"line_number":60,"context_line":"  advertised as a /32 or /128 route with a next-hop of either the floating IP"},{"line_number":61,"context_line":"  (FIP) or back-end (tenant network) IP assigned to the amphora instance."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet when"},{"line_number":64,"context_line":"  creating a new load balancer. When an anycast VIP/subnet is specified the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_8dddf871","line":61,"range":{"start_line":60,"start_character":66,"end_line":61,"end_character":7},"updated":"2017-04-07 22:55:22.000000000","message":"I am wondering if it makes sense to call out floating IPs here.  In OpenStack a floating IP is a NAT instance typically used to map a public IP to a tenant IP.  I wonder if neutron\u0027s floating IP implementation would handle this correctly.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":57,"context_line":""},{"line_number":58,"context_line":"* Each amphora will run a *bgp speaker* in the default namespace in order to"},{"line_number":59,"context_line":"  announce the anycast VIP into the L3 fabric. The anycast VIP will get"},{"line_number":60,"context_line":"  advertised as a /32 or /128 route with a next-hop of either the floating IP"},{"line_number":61,"context_line":"  (FIP) or back-end (tenant network) IP assigned to the amphora instance."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet when"},{"line_number":64,"context_line":"  creating a new load balancer. When an anycast VIP/subnet is specified the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_da1fdb69","line":61,"range":{"start_line":60,"start_character":66,"end_line":61,"end_character":7},"in_reply_to":"bff0334d_8dddf871","updated":"2017-04-28 21:29:46.000000000","message":"Good call out.  I don\u0027t see a way via the floating IP api to create an exception rule. An exception would need to get added to iptables to not perform SNAT, if the source IP equals the anycast VIP IP. Since leveraging FIPs would require neutron changes to floating IPs, I can remove FIP references.  Let me know if this works.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":60,"context_line":"  advertised as a /32 or /128 route with a next-hop of either the floating IP"},{"line_number":61,"context_line":"  (FIP) or back-end (tenant network) IP assigned to the amphora instance."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet when"},{"line_number":64,"context_line":"  creating a new load balancer. When an anycast VIP/subnet is specified the"},{"line_number":65,"context_line":"  amphora will have ports on three networks (anycast, tenant, management). The"},{"line_number":66,"context_line":"  anycast VIP will get configured on the loopback interface inside the"},{"line_number":67,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_ad443c1c","line":65,"range":{"start_line":63,"start_character":2,"end_line":65,"end_character":74},"updated":"2017-04-07 22:55:22.000000000","message":"How is this different than the capability today?  We allow the user to specify the network for the VIP.  Is it an issue with using a network that is not owned by the tenant?","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":60,"context_line":"  advertised as a /32 or /128 route with a next-hop of either the floating IP"},{"line_number":61,"context_line":"  (FIP) or back-end (tenant network) IP assigned to the amphora instance."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet when"},{"line_number":64,"context_line":"  creating a new load balancer. When an anycast VIP/subnet is specified the"},{"line_number":65,"context_line":"  amphora will have ports on three networks (anycast, tenant, management). The"},{"line_number":66,"context_line":"  anycast VIP will get configured on the loopback interface inside the"},{"line_number":67,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":68,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_3dc80172","line":65,"range":{"start_line":63,"start_character":2,"end_line":65,"end_character":74},"in_reply_to":"bff0334d_ad443c1c","updated":"2017-04-28 21:29:46.000000000","message":"When you specify the network for the amphora it provisions the interface (eth1) with two IP addresses on the same network.  One IP is the VIP and the other is tenant IP that is used to communicate to the back-ends.  With active/active L3 it is required to have an additional anycast network, which is not tied to any specific rack.  All IP addresses from this network will be advertised as /32 or /128 routes with a next hop address of the tenant IP, which must be routeable from the L3 network. The anycast network would need to be either a neutron network, or a network the user defines in Octavia.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":63,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet when"},{"line_number":64,"context_line":"  creating a new load balancer. When an anycast VIP/subnet is specified the"},{"line_number":65,"context_line":"  amphora will have ports on three networks (anycast, tenant, management). The"},{"line_number":66,"context_line":"  anycast VIP will get configured on the loopback interface inside the"},{"line_number":67,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* The operator shall be able to define a *bgp peer profile*, which includes the"},{"line_number":70,"context_line":"  required metadata for the amphora to establish a bgp peering session with"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_4d55b0c5","line":67,"range":{"start_line":66,"start_character":2,"end_line":67,"end_character":38},"updated":"2017-04-07 22:55:22.000000000","message":"It may not be necessary to setup the loopback.  We enable non-local binding in the kernel which allows the IP we configure inside HAProxy to not be bound on an interface.  So, this may \"just work\" as we have already designed it.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":63,"context_line":"* Octavia shall implement the ability to specify an anycast VIP/subnet when"},{"line_number":64,"context_line":"  creating a new load balancer. When an anycast VIP/subnet is specified the"},{"line_number":65,"context_line":"  amphora will have ports on three networks (anycast, tenant, management). The"},{"line_number":66,"context_line":"  anycast VIP will get configured on the loopback interface inside the"},{"line_number":67,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* The operator shall be able to define a *bgp peer profile*, which includes the"},{"line_number":70,"context_line":"  required metadata for the amphora to establish a bgp peering session with"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_ddbd95b7","line":67,"range":{"start_line":66,"start_character":2,"end_line":67,"end_character":38},"in_reply_to":"bff0334d_4d55b0c5","updated":"2017-04-28 21:29:46.000000000","message":"I have done some testing and without configuring the anycast IP on the loop back interface the Linux networking stack will not pass on the packet to the haproxy process. Using non-local binding will work as usual.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":66,"context_line":"  anycast VIP will get configured on the loopback interface inside the"},{"line_number":67,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* The operator shall be able to define a *bgp peer profile*, which includes the"},{"line_number":70,"context_line":"  required metadata for the amphora to establish a bgp peering session with"},{"line_number":71,"context_line":"  the L3 fabric. The amphora will have to retrieve this metadata from Octavia"},{"line_number":72,"context_line":"  during initialization in order to create a BGP peer and announce its anycast"},{"line_number":73,"context_line":"  VIP."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"* [P2]_ The *Amphora Cluster* will provide resilient flow handling in order to"},{"line_number":76,"context_line":"  handle ECMP group flow remapping events and support amphora connection"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_8d74d869","line":73,"range":{"start_line":69,"start_character":2,"end_line":73,"end_character":6},"updated":"2017-04-07 22:55:22.000000000","message":"I would propose that this is passed into our amphora-agent configuration file via config drive at amphora instance boot.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":66,"context_line":"  anycast VIP will get configured on the loopback interface inside the"},{"line_number":67,"context_line":"  *amphora-haproxy* network namespace."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"* The operator shall be able to define a *bgp peer profile*, which includes the"},{"line_number":70,"context_line":"  required metadata for the amphora to establish a bgp peering session with"},{"line_number":71,"context_line":"  the L3 fabric. The amphora will have to retrieve this metadata from Octavia"},{"line_number":72,"context_line":"  during initialization in order to create a BGP peer and announce its anycast"},{"line_number":73,"context_line":"  VIP."},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"* [P2]_ The *Amphora Cluster* will provide resilient flow handling in order to"},{"line_number":76,"context_line":"  handle ECMP group flow remapping events and support amphora connection"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_3d580154","line":73,"range":{"start_line":69,"start_character":2,"end_line":73,"end_character":6},"in_reply_to":"bff0334d_8d74d869","updated":"2017-04-28 21:29:46.000000000","message":"Added.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":84,"context_line":"The below diagram shows the interaction between 2 .. n amphora instances from"},{"line_number":85,"context_line":"each tenant and how they interact with the L3 network distributor."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":".. image:: active-active-l3-distributor.png"},{"line_number":88,"context_line":"   :align: center"},{"line_number":89,"context_line":"   :alt: L3 Distributor Topology"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"* Whenever a new amphora is instantiated it will create BGP peering session(s)"},{"line_number":92,"context_line":"  over the lb-mgmt-net to the L3 fabric. The BGP peer will need to have a"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_4d1c9011","line":89,"range":{"start_line":87,"start_character":0,"end_line":89,"end_character":32},"updated":"2017-04-07 22:55:22.000000000","message":"A couple of comments on the diagram:\n1. We should connect up the anycast network to the anycast VIP.\n2. We need to clarify the role of the tenant network here.  We can plug in many tenant networks to provide access to members.  Is the tenant network here used for traffic return path for the VIP or is it representing the one or more member networks?\n3. Are you proposing the anycast network is connected across tenants?","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":84,"context_line":"The below diagram shows the interaction between 2 .. n amphora instances from"},{"line_number":85,"context_line":"each tenant and how they interact with the L3 network distributor."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":".. image:: active-active-l3-distributor.png"},{"line_number":88,"context_line":"   :align: center"},{"line_number":89,"context_line":"   :alt: L3 Distributor Topology"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"* Whenever a new amphora is instantiated it will create BGP peering session(s)"},{"line_number":92,"context_line":"  over the lb-mgmt-net to the L3 fabric. The BGP peer will need to have a"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_c9527713","line":89,"range":{"start_line":87,"start_character":0,"end_line":89,"end_character":32},"in_reply_to":"bff0334d_4d1c9011","updated":"2017-04-28 21:29:46.000000000","message":"1) Agreed, I will fix the diagram. \n\n2) The tenant network in the diagram represents the network used for communicating with the client.  The client will communicate with the anycast VIP, which will be routed out the tenant / provider network. If there are multiple provider or tenant networks assigned to the amphora, one of those networks will need to be chosen to handle the traffic for the anycast VIP. The chosen tenant / provider network interface will be listed as the next-hop address for the anycast VIP in the BGP advertisement. \n\n3) The anycast network is used to carve out /32 or /128 routes that can be assigned to amphora\u0027s belonging to different tenants.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":88,"context_line":"   :align: center"},{"line_number":89,"context_line":"   :alt: L3 Distributor Topology"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"* Whenever a new amphora is instantiated it will create BGP peering session(s)"},{"line_number":92,"context_line":"  over the lb-mgmt-net to the L3 fabric. The BGP peer will need to have a"},{"line_number":93,"context_line":"  neighbor definition in order to allow the peering sessions from the amphoras."},{"line_number":94,"context_line":"  In order to ease configuration, a neighbor statement allowing peers from the"},{"line_number":95,"context_line":"  entire lb-mgmt-net IP prefix range can be defined: ``neighbor 10.10.10.0/24``"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_8de73813","line":92,"range":{"start_line":91,"start_character":56,"end_line":92,"end_character":39},"updated":"2017-04-07 22:55:22.000000000","message":"Ok, this clarifies my above question.  Maybe just clarify that above as well.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":88,"context_line":"   :align: center"},{"line_number":89,"context_line":"   :alt: L3 Distributor Topology"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"* Whenever a new amphora is instantiated it will create BGP peering session(s)"},{"line_number":92,"context_line":"  over the lb-mgmt-net to the L3 fabric. The BGP peer will need to have a"},{"line_number":93,"context_line":"  neighbor definition in order to allow the peering sessions from the amphoras."},{"line_number":94,"context_line":"  In order to ease configuration, a neighbor statement allowing peers from the"},{"line_number":95,"context_line":"  entire lb-mgmt-net IP prefix range can be defined: ``neighbor 10.10.10.0/24``"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_3d2da1a7","line":92,"range":{"start_line":91,"start_character":56,"end_line":92,"end_character":39},"in_reply_to":"bff0334d_8de73813","updated":"2017-04-28 21:29:46.000000000","message":"Done","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":94,"context_line":"  In order to ease configuration, a neighbor statement allowing peers from the"},{"line_number":95,"context_line":"  entire lb-mgmt-net IP prefix range can be defined: ``neighbor 10.10.10.0/24``"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"* The peer IP can either be a route reflector (RR) or any other network device"},{"line_number":98,"context_line":"  that will redistribute routes learned from the amphora BGP speaker. In order"},{"line_number":99,"context_line":"  to help scaling, it is possible to peer with the ToR switch based on the"},{"line_number":100,"context_line":"  rack the amphora instance is provisioned in. The configuration can be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_cdb2e0e6","line":97,"range":{"start_line":97,"start_character":6,"end_line":97,"end_character":13},"updated":"2017-04-07 22:55:22.000000000","message":"Can we call out BGP peer here.  We have other \"peer\" terms in out system.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":94,"context_line":"  In order to ease configuration, a neighbor statement allowing peers from the"},{"line_number":95,"context_line":"  entire lb-mgmt-net IP prefix range can be defined: ``neighbor 10.10.10.0/24``"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"* The peer IP can either be a route reflector (RR) or any other network device"},{"line_number":98,"context_line":"  that will redistribute routes learned from the amphora BGP speaker. In order"},{"line_number":99,"context_line":"  to help scaling, it is possible to peer with the ToR switch based on the"},{"line_number":100,"context_line":"  rack the amphora instance is provisioned in. The configuration can be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_3d06c121","line":97,"range":{"start_line":97,"start_character":6,"end_line":97,"end_character":13},"in_reply_to":"bff0334d_cdb2e0e6","updated":"2017-04-28 21:29:46.000000000","message":"Done","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":102,"context_line":"  which will provide a consistent BGP peer IP regardless of which rack or"},{"line_number":103,"context_line":"  hypervisor is hosting the amphora instance."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"* Once a peering session is established between an amphora and the L3 fabric,"},{"line_number":106,"context_line":"  the amphora will need to announce its anycast VIP with a next-hop address of"},{"line_number":107,"context_line":"  either its FIP or back-end network IP. The back-end network IP (tenant) must"},{"line_number":108,"context_line":"  be routable and reachable from the L3 network in order to be used."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"* In order to leverage ECMP for distributing traffic across multiple amphoras,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_4df49086","line":107,"range":{"start_line":105,"start_character":2,"end_line":107,"end_character":40},"updated":"2017-04-07 22:55:22.000000000","message":"Does this work for tenant network \"A\" to tenant network \"A\" scenarios?  For example internal to the tenant load balancing connections to a database that is also internal?","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":102,"context_line":"  which will provide a consistent BGP peer IP regardless of which rack or"},{"line_number":103,"context_line":"  hypervisor is hosting the amphora instance."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"* Once a peering session is established between an amphora and the L3 fabric,"},{"line_number":106,"context_line":"  the amphora will need to announce its anycast VIP with a next-hop address of"},{"line_number":107,"context_line":"  either its FIP or back-end network IP. The back-end network IP (tenant) must"},{"line_number":108,"context_line":"  be routable and reachable from the L3 network in order to be used."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"* In order to leverage ECMP for distributing traffic across multiple amphoras,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_40919cf0","line":107,"range":{"start_line":105,"start_character":2,"end_line":107,"end_character":40},"in_reply_to":"bff0334d_4df49086","updated":"2017-04-28 21:29:46.000000000","message":"This would work from tenant network A to tenant network A assuming the IPs are routeable from the L3 fabric.  Private networks that are not routeable or advertised to the L3 network would not work, as the network itself is serving as the distributor.  \n\nThe fully private model (overlay/vxlan) would require something like the distributor in the active/active model. If the private tenant networks were implemented with VRFs in the physical network fabric then it should be possible to still leverage the fabric, as the distributor.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"* Once a peering session is established between an amphora and the L3 fabric,"},{"line_number":106,"context_line":"  the amphora will need to announce its anycast VIP with a next-hop address of"},{"line_number":107,"context_line":"  either its FIP or back-end network IP. The back-end network IP (tenant) must"},{"line_number":108,"context_line":"  be routable and reachable from the L3 network in order to be used."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"* In order to leverage ECMP for distributing traffic across multiple amphoras,"},{"line_number":111,"context_line":"  multiple equal-cost routes must be installed into the network for the anycast"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_0d07a884","line":108,"range":{"start_line":107,"start_character":41,"end_line":108,"end_character":68},"updated":"2017-04-07 22:55:22.000000000","message":"How do we handle tenants with overlapping IP address space?","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"* Once a peering session is established between an amphora and the L3 fabric,"},{"line_number":106,"context_line":"  the amphora will need to announce its anycast VIP with a next-hop address of"},{"line_number":107,"context_line":"  either its FIP or back-end network IP. The back-end network IP (tenant) must"},{"line_number":108,"context_line":"  be routable and reachable from the L3 network in order to be used."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"* In order to leverage ECMP for distributing traffic across multiple amphoras,"},{"line_number":111,"context_line":"  multiple equal-cost routes must be installed into the network for the anycast"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_e0e2f0d4","line":108,"range":{"start_line":107,"start_character":41,"end_line":108,"end_character":68},"in_reply_to":"bff0334d_0d07a884","updated":"2017-04-28 21:29:46.000000000","message":"For overlapping back-end network IPs, it would require implementing VRFs on the back-end L3 network. Essentially this is overlapping provider networks. Overlapping tenant networks, such as VXLAN would require the changes to floating IPs I described above.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":109,"context_line":""},{"line_number":110,"context_line":"* In order to leverage ECMP for distributing traffic across multiple amphoras,"},{"line_number":111,"context_line":"  multiple equal-cost routes must be installed into the network for the anycast"},{"line_number":112,"context_line":"  VIP. This requires the L3 network to have ``BGP Multipath Load Sharing``"},{"line_number":113,"context_line":"  enabled, so BGP installs multiple paths and does not select a single best"},{"line_number":114,"context_line":"  path."},{"line_number":115,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_0d632844","line":112,"range":{"start_line":112,"start_character":44,"end_line":112,"end_character":74},"updated":"2017-04-07 22:55:22.000000000","message":"I think this is a vendor specific term but we can use multiple vendors implementation of this right? Can we use RFC terms or vendor neutral terms?","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":109,"context_line":""},{"line_number":110,"context_line":"* In order to leverage ECMP for distributing traffic across multiple amphoras,"},{"line_number":111,"context_line":"  multiple equal-cost routes must be installed into the network for the anycast"},{"line_number":112,"context_line":"  VIP. This requires the L3 network to have ``BGP Multipath Load Sharing``"},{"line_number":113,"context_line":"  enabled, so BGP installs multiple paths and does not select a single best"},{"line_number":114,"context_line":"  path."},{"line_number":115,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_80fb24bf","line":112,"range":{"start_line":112,"start_character":44,"end_line":112,"end_character":74},"in_reply_to":"bff0334d_0d632844","updated":"2017-04-28 21:29:46.000000000","message":"Multipath BGP is used by several vendors to describe this.  I removed the Load Sharing part, so now it reads \u0027Multipath BGP\u0027","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":141,"context_line":"  is recommended to enable the ``resilient hashing`` feature on ECMP groups in"},{"line_number":142,"context_line":"  order to minimize flow remapping."},{"line_number":143,"context_line":""},{"line_number":144,"context_line":"Specific proposed changes"},{"line_number":145,"context_line":"-------------------------"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Data model impact"},{"line_number":148,"context_line":"-----------------"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"Security impact"},{"line_number":151,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_2de02c96","line":148,"range":{"start_line":144,"start_character":0,"end_line":148,"end_character":17},"updated":"2017-04-07 22:55:22.000000000","message":"These will obviously need to be filled in.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"e7f62f3b1df3011eb8cfbf671e56cba0734eb7fa","unresolved":false,"context_lines":[{"line_number":147,"context_line":"Data model impact"},{"line_number":148,"context_line":"-----------------"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"Security impact"},{"line_number":151,"context_line":"---------------"},{"line_number":152,"context_line":"The distributor inherently supports multi-tenancy, as it is simply providing"},{"line_number":153,"context_line":"traffic distribution across multiple amphoras. Network isolation on a per tenant"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_2d9a6c6d","line":150,"range":{"start_line":150,"start_character":0,"end_line":150,"end_character":15},"updated":"2017-04-07 22:55:22.000000000","message":"We should talk about the BGP security here too.\nCan a compromised amp advertise for another tenant\u0027s VIP?","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"a708a6ad7cbd8a0a6d79a9037efb6f6263b4a243","unresolved":false,"context_lines":[{"line_number":147,"context_line":"Data model impact"},{"line_number":148,"context_line":"-----------------"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"Security impact"},{"line_number":151,"context_line":"---------------"},{"line_number":152,"context_line":"The distributor inherently supports multi-tenancy, as it is simply providing"},{"line_number":153,"context_line":"traffic distribution across multiple amphoras. Network isolation on a per tenant"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bff0334d_ae6bb9ee","line":150,"range":{"start_line":150,"start_character":0,"end_line":150,"end_character":15},"in_reply_to":"bff0334d_2d9a6c6d","updated":"2017-04-28 21:29:46.000000000","message":"We can use route-maps, prefix-lists, and communities to control what routes are allowed to be advertised in the L3 network from a particular BGP peer.  \n\nIf an amphora is compromised (escape network namespace), it means the attacker has internal access to the lb-mgmt-net, which can access the openstack control plane, as well as create BGP peering sessions with the network fabric. One way to limit access would be to have a separate lb-mgmt-net or dedicated network for BGP peering for each tenant. \n\nThis type of attack seems like it would be extremely difficult to pull off, but would like to hear more from other members of the community to weigh in.","commit_id":"0ddc764f1a75913c0939471f9e54883072a3bcea"},{"author":{"_account_id":10850,"name":"German Eichberger","email":"german.eichberger@gmail.com","username":"german"},"change_message_id":"d809faae7ee92aec1edfcbc51db59cafe4280ba4","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Each amphora will run a *bgp speaker* in the default namespace in order to"},{"line_number":58,"context_line":"  announce the anycast VIP into the L3 fabric. BGP peering and announcements"},{"line_number":59,"context_line":"  will occur over the lb-mgmt-net network. The anycast VIP will get advertised"},{"line_number":60,"context_line":"  as a /32 or /128 route with a next-hop of the back-end (tenant network) IP"},{"line_number":61,"context_line":"  assigned to the amphora instance. The back-end network IPs must be directly"},{"line_number":62,"context_line":"  routable from the L3 fabric, such as in the provider networking model."}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_7738469f","line":59,"updated":"2017-06-13 21:22:41.000000000","message":"should we make that configurable to advertise potentially on the tenant network?","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"332c71ecb00b2fbbb76e04d26bcc8684cc126dd1","unresolved":false,"context_lines":[{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* Each amphora will run a *bgp speaker* in the default namespace in order to"},{"line_number":58,"context_line":"  announce the anycast VIP into the L3 fabric. BGP peering and announcements"},{"line_number":59,"context_line":"  will occur over the lb-mgmt-net network. The anycast VIP will get advertised"},{"line_number":60,"context_line":"  as a /32 or /128 route with a next-hop of the back-end (tenant network) IP"},{"line_number":61,"context_line":"  assigned to the amphora instance. The back-end network IPs must be directly"},{"line_number":62,"context_line":"  routable from the L3 fabric, such as in the provider networking model."}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_a02f6f86","line":59,"in_reply_to":"7f231b9d_7738469f","updated":"2017-06-15 18:13:06.000000000","message":"I\u0027m not against this, but I am trying to think of a specific use-case for peering over the tenant / provider network instead of the lb-mgmt-net.  Perhaps we can mark this for after initial release and then figure out if this use-case is needed.","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":10850,"name":"German Eichberger","email":"german.eichberger@gmail.com","username":"german"},"change_message_id":"d809faae7ee92aec1edfcbc51db59cafe4280ba4","unresolved":false,"context_lines":[{"line_number":74,"context_line":"  the amphora will use the bgp peering information to establish a BGP peer and"},{"line_number":75,"context_line":"  announce its anycast VIP."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"* [`P2`_] Add the option to allow the *bgp speaker* to run outside the amphora"},{"line_number":78,"context_line":"  instance. In this model a *bgp speaker* could advertise anycast VIPs for"},{"line_number":79,"context_line":"  one or more amphoras. Each BGP speaker can only announce a single next hop"},{"line_number":80,"context_line":"  route for a particular anycast VIP."}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_5732eaa8","line":77,"updated":"2017-06-13 21:22:41.000000000","message":"where would it run? In the control plane or would we need to manage an extra speaker-vm?","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"332c71ecb00b2fbbb76e04d26bcc8684cc126dd1","unresolved":false,"context_lines":[{"line_number":74,"context_line":"  the amphora will use the bgp peering information to establish a BGP peer and"},{"line_number":75,"context_line":"  announce its anycast VIP."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"* [`P2`_] Add the option to allow the *bgp speaker* to run outside the amphora"},{"line_number":78,"context_line":"  instance. In this model a *bgp speaker* could advertise anycast VIPs for"},{"line_number":79,"context_line":"  one or more amphoras. Each BGP speaker can only announce a single next hop"},{"line_number":80,"context_line":"  route for a particular anycast VIP."}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_8c3dfbde","line":77,"in_reply_to":"7f231b9d_5732eaa8","updated":"2017-06-15 18:13:06.000000000","message":"We would need to spin up and manage an extra BGP speaker VM.  Other active/active models, such as L2 require spinning up a distributor VM with something like OVS. \n\nIf we don\u0027t want a separate distributor VM , we can run the bgp speaker in an amphora without enabling the load balancer (HAProxy). There would still need to be an additional amphora VM that ran HAProxy.\n\nThe main question is if we are expanding the definition of \"amphora\" to include running services to implement the distributor and consider it valid to run amphoras with no load balancer enabled.","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":10850,"name":"German Eichberger","email":"german.eichberger@gmail.com","username":"german"},"change_message_id":"d809faae7ee92aec1edfcbc51db59cafe4280ba4","unresolved":false,"context_lines":[{"line_number":83,"context_line":"  to handle ECMP group flow remapping events and support amphora connection"},{"line_number":84,"context_line":"  draining."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"* [`P2`_] Support Floating IPs (FIPs). In order to support FIPs the existing"},{"line_number":87,"context_line":"  Neutron *floatingips* API would need to be extended. The API would need to"},{"line_number":88,"context_line":"  allow excluding specific IPs from getting SNAT\u0027ed to the FIP. In the case of"},{"line_number":89,"context_line":"  this design, the anycast VIP would need a rule to prevent SNAT to the FIP."}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_17ba32dd","line":86,"updated":"2017-06-13 21:22:41.000000000","message":"since an API in a different project needs to be extended this will require a separate spec. Let\u0027s say this will be \"described in more detail in a separate spec in the Neutron project\"","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"332c71ecb00b2fbbb76e04d26bcc8684cc126dd1","unresolved":false,"context_lines":[{"line_number":83,"context_line":"  to handle ECMP group flow remapping events and support amphora connection"},{"line_number":84,"context_line":"  draining."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"* [`P2`_] Support Floating IPs (FIPs). In order to support FIPs the existing"},{"line_number":87,"context_line":"  Neutron *floatingips* API would need to be extended. The API would need to"},{"line_number":88,"context_line":"  allow excluding specific IPs from getting SNAT\u0027ed to the FIP. In the case of"},{"line_number":89,"context_line":"  this design, the anycast VIP would need a rule to prevent SNAT to the FIP."}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_4fceb582","line":86,"in_reply_to":"7f231b9d_17ba32dd","updated":"2017-06-15 18:13:06.000000000","message":"Done","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":10850,"name":"German Eichberger","email":"german.eichberger@gmail.com","username":"german"},"change_message_id":"d809faae7ee92aec1edfcbc51db59cafe4280ba4","unresolved":false,"context_lines":[{"line_number":146,"context_line":"  │             ╞═══════════════════════════════════════════════Anycast"},{"line_number":147,"context_line":"  └─────────────┘                     1..*                      Network"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Whenever a new amphora is instantiated it will create BGP peering session(s)"},{"line_number":150,"context_line":"  over the lb-mgmt-net to the L3 fabric. The BGP peer will need to have a"},{"line_number":151,"context_line":"  neighbor definition in order to allow the peering sessions from the amphoras."},{"line_number":152,"context_line":"  In order to ease configuration, a neighbor statement allowing peers from the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_b7ca5e6a","line":149,"updated":"2017-06-13 21:22:41.000000000","message":"active-active amphora","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"332c71ecb00b2fbbb76e04d26bcc8684cc126dd1","unresolved":false,"context_lines":[{"line_number":146,"context_line":"  │             ╞═══════════════════════════════════════════════Anycast"},{"line_number":147,"context_line":"  └─────────────┘                     1..*                      Network"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"* Whenever a new amphora is instantiated it will create BGP peering session(s)"},{"line_number":150,"context_line":"  over the lb-mgmt-net to the L3 fabric. The BGP peer will need to have a"},{"line_number":151,"context_line":"  neighbor definition in order to allow the peering sessions from the amphoras."},{"line_number":152,"context_line":"  In order to ease configuration, a neighbor statement allowing peers from the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_2f6139d4","line":149,"in_reply_to":"7f231b9d_b7ca5e6a","updated":"2017-06-15 18:13:06.000000000","message":"Done","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":10850,"name":"German Eichberger","email":"german.eichberger@gmail.com","username":"german"},"change_message_id":"d809faae7ee92aec1edfcbc51db59cafe4280ba4","unresolved":false,"context_lines":[{"line_number":307,"context_line":"* auth_type ``(String(16) , nullable\u003dTrue)``"},{"line_number":308,"context_line":"    Authentication type, such as ``md5``."},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"* auth_password ``(String(255) , nullable\u003dTrue)``"},{"line_number":311,"context_line":"    Password for md5 authentication."},{"line_number":312,"context_line":""},{"line_number":313,"context_line":"* ttl_hops ``(Integer , nullable\u003dTrue)``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_379ace62","line":310,"updated":"2017-06-13 21:22:41.000000000","message":"we should avoid storing passwords/username in DB because this is a big security no-no. Now our database has sensitive information which complicatetes backup strategies. I would recommend if possible having this in a configuration file on disk (which potentially can be encrypted, delete after startup, etc.)","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"},{"author":{"_account_id":20279,"name":"Jason Niesz","email":"Jason.Niesz@walmart.com","username":"jniesz"},"change_message_id":"332c71ecb00b2fbbb76e04d26bcc8684cc126dd1","unresolved":false,"context_lines":[{"line_number":307,"context_line":"* auth_type ``(String(16) , nullable\u003dTrue)``"},{"line_number":308,"context_line":"    Authentication type, such as ``md5``."},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"* auth_password ``(String(255) , nullable\u003dTrue)``"},{"line_number":311,"context_line":"    Password for md5 authentication."},{"line_number":312,"context_line":""},{"line_number":313,"context_line":"* ttl_hops ``(Integer , nullable\u003dTrue)``"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7f231b9d_ba0c69ec","line":310,"in_reply_to":"7f231b9d_379ace62","updated":"2017-06-15 18:13:06.000000000","message":"How about storing this in Barbican?","commit_id":"7311b6857c1a164425d4f2efd4b18c0dbb631176"}]}