)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":5638,"name":"Davanum Srinivas","email":"davanum@gmail.com","username":"dims-v"},"change_message_id":"286a99e8eadcaef66cdcc44b9e4750b35512cbda","unresolved":false,"context_lines":[{"line_number":12,"context_line":"This means that all messaging patterns are implemented over a single socket"},{"line_number":13,"context_line":"pipeline."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"There were proposed to switch from a splitted universal"},{"line_number":16,"context_line":"forward(PUSH/PULL)+backward(PUB/SUB) pipeline to a unified REQ/REP"},{"line_number":17,"context_line":"bidirectional pipeline. The change simplifies the driver implementation and"},{"line_number":18,"context_line":"makes it more reliable but the pipeline still remains universal."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"5afe65bd_397989fd","line":15,"updated":"2015-06-02 00:11:22.000000000","message":"Oleksii, we have to make it clear that \n\n    1. This is a new driver, it will live parallel to the old driver\n    2. Existing users can choose to use the old or the new driver.\n    3. Once the new driver stabilizes, we will start a deprecation period for the old driver\n\nWe also need to factor in the following somewhere in the doc.\n\n    1. functional testing\n    2. devstack support\n    3. CI testing with actual devstack+tempest\n\nthanks,\ndims","commit_id":"4f70d71ec988c60aeebe40a102da2e5d0f702b67"}],"specs/liberty/zmq-patterns-usage.rst":[{"author":{"_account_id":7491,"name":"Viktor Serhieiev","email":"viktor.sergeyev@gmail.com","username":"vsergeyev"},"change_message_id":"b05238f5973604014c2e67188c4a98ac7c86f858","unresolved":false,"context_lines":[{"line_number":191,"context_line":"This gate should be used for the new driver to check it."},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"We also need some kind of multi-node deployment testing."},{"line_number":194,"context_line":"Some HA scenarios testing is also needed."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5afe65bd_3d1163ad","line":194,"updated":"2015-06-02 12:20:32.000000000","message":"these two are cool. Is there a similar testing in gates already?","commit_id":"c981cc214f27ce96579d01e7aafba040b5b47947"},{"author":{"_account_id":7491,"name":"Viktor Serhieiev","email":"viktor.sergeyev@gmail.com","username":"vsergeyev"},"change_message_id":"b05238f5973604014c2e67188c4a98ac7c86f858","unresolved":false,"context_lines":[{"line_number":252,"context_line":"Anticipated API Stabilization"},{"line_number":253,"context_line":"-----------------------------"},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"The new driver should successfully run with devstack."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5afe65bd_3d872311","line":255,"updated":"2015-06-02 12:20:32.000000000","message":"what about an existing functional driver-independent tests?","commit_id":"c981cc214f27ce96579d01e7aafba040b5b47947"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"5164ccb5d16cfc4e6d0aa46fbe3bc2bc37b33974","unresolved":false,"context_lines":[{"line_number":252,"context_line":"Anticipated API Stabilization"},{"line_number":253,"context_line":"-----------------------------"},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"The new driver should successfully run with devstack."},{"line_number":256,"context_line":""},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"Documentation Impact"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5afe65bd_982625b9","line":255,"in_reply_to":"5afe65bd_3d872311","updated":"2015-06-02 13:09:58.000000000","message":"Done","commit_id":"c981cc214f27ce96579d01e7aafba040b5b47947"},{"author":{"_account_id":7491,"name":"Viktor Serhieiev","email":"viktor.sergeyev@gmail.com","username":"vsergeyev"},"change_message_id":"b05238f5973604014c2e67188c4a98ac7c86f858","unresolved":false,"context_lines":[{"line_number":258,"context_line":"Documentation Impact"},{"line_number":259,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"We need to cover all aspects of the new driver with detailed documentation and UML charts."},{"line_number":262,"context_line":""},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5afe65bd_7d091b95","line":261,"updated":"2015-06-02 12:20:32.000000000","message":"Deployment guide will be the same?","commit_id":"c981cc214f27ce96579d01e7aafba040b5b47947"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"5164ccb5d16cfc4e6d0aa46fbe3bc2bc37b33974","unresolved":false,"context_lines":[{"line_number":258,"context_line":"Documentation Impact"},{"line_number":259,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"We need to cover all aspects of the new driver with detailed documentation and UML charts."},{"line_number":262,"context_line":""},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"Dependencies"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5afe65bd_b81a016d","line":261,"in_reply_to":"5afe65bd_7d091b95","updated":"2015-06-02 13:09:58.000000000","message":"Done","commit_id":"c981cc214f27ce96579d01e7aafba040b5b47947"},{"author":{"_account_id":6804,"name":"bruce-benjamin","email":"bruce.benjamin@jhuapl.edu","username":"bruce-benjamin"},"change_message_id":"4a28310bae415c97d65c307b63353e166bc80f58","unresolved":false,"context_lines":[{"line_number":48,"context_line":"The alternative path is to use dynamic port binding which in terms of zmq API"},{"line_number":49,"context_line":"means using of a :meth:`zmq.Socket.bind_to_random_port` method."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Despite this approach seems more attractive because of its brokerlessness"},{"line_number":52,"context_line":"it has some serious pitfalls which show the proxy solution as more simple"},{"line_number":53,"context_line":"and reliable."},{"line_number":54,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_86168453","line":51,"updated":"2015-06-03 17:02:36.000000000","message":"Suggested update: s/Despite this approach seems more attractive because of its brokerlessness it has some serious pitfalls which show the proxy solution as more simple and reliable/Despite the fact that brokerless dynamic port binding seems attractive, the following serious pitfalls show that the proxy solution is actually simpler and more reliable/","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":48,"context_line":"The alternative path is to use dynamic port binding which in terms of zmq API"},{"line_number":49,"context_line":"means using of a :meth:`zmq.Socket.bind_to_random_port` method."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Despite this approach seems more attractive because of its brokerlessness"},{"line_number":52,"context_line":"it has some serious pitfalls which show the proxy solution as more simple"},{"line_number":53,"context_line":"and reliable."},{"line_number":54,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_3e4a4fc6","line":51,"in_reply_to":"5afe65bd_86168453","updated":"2015-06-08 10:38:11.000000000","message":"Thanks for grammar nit!","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"    1. Dynamic port binding consumes more time to establish a connection."},{"line_number":56,"context_line":"       Time exceeding may happen if too many ports in a system are busy"},{"line_number":57,"context_line":"       and the function needs a lot of retries."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    2. Number of ports is limited to a number of 16-bit integer 2^16 \u003d 65535."},{"line_number":60,"context_line":"       So a number of unoccupied ports may also come to the end especially for"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_fa214cfc","line":57,"updated":"2015-06-03 17:13:57.000000000","message":"Binding to random port may take more instruction cycles than binding on a fixed port but this is negligible compared to other work happening when booting a node (won\u0027t be measurable). Moreover binding is happening only once when the stack is brought up. Assume we have 2 end points (e.g. compute node with ovs agent and nova compute), with proxy \u003d 3 binds (1 tcp bind on fixed port + 2 ipc bind which are unix socket binds by name), without proxy \u003d 2 tcp bind on random port. Basically we\u0027re trading off between 1 tcp bind on fixed port + N ipc bind with N tcp binds on random ports.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"b988973047fd09dbcb2c6ca03f71e17bf7a9964c","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"    1. Dynamic port binding consumes more time to establish a connection."},{"line_number":56,"context_line":"       Time exceeding may happen if too many ports in a system are busy"},{"line_number":57,"context_line":"       and the function needs a lot of retries."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    2. Number of ports is limited to a number of 16-bit integer 2^16 \u003d 65535."},{"line_number":60,"context_line":"       So a number of unoccupied ports may also come to the end especially for"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_b7ee3cb2","line":57,"in_reply_to":"3afb71cf_5ed74bbf","updated":"2015-06-11 02:35:23.000000000","message":"the point I want to make is that there might be some differences in instruction cycles (which can go either way) but it does not matter that much because it is negligible in the overall picture. For port binding, we\u0027re talking about usecs not even msec!\nI think there are much more relevant and critical issues to address.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"    1. Dynamic port binding consumes more time to establish a connection."},{"line_number":56,"context_line":"       Time exceeding may happen if too many ports in a system are busy"},{"line_number":57,"context_line":"       and the function needs a lot of retries."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    2. Number of ports is limited to a number of 16-bit integer 2^16 \u003d 65535."},{"line_number":60,"context_line":"       So a number of unoccupied ports may also come to the end especially for"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_5ed74bbf","line":57,"in_reply_to":"5afe65bd_fa214cfc","updated":"2015-06-08 10:38:11.000000000","message":"Here I assume that 1 TCP bind consumes much more time than 1 IPC so we can not say N tcp \u003d N ipc.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    2. Number of ports is limited to a number of 16-bit integer 2^16 \u003d 65535."},{"line_number":60,"context_line":"       So a number of unoccupied ports may also come to the end especially for"},{"line_number":61,"context_line":"       a large-scale clouds."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"    3. Less critical but significant complexity is a necessity to notify a"},{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_5a8740d7","line":61,"updated":"2015-06-03 17:13:57.000000000","message":"binding port numbers are local to each node and independent of the size of the cloud. When it comes to tcp connections you\u0027d need to look at pairs of ports (64K * 64K) between any given pair of host. Practically speaking, chances of running out of ports especially at system bring up time will never happen. In this case, we\u0027re talking about a dozen of binding port max (maybe on the controller node), this is really small change.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    2. Number of ports is limited to a number of 16-bit integer 2^16 \u003d 65535."},{"line_number":60,"context_line":"       So a number of unoccupied ports may also come to the end especially for"},{"line_number":61,"context_line":"       a large-scale clouds."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"    3. Less critical but significant complexity is a necessity to notify a"},{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_1e8f93cd","line":61,"in_reply_to":"5afe65bd_5a8740d7","updated":"2015-06-08 10:38:11.000000000","message":"Agree, we have to consider host + port which gives much more combinations.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"},{"line_number":65,"context_line":"       using Redis to pass information between nodes. Time for Redis to"},{"line_number":66,"context_line":"       sync tables has to be added to a connection time in 1. Here the delay"},{"line_number":67,"context_line":"       may be even greater than in 1, because of the network latency."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"All these points makes us to give preference to a proxy solution. It also"},{"line_number":70,"context_line":"has its drawbacks, but advantages are:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_ba61943b","line":67,"updated":"2015-06-03 17:13:57.000000000","message":"if a name server is not an option (and this is a big if) then the use of a proxy with a small number of well known ports is indeed needed and that would be IMHO the main valid reason to go that path. Note that using a name server should not be considered as out of line or unorthodox since this is a widely used pattern with well understood implications and expectations.\nAlso note that Redis could be used as a name server although other open source alternatives could provide a better fit (such as etcd which provides strong consistency, proven HA and encryption).\nOn the other hand the lack of a name server in a such a distributed and large scale system should be viewed as an exception especially when the number of proxies is proportional to the number of nodes in the cloud, to which we must be extremely careful about making sure that the proposed design will scale on very large deployments. Note that the RabbitMQ model also uses a broker (or proxy) but differs in the number of brokers (typically 3 brokers in a fully HA deployment for any scale), which is quite different from the proposed model.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"b988973047fd09dbcb2c6ca03f71e17bf7a9964c","unresolved":false,"context_lines":[{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"},{"line_number":65,"context_line":"       using Redis to pass information between nodes. Time for Redis to"},{"line_number":66,"context_line":"       sync tables has to be added to a connection time in 1. Here the delay"},{"line_number":67,"context_line":"       may be even greater than in 1, because of the network latency."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"All these points makes us to give preference to a proxy solution. It also"},{"line_number":70,"context_line":"has its drawbacks, but advantages are:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_5d521836","line":67,"in_reply_to":"3afb71cf_8742163d","updated":"2015-06-11 02:35:23.000000000","message":"I\u0027d suggest not to jump to conclusions (as in #1 we do not need a name server for CAST/CALL). We can discuss more about fanout off gerrit.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"},{"line_number":65,"context_line":"       using Redis to pass information between nodes. Time for Redis to"},{"line_number":66,"context_line":"       sync tables has to be added to a connection time in 1. Here the delay"},{"line_number":67,"context_line":"       may be even greater than in 1, because of the network latency."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"All these points makes us to give preference to a proxy solution. It also"},{"line_number":70,"context_line":"has its drawbacks, but advantages are:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_8742163d","line":67,"in_reply_to":"5afe65bd_ba61943b","updated":"2015-06-08 10:38:11.000000000","message":"The only common thing about zeromq proxy device and the RabbitMQ broker is messages dispatching. Rabbit broker is complicated broker solution which serves as messaging queue running on a controller(s) node and it is completely responsible for routing messages between nodes. Zmq proxy is a thread (function) to transfer messages from one socket to the other *on a node locally*. No message queue, no persistent state, no routing logic.\n\nThe name server solution is acceptable and it would work. But specific points are:\n1. We do not need a name server for CALL and direct CAST. Data from Target is enough to deliver message to the end-point. Requests to the name server will consume a lot of time.\n\n2. Name server becomes a single point of failure. Clustering for HA is what RabbitMQ does, but in practice the clustering solution is not reliable enough.\n\n3. We probably need a name server for FANOUT CAST. I\u0027m not yet ready to claim, but PUB/SUB to tcp://* (a whole network) may be a better solution in theory. We need some prototype for this to clarify details.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Important point is the proxy should stay as close to the rpc-server as possible"},{"line_number":77,"context_line":"and should be considered as a part of an rpc-server. The proxy is not a central broker"},{"line_number":78,"context_line":"to run it on a separate host even if it would be some kind of a virtual host."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"The proxy communicates with rpc-servers over IPC protocol and shields them from TCP network."},{"line_number":81,"context_line":"To implement the concept of a `Dedicated socket pipeline` we can use a separate proxy for"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_3a56c45c","line":78,"updated":"2015-06-03 17:13:57.000000000","message":"Adding a proxy server has implications on production installers (i.e. anything but devstack). Many deployments use containers or VMs to package openstack services (service cloud concept, see Kolla and triple-O). The proxy could be deployed bare metal, in a container or even in a VM (why not). The local end points can be deployed with the same options. That brings up an interesting array of combinations that have a direct impact on performance and control plane addressing. When an end point is in a VM, we cannot use zmq IPC (since they are based on Unix domain sockets) to communicate between the proxy (which may run bare metal or in a container) and a process in a VM - can only use tcp. When containers are used, the IPC path must be shared across containers.\nThe point I want to make is that adding a proxy is not a small decision when it comes to ease of deployment and may be an obstacle for adoption for more elaborate deployments.\nIn general do not limit design considerations to devstack! Nobody deploys under devstack.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"b988973047fd09dbcb2c6ca03f71e17bf7a9964c","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Important point is the proxy should stay as close to the rpc-server as possible"},{"line_number":77,"context_line":"and should be considered as a part of an rpc-server. The proxy is not a central broker"},{"line_number":78,"context_line":"to run it on a separate host even if it would be some kind of a virtual host."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"The proxy communicates with rpc-servers over IPC protocol and shields them from TCP network."},{"line_number":81,"context_line":"To implement the concept of a `Dedicated socket pipeline` we can use a separate proxy for"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_29ce74b4","line":78,"in_reply_to":"3afb71cf_e78b0a1d","updated":"2015-06-11 02:35:23.000000000","message":"Understood and agreed, I was not implying to run the proxy on another node.\nIf you run the proxy on each target node (to service the node local end points) then there is impact on all installers (because they need now to install one more package on every node) and there may be complication in the addressing part of the design (please re-read the above comment they all apply when the proxy is local to each node, for example when you use VMs as deployment model, you can\u0027t use IPC/UNIX socket domains even if the VMs run on the same node). These are considerations we need to worry now because they can hurt adoption.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Important point is the proxy should stay as close to the rpc-server as possible"},{"line_number":77,"context_line":"and should be considered as a part of an rpc-server. The proxy is not a central broker"},{"line_number":78,"context_line":"to run it on a separate host even if it would be some kind of a virtual host."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"The proxy communicates with rpc-servers over IPC protocol and shields them from TCP network."},{"line_number":81,"context_line":"To implement the concept of a `Dedicated socket pipeline` we can use a separate proxy for"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_e78b0a1d","line":78,"in_reply_to":"5afe65bd_3a56c45c","updated":"2015-06-08 10:38:11.000000000","message":"As I\u0027ve mentioned above the proxy is not a separate server(service), and running it on a separate host is a mistake of deployment. Running a proxy in a separate process is needed to share it between services on a node, but if this is a problem we can avoid it by assigning a different port for each service and each service will have its own proxy running in a thread. So the proxy should not be separated from a service. That\u0027s not correct.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":100,"context_line":"2. PUB/SUB for FANOUT CAST"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"The nice thing in zmq is possibility to publish to `tcp://*:port` and subscribing"},{"line_number":103,"context_line":"to a specific topic. This is exactly what is needed to implement FANOUT CAST."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"We can also do a direct CAST over this pipeline publishing to a specific target instead of `*`."},{"line_number":106,"context_line":"So, number 3 is optional."}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_3d435e44","line":103,"updated":"2015-06-03 17:13:57.000000000","message":"The slides show that the proxy would be acting as xsub and xpub.\nCan you precise exactly in this model who connects to who?\nLocal services (SUB) would connect to the local proxy (XPUB).\nIn the absence of any naming service, it looks like the publishers (PUB) will have to explicitly connect to the proxy servers (XSUB) running on *every* node in the cloud. \nCould you describe how publishers get the list of all nodes, what happens when new nodes are added/removed at runtime...\n \nzmq pub/sub/xpub/xsub do not guarantee delivery since messages will be silently dropped should any queue become full (that is made very clear in the zmq pub/sub spec).\nWill that be acceptable for Oslo messaging or are we assuming that queue full conditions will never happen?","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"b988973047fd09dbcb2c6ca03f71e17bf7a9964c","unresolved":false,"context_lines":[{"line_number":100,"context_line":"2. PUB/SUB for FANOUT CAST"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"The nice thing in zmq is possibility to publish to `tcp://*:port` and subscribing"},{"line_number":103,"context_line":"to a specific topic. This is exactly what is needed to implement FANOUT CAST."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"We can also do a direct CAST over this pipeline publishing to a specific target instead of `*`."},{"line_number":106,"context_line":"So, number 3 is optional."}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_09a5f8a0","line":103,"in_reply_to":"3afb71cf_47ae5e86","updated":"2015-06-11 02:35:23.000000000","message":"the second point (managing nodes) is where I think this solution might fall short. Nodes can be added/removed anytime after openstack is started. I think using the ZMQ pub/sub/xpub/xsub model will be dicey to operate because they basically require each publisher (and there can be many) to keep track of all the nodes at all times (since you have no real name service) and that sort of book keeping tends to make things more complex..\nFor example assume you start openstack with 10 nodes, then publishers start to send fanout messages to all 10 (the zmq driver will have to first explicitly connect to each of the 10 nodes before sending any message). When a new node is added, each publisher must be notified because it will have to connect to the new node. \nThe ZMQ guide makes it look easy and simple because it only shows 1 proxy (XSUB) and the publishers only need to connect to that one proxy. In our case we have as many proxies as there are nodes which could be a very large number (hundreds or thousands).\nThe possibility to have messages being silently dropped (when any queue gets full) is also a matter of concern we need to to look at.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":100,"context_line":"2. PUB/SUB for FANOUT CAST"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"The nice thing in zmq is possibility to publish to `tcp://*:port` and subscribing"},{"line_number":103,"context_line":"to a specific topic. This is exactly what is needed to implement FANOUT CAST."},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"We can also do a direct CAST over this pipeline publishing to a specific target instead of `*`."},{"line_number":106,"context_line":"So, number 3 is optional."}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_47ae5e86","line":103,"in_reply_to":"5afe65bd_3d435e44","updated":"2015-06-08 10:38:11.000000000","message":"I agree this part should be clarified.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":109,"context_line":"3. PUSH/PULL for direct CAST"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"As mentioned above this is optional, but PUSH/PULL is exact pattern for direct CAST,"},{"line_number":112,"context_line":"so should be the most optimal."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"4. PUB/SUB for Notifier"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_7d6f96c5","line":112,"updated":"2015-06-03 17:13:57.000000000","message":"seems to contradict slide page 9 (CAST new driver) which shows  DEALER/ROUTER/DEALER/REP","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":109,"context_line":"3. PUSH/PULL for direct CAST"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"As mentioned above this is optional, but PUSH/PULL is exact pattern for direct CAST,"},{"line_number":112,"context_line":"so should be the most optimal."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"4. PUB/SUB for Notifier"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_c70c8e96","line":112,"in_reply_to":"5afe65bd_7d6f96c5","updated":"2015-06-08 10:38:11.000000000","message":"Sorry for a little mess in the slides. Slide 9 actually shows the change in [1], while here in the spec we rather are talking about the slide 15.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":128,"context_line":"Current implementation of zmq driver uses Redis to pass host+topic"},{"line_number":129,"context_line":"linkage between nodes. In paragraphs above there was specified that PUB/SUB filtering"},{"line_number":130,"context_line":"attempts to resolve this problem. If PUB/SUB would not be enough for FANOUT"},{"line_number":131,"context_line":"implementation we can reuse the solution with Redis."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Redis is a great in-memory distributed storage but zmq driver should move away from"},{"line_number":134,"context_line":"external services to become self-sufficient driver."}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_3d6a3eb4","line":131,"updated":"2015-06-03 17:13:57.000000000","message":"I think getting rid of matchmaker would be a great simplification but what would \"would not be enough\" mean?","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"b988973047fd09dbcb2c6ca03f71e17bf7a9964c","unresolved":false,"context_lines":[{"line_number":128,"context_line":"Current implementation of zmq driver uses Redis to pass host+topic"},{"line_number":129,"context_line":"linkage between nodes. In paragraphs above there was specified that PUB/SUB filtering"},{"line_number":130,"context_line":"attempts to resolve this problem. If PUB/SUB would not be enough for FANOUT"},{"line_number":131,"context_line":"implementation we can reuse the solution with Redis."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Redis is a great in-memory distributed storage but zmq driver should move away from"},{"line_number":134,"context_line":"external services to become self-sufficient driver."}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_e9f4dc87","line":131,"in_reply_to":"3afb71cf_a7a2b249","updated":"2015-06-11 02:35:23.000000000","message":"I\u0027m fine with it as long as we assess properly the performance/scale characteristics of such design and  the impact on real deployments.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":128,"context_line":"Current implementation of zmq driver uses Redis to pass host+topic"},{"line_number":129,"context_line":"linkage between nodes. In paragraphs above there was specified that PUB/SUB filtering"},{"line_number":130,"context_line":"attempts to resolve this problem. If PUB/SUB would not be enough for FANOUT"},{"line_number":131,"context_line":"implementation we can reuse the solution with Redis."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"Redis is a great in-memory distributed storage but zmq driver should move away from"},{"line_number":134,"context_line":"external services to become self-sufficient driver."}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_a7a2b249","line":131,"in_reply_to":"5afe65bd_3d6a3eb4","updated":"2015-06-08 10:38:11.000000000","message":"We need to get the proof for PUB/SUB with a prototype application to make a decision here.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":141,"context_line":"CALL (REQ/REP) pipeline can pass heartbeats both ways (ping-pong heartbeating),"},{"line_number":142,"context_line":"while others work only in one direction. If we accept messages with payload they are"},{"line_number":143,"context_line":"also treated as heartbeats. Heartbeats should go over the same channel as real data."},{"line_number":144,"context_line":"We should not open a separate connection to pass heartbeats."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_1d6542e0","line":144,"updated":"2015-06-03 17:13:57.000000000","message":"why not simply use TCP keepalive?","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":141,"context_line":"CALL (REQ/REP) pipeline can pass heartbeats both ways (ping-pong heartbeating),"},{"line_number":142,"context_line":"while others work only in one direction. If we accept messages with payload they are"},{"line_number":143,"context_line":"also treated as heartbeats. Heartbeats should go over the same channel as real data."},{"line_number":144,"context_line":"We should not open a separate connection to pass heartbeats."},{"line_number":145,"context_line":""},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_6785daab","line":144,"in_reply_to":"5afe65bd_1d6542e0","updated":"2015-06-08 10:38:11.000000000","message":"Accepted.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":164,"context_line":"Performance Impact"},{"line_number":165,"context_line":"------------------"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"None."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"Configuration Impact"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_bd9ceec0","line":167,"updated":"2015-06-03 17:13:57.000000000","message":"It would be good I think if we could have some supportive data to show that the proposed design can scale and to what level it can scale. Comparison data with the RabbitMQ driver will also be necessary to convince users to switch over to zmq.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":164,"context_line":"Performance Impact"},{"line_number":165,"context_line":"------------------"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"None."},{"line_number":168,"context_line":""},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"Configuration Impact"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_67acba35","line":167,"in_reply_to":"5afe65bd_bd9ceec0","updated":"2015-06-08 10:38:11.000000000","message":"Accepted.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":170,"context_line":"Configuration Impact"},{"line_number":171,"context_line":"--------------------"},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"Configuration options should be added to setup pre-allocated ports."},{"line_number":174,"context_line":""},{"line_number":175,"context_line":""},{"line_number":176,"context_line":"Developer Impact"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_7da67611","line":173,"updated":"2015-06-03 17:13:57.000000000","message":"We need to precise what configuration parameters will need to be provisioned for an operational deployment.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":170,"context_line":"Configuration Impact"},{"line_number":171,"context_line":"--------------------"},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"Configuration options should be added to setup pre-allocated ports."},{"line_number":174,"context_line":""},{"line_number":175,"context_line":""},{"line_number":176,"context_line":"Developer Impact"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_4759be59","line":173,"in_reply_to":"5afe65bd_7da67611","updated":"2015-06-08 10:38:11.000000000","message":"Accepted.","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"e871f3793d57f188b6eea41a487a2dad93ccd444","unresolved":false,"context_lines":[{"line_number":191,"context_line":"This gate should be used for the new driver to check it."},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"We also need some kind of multi-node deployment testing."},{"line_number":194,"context_line":"Some HA scenarios testing is also needed."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5afe65bd_7d7d566d","line":194,"updated":"2015-06-03 17:13:57.000000000","message":"Absolutely agree!","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"78daf5ce8a54fdcd922d4f9e247d4bda4f4340d5","unresolved":false,"context_lines":[{"line_number":191,"context_line":"This gate should be used for the new driver to check it."},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"We also need some kind of multi-node deployment testing."},{"line_number":194,"context_line":"Some HA scenarios testing is also needed."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":5,"id":"3afb71cf_67739ae8","line":194,"in_reply_to":"5afe65bd_7d7d566d","updated":"2015-06-08 10:38:11.000000000","message":":)","commit_id":"2c97204aba96b2bd5817e81db0f368a3d7af1765"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"521bf766385e5c70be8e297482b23c610213ffdb","unresolved":false,"context_lines":[{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"},{"line_number":65,"context_line":"       using Redis to pass information between nodes. Time for Redis to"},{"line_number":66,"context_line":"       sync tables has to be added to a connection time in 1. Here the delay"},{"line_number":67,"context_line":"       may be even greater than in 1, because of the network latency."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"All these points makes us to give preference to a proxy solution. It also"},{"line_number":70,"context_line":"has its drawbacks, but advantages are:"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_90186086","line":67,"updated":"2015-06-29 18:43:20.000000000","message":"I\u0027ll have to disagree on #1 and #2 described as \"serious pitfalls\".\nI have commented already in previous reviews that none of the 3 is serious and yet have not seen any reply stating the contrary based on any evidence.\n\n#3 could be moot (see DS discussion below)","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"9474fda85a59754316eeee9fc242760f93098cef","unresolved":false,"context_lines":[{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"},{"line_number":65,"context_line":"       using Redis to pass information between nodes. Time for Redis to"},{"line_number":66,"context_line":"       sync tables has to be added to a connection time in 1. Here the delay"},{"line_number":67,"context_line":"       may be even greater than in 1, because of the network latency."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"All these points makes us to give preference to a proxy solution. It also"},{"line_number":70,"context_line":"has its drawbacks, but advantages are:"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_b3578001","line":67,"in_reply_to":"ba3cc151_82e7a8e3","updated":"2015-06-30 16:03:49.000000000","message":"I see your point.\n\nWhat I have been suggesting is more about a general design that is clean and scales, and from that angle what you propose is really a variant of that design that introduces \"shortcuts\" (not in a negative sense but for the sake of optimization as you describe):\n\na) service ports are fixed/hardcoded to avoid an initial lookup\n\nb) add a proxy to do the local forwarding \n\nAlthough not my preferred option, I\u0027m fine with using fixed ports (a).\nAs for (b), as long as I can see a \"workable\" path to introduce a proxy-less design variant from the code that is committed, I am fine with the proposal as it is a great step in the right direction.\n\n\u003eAs we bind server to any host (tcp://*) the number of available address combinations shortens to 64k. So, do we really like to have such kind of an architectural limitation even if it seems to be unreachable? Can we guarantee that exceeding of the port numbers will never happen? Formally we can not.\n\nAs you say it that \"architectural limitation\" is baked-in in all apps that run on Linux and use port binding. And in that regard, all these apps would be associated to a \"serious pitfall\".\n\nThe range of available ports for dynamic binding is a configuration of the linux tcp stack. In general it is in the range of 32K-49K to 64K (depends on the distro default config).\nIn theory you can run out of dynamic port number for local binding but I can guarantee you that if that ever happens you\u0027ll have much more serious trouble than dealing with a failure in the oslo messaging layer. In practice this never happens at start time (since the binding occurs when you start the system it would mean that you\u0027d have well over 10,000 dynamic ports used when you start OpenStack which is pretty much unimaginable on a cloud node).\nBut this argument of port binding is almost nitpicking given that there are so many other things to get right, so I\u0027m going to close these comments and get over it if you do not want to remove these \"serious\" pitfalls from the spec ;-)\n\n\nNote that using fixed port numbers normally requires registration of these ports to the IETF/IANA (I see that 9501-9521 is currently unassigned). Maybe OpenStack should register all those port numbers before someone else reserves them? Because then we could be in a big trouble.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"04ccd577877f9b838a9a12b4f28d08ec74b1e1a7","unresolved":false,"context_lines":[{"line_number":64,"context_line":"       client about the chosen port somehow. Such mechanism may be implemented"},{"line_number":65,"context_line":"       using Redis to pass information between nodes. Time for Redis to"},{"line_number":66,"context_line":"       sync tables has to be added to a connection time in 1. Here the delay"},{"line_number":67,"context_line":"       may be even greater than in 1, because of the network latency."},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"All these points makes us to give preference to a proxy solution. It also"},{"line_number":70,"context_line":"has its drawbacks, but advantages are:"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_82e7a8e3","line":67,"in_reply_to":"ba3cc151_90186086","updated":"2015-06-30 12:18:53.000000000","message":"I respectfully disagree with previous comments.\n\nShort answer: We are not considering dynamic port binding in current solution because of it\u0027s unneeded additional complexity in most cases. \n\nIn details:\n\n\u003e I\u0027d suggest not to jump to conclusions (as in #1 we do not need a name server for CAST/CALL).\n\nWhy do we need a name server for direct CALL and direct CAST? All the necessary information is available for us from the target. Why to bring some additional steps to a simple procedure? We get rid of a local proxy but rely on a remote name server?\n\n\u003e  Note that using a name server should not be considered as out of line or unorthodox since this is a widely used pattern with well understood implications and expectations\n\u003e Note that the RabbitMQ model also uses a broker (or proxy) but differs in the number of brokers\n\nI do not consider the name service (or any other solution) unorthodox or something, as you can see I\u0027m trying to figure out all pros and cons and then to take a final decision. The overall purpose of this specific driver is to provide a benefits of scalability compared to a central broker solution which we have with RabbitMQ and Qpid. Which means we have to reduce dependency on central nodes as much as we can. I see 2 wins given by the currently described solution:\n1. We do not pass messages over a central node, so we have peer nodes. (Please, do not consider a proxy as a RabbitMQ broker, earlier I\u0027ve already described why)\n2. We use name service for fanout only (notifier maybe) - other patterns don\u0027t depend on the name service.\n\n\u003e binding port numbers are local to each node and independent of the size of the cloud. When it comes to tcp connections you\u0027d need to look at pairs of ports (64K * 64K) between any given pair of host.\n\nAs we bind server to any host (tcp://*) the number of available address combinations shortens to 64k. So, do we really like to have such kind of an architectural limitation even if it seems to be unreachable? Can we guarantee that exceeding of the port numbers will never happen? Formally we can not.\n\n\u003e Binding to random port may take more instruction cycles than binding on a fixed port but this is negligible\n\nNot only instruction cycles but also additional requests over the network (requests to name service from both sides) to sync the server with the client on the same port.\n\n\u003e but it does not matter that much because it is negligible in the overall picture.\n\nNegligible non-zero value becomes zero value with proxy, so I vote for it )).\n\n\u003e If you run the proxy on each target node (to service the node local end points) then there is impact on all installers\n\nWe have impact on all installers in any case (name service is also an issue) - deployment guide should cover all this in details.\n\n\u003e for example when you use VMs as deployment model you can\u0027t use IPC/UNIX socket domains even if the VMs run on the same node\n\nI agree that Kolla may probably be the case where IPC won\u0027t work, and dynamic port binding may be a solution. But AFAIK it is not widely applied at the moment. So we can postpone dynamic port binding implementation.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"521bf766385e5c70be8e297482b23c610213ffdb","unresolved":false,"context_lines":[{"line_number":119,"context_line":"without DS up and running. So for real-life deployments all standard HA"},{"line_number":120,"context_line":"methods should be applied for the service."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The first candidate for DS implementation is Redis, but it is not necessarily so."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"It worth noting that using of a DS doesn\u0027t mimic a central-broker concept."},{"line_number":125,"context_line":"All messages go from node to node directly, therefore we still stay"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_822b3aeb","line":122,"updated":"2015-06-29 18:43:20.000000000","message":"Thanks for adding this, that is a great addition!\nIt would be great to define an interface so that we can plug in an alternate implementation.\nA potential issue with Redis is that clusters may still not be production ready but there are open source alternatives that can work as well or better.\nI think the spec should capture this concept of abstract DS service with pluggable implementation and define a more detailed description of what the abstract DS service interface looks like.\n\nRegarding the use of DS, we need to clarify in the spec the sequence of events (interaction diagram) between the different parties: publishers, DS, subscribers and proxies (if proxies are used), and how topic filtering is done.\n\nThere are basically 2 ways to get everybody connected:\n- publisher initiated connection\n- subscriber or proxy initiated connection\n\nPublisher initiated connection:\nPublishers register to the DS for notification for a given topic\nSubscribers or proxies registers their address+topic to the DS when they start, this will trigger notification by the DS to all publishers interested in topic\nPublisher upon reception of notification will connect to the new recipient and start sending messages to it,\n\nSubscriber/Proxy initiated connection:\nIs the reverse, subscribers/proxies register to the DS for notification of given topic.\nWhen a publisher starts it registers its address+topic to the DS.\nSubscribers/proxies are notified by the DS if the topic matches, then connect to the publisher, then receive fanout messages as they are sent by the publisher.\n\nWhich way were you considering (or maybe you had a different way of connecting) ?\n\nNote that in either case, the connection acceptor (the one that binds) could bind on a well known port or use a random port and advertise that port (with the host IP) to the DS, so the initiator does not have to reserve/hardcode the destination port#.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"288c27a48a362398b3a17f840349b2258c132226","unresolved":false,"context_lines":[{"line_number":119,"context_line":"without DS up and running. So for real-life deployments all standard HA"},{"line_number":120,"context_line":"methods should be applied for the service."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The first candidate for DS implementation is Redis, but it is not necessarily so."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"It worth noting that using of a DS doesn\u0027t mimic a central-broker concept."},{"line_number":125,"context_line":"All messages go from node to node directly, therefore we still stay"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_54ae1786","line":122,"in_reply_to":"ba3cc151_3151250f","updated":"2015-06-30 17:26:40.000000000","message":"ok that makes it more clear and makes it close to what I call publisher initiated connection.\nIn my description I use DS notification in order to deal with subscriber discovery since subscribers can come and go based on deployment schedules.\nA simple lookup as you propose (publisher retrieves the list of matching subscribers when sending a fanout) would be either costly (if you do a lookup per fanout message) or incomplete (if you cache the matching list and subscribers come and go).\nIn general notifications would be used to notify publishers that something changed so they can update their cached list.\n\nOne important note is that, in all cases (even for notification based updates), subscribers might miss some fanout messages at the time they start (this is well documented in the ZMQ guide) and I\u0027m not sure feature code using oslo messaging are aware about that peculiarity. This can also apply to rabbitMQ (it is a matter of timing between the sending of fanout messages and the subscriber being ready to accept fanout messages). In general there should be some kind of \"phase locking\" or sync up happening between the subscriber and the publisher to make sure the subscriber hooks up to the fanout stream without losing important fanout messages. I\u0027m not sure existing oslo messaging users implement this properly today.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"9474fda85a59754316eeee9fc242760f93098cef","unresolved":false,"context_lines":[{"line_number":119,"context_line":"without DS up and running. So for real-life deployments all standard HA"},{"line_number":120,"context_line":"methods should be applied for the service."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The first candidate for DS implementation is Redis, but it is not necessarily so."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"It worth noting that using of a DS doesn\u0027t mimic a central-broker concept."},{"line_number":125,"context_line":"All messages go from node to node directly, therefore we still stay"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_fe986f1d","line":122,"in_reply_to":"ba3cc151_5dc2e186","updated":"2015-06-30 16:03:49.000000000","message":"Sorry could you rephrase using publisher/subscriber terms instead of client and server/listener which is ambiguous? Who would initiate the connection and how would the DS interact? I described 2 possible ways of doing this (publisher initiated connection or subscriber initiated connection), which one is it of maybe is it a 3rd way of doing this?","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"04ccd577877f9b838a9a12b4f28d08ec74b1e1a7","unresolved":false,"context_lines":[{"line_number":119,"context_line":"without DS up and running. So for real-life deployments all standard HA"},{"line_number":120,"context_line":"methods should be applied for the service."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The first candidate for DS implementation is Redis, but it is not necessarily so."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"It worth noting that using of a DS doesn\u0027t mimic a central-broker concept."},{"line_number":125,"context_line":"All messages go from node to node directly, therefore we still stay"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_5dc2e186","line":122,"in_reply_to":"ba3cc151_822b3aeb","updated":"2015-06-30 12:18:53.000000000","message":"The driver works with an assumption that listeners always start before any request from the client side comes. With fanout DS should be updated from server(listener)-side before the client starts to ask it.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":13290,"name":"Oleksii Zamiatin","email":"ozamiatin@mirantis.com","username":"ozamiatin"},"change_message_id":"c4962cdcea3570aa7d5e70f7fc1765a6cef6474b","unresolved":false,"context_lines":[{"line_number":119,"context_line":"without DS up and running. So for real-life deployments all standard HA"},{"line_number":120,"context_line":"methods should be applied for the service."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"The first candidate for DS implementation is Redis, but it is not necessarily so."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"It worth noting that using of a DS doesn\u0027t mimic a central-broker concept."},{"line_number":125,"context_line":"All messages go from node to node directly, therefore we still stay"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_3151250f","line":122,"in_reply_to":"ba3cc151_fe986f1d","updated":"2015-06-30 16:54:12.000000000","message":"Server is a subscriber, client is a publisher. DS doesn\u0027t notify but replies on requests from publishers/subscribers. Subscriber should request DS to register topic + host when it starts listening. So, before the publisher does fanout it requests the list of hosts from DS by topic and then publishes message to all received addresses.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"521bf766385e5c70be8e297482b23c610213ffdb","unresolved":false,"context_lines":[{"line_number":126,"context_line":"peer-to-peer on messages level which is important for scalability."},{"line_number":127,"context_line":"We also don\u0027t waste time requesting DS while using a direct messaging patterns like CALL."},{"line_number":128,"context_line":"DS state caching on local brokers (proxy) may also help to reduce dependency"},{"line_number":129,"context_line":"on a central node(s). But latter is a kind of optimization matter."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"3. PUSH/PULL for direct CAST"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_0bdc25e3","line":129,"updated":"2015-06-29 18:43:20.000000000","message":"Worst case you have 1 DS lookup which can be cached, so this will not be a significant or noticeable overhead.\nFurthermore, if the local cache fails, another lookup could be used to pick the new address, which solves the problem of service relocation. Today the code relies so much on hardcoded addresses it makes it difficult to relocate services to another node unless it has the same IP address.\nMore generally, use of a DS in any form (DNS, etcd, mDNS, Homenet...) is today considered mainstream, scalable and HA, so I\u0027m a bit surprised of the reticence to use it in oslo messaging. If we don\u0027t use a DS we will be reinventing the wheel or be pushed in a corner with limitations of all sorts on how to configure/deploy services in an agile way.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"},{"author":{"_account_id":11744,"name":"Alec Hothan","email":"ahothan@cisco.com","username":"ahothan"},"change_message_id":"521bf766385e5c70be8e297482b23c610213ffdb","unresolved":false,"context_lines":[{"line_number":143,"context_line":"from each other we need to make this in a separate pipeline in spite of 2 also uses PUB/SUB."},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Notifier is not as widely used as an RPC (1-3), so this item may be optionally implemented"},{"line_number":146,"context_line":"in next iterations. Ceilometer seems to be the main client of the pattern."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"Heartbeats"}],"source_content_type":"text/x-rst","patch_set":7,"id":"ba3cc151_cbc4fd41","line":146,"updated":"2015-06-29 18:43:20.000000000","message":"If we consider the broker less pub/sub (no proxy), we be able to support both fanout and notification with the same code.","commit_id":"b0bcde3ed31cced0d39e99288de8d5fe40b49b67"}]}