)]}'
{"specs/juno/central-agent-ha.rst":[{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"e8339caeba4ffd92ed1f96348b1713ad8e6f31ce","unresolved":false,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Horizontal scaling and HA of the Central Agent"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/ceilometer/+spec/central-agent-ha"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_65529137","line":8,"updated":"2014-08-01 03:33:13.000000000","message":"Observation: IIUC this approach is more related to the horizontal scaling aspect (by enabling workload partitioning across multiple running central agents), than high availability (in the sense of ensuring there is a minimum quorum of central agents running, and restarting if necessary).\n\nI wonder should that distinction be reflected in the title?\n\n(Though IIRC, a similar observation could to applied equally to the other central agent spec[1])\n\n[1] https://review.openstack.org/#/c/101282/5/specs/juno/central_agent_ha_active_active.rst","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":8052,"name":"Nejc Saje","email":"nejc@saje.info","username":"nsaje"},"change_message_id":"b11c8ae5166b05ab1f3a7d86c080627a87f176db","unresolved":false,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Horizontal scaling and HA of the Central Agent"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/ceilometer/+spec/central-agent-ha"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_b9777007","line":8,"in_reply_to":"baada198_65529137","updated":"2014-08-01 06:31:53.000000000","message":"But wouldn\u0027t the proposed solution actually provide active/active HA? You can have multiple agents running and if one crashes, others automatically take over his work.\n\nWikipedia says there are three principles of HA:\n1. Elimination of single points of failure. [check]\n2. Reliable crossover [check]\n3. Detection of failures as they occur [check, via tooz group membership using heartbeats]","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"e8339caeba4ffd92ed1f96348b1713ad8e6f31ce","unresolved":false,"context_lines":[{"line_number":38,"context_line":""},{"line_number":39,"context_line":"* Discovery plugins on different Central Agents that discover the same set of"},{"line_number":40,"context_line":"  resources would join the same group in *tooz*."},{"line_number":41,"context_line":"* For each polling interval, after retrieving the set of resources to poll,"},{"line_number":42,"context_line":"  they would get a list of all group members (Discovery plugins in other"},{"line_number":43,"context_line":"  Central Agents that have the same set of resources)."},{"line_number":44,"context_line":"* They would use hashing to determine the set of resources assigned to the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_a5774907","line":41,"updated":"2014-08-01 03:33:13.000000000","message":"Clearer as:\n\ns/after retrieving the set of resources to poll/after independently discovering the global set of resources that need to be polled by one or other of the central agents/","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"e8339caeba4ffd92ed1f96348b1713ad8e6f31ce","unresolved":false,"context_lines":[{"line_number":40,"context_line":"  resources would join the same group in *tooz*."},{"line_number":41,"context_line":"* For each polling interval, after retrieving the set of resources to poll,"},{"line_number":42,"context_line":"  they would get a list of all group members (Discovery plugins in other"},{"line_number":43,"context_line":"  Central Agents that have the same set of resources)."},{"line_number":44,"context_line":"* They would use hashing to determine the set of resources assigned to the"},{"line_number":45,"context_line":"  Discovery\u0027s Central Agent."},{"line_number":46,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_256b3920","line":43,"updated":"2014-08-01 03:33:13.000000000","message":"Is a list really required, or is it just the *number* of members that\u0027s significant here?\n\n(so that the number of buckets used by the hash can be sized correctly)\n\nAlso, is there *separate* group membership for:\n\n* each discovery extension within an agent, or\n* just the agent itself\n\nI was thinking only the latter would be required, but the lines 42-43 seem ambiguous.","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":8052,"name":"Nejc Saje","email":"nejc@saje.info","username":"nsaje"},"change_message_id":"b11c8ae5166b05ab1f3a7d86c080627a87f176db","unresolved":false,"context_lines":[{"line_number":40,"context_line":"  resources would join the same group in *tooz*."},{"line_number":41,"context_line":"* For each polling interval, after retrieving the set of resources to poll,"},{"line_number":42,"context_line":"  they would get a list of all group members (Discovery plugins in other"},{"line_number":43,"context_line":"  Central Agents that have the same set of resources)."},{"line_number":44,"context_line":"* They would use hashing to determine the set of resources assigned to the"},{"line_number":45,"context_line":"  Discovery\u0027s Central Agent."},{"line_number":46,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_b993f0fe","line":43,"in_reply_to":"baada198_256b3920","updated":"2014-08-01 06:31:53.000000000","message":"1. We need a list currently, since we need to know our position in that list.\n2. Let\u0027s discuss this below.","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"e8339caeba4ffd92ed1f96348b1713ad8e6f31ce","unresolved":false,"context_lines":[{"line_number":72,"context_line":""},{"line_number":73,"context_line":"For example, if a Discovery plugin isn\u0027t constrained to a subset of"},{"line_number":74,"context_line":"resources, as is the case for most Discovery plugins, then it should simply"},{"line_number":75,"context_line":"be in a statically named group like \u0027network-discovery-group\u0027."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"If, on the other hand, the resources that the Discovery plugin can discover"},{"line_number":78,"context_line":"are constrained, like in the case of Compute Discovery, then the group name"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_25d979f4","line":75,"updated":"2014-08-01 03:33:13.000000000","message":"I don\u0027t understand the need for this discovery group name, if the tooz group membership is at the level of individual agents. Can you elaborate?","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":8052,"name":"Nejc Saje","email":"nejc@saje.info","username":"nsaje"},"change_message_id":"b11c8ae5166b05ab1f3a7d86c080627a87f176db","unresolved":false,"context_lines":[{"line_number":72,"context_line":""},{"line_number":73,"context_line":"For example, if a Discovery plugin isn\u0027t constrained to a subset of"},{"line_number":74,"context_line":"resources, as is the case for most Discovery plugins, then it should simply"},{"line_number":75,"context_line":"be in a statically named group like \u0027network-discovery-group\u0027."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"If, on the other hand, the resources that the Discovery plugin can discover"},{"line_number":78,"context_line":"are constrained, like in the case of Compute Discovery, then the group name"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_fef8ee7f","line":75,"in_reply_to":"baada198_25d979f4","updated":"2014-08-01 06:31:53.000000000","message":"You\u0027re right. In the case of non-constrained discoveries, separate groups per-discovery aren\u0027t necessary.","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"e8339caeba4ffd92ed1f96348b1713ad8e6f31ce","unresolved":false,"context_lines":[{"line_number":78,"context_line":"are constrained, like in the case of Compute Discovery, then the group name"},{"line_number":79,"context_line":"should reflect their scope. An example of this would be"},{"line_number":80,"context_line":"\u0027compute-\u003chostname\u003e-discovery\u0027. This only the pollsters that are polling the"},{"line_number":81,"context_line":"same host will share their workload between them."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"Alternatives"},{"line_number":84,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_8545e59c","line":81,"updated":"2014-08-01 03:33:13.000000000","message":"Do we need to consider the compute agent in this spec, given that this is naturally self-scaling? i.e. the compute agent running on a compute node is only responsible for polling the instances running on that node.\n\n(though of course that self-scaling nature doesn\u0027t address the SPoF aspect, e.g. ensuring a compute agent is restarted if necessary after failure)","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":8052,"name":"Nejc Saje","email":"nejc@saje.info","username":"nsaje"},"change_message_id":"b11c8ae5166b05ab1f3a7d86c080627a87f176db","unresolved":false,"context_lines":[{"line_number":78,"context_line":"are constrained, like in the case of Compute Discovery, then the group name"},{"line_number":79,"context_line":"should reflect their scope. An example of this would be"},{"line_number":80,"context_line":"\u0027compute-\u003chostname\u003e-discovery\u0027. This only the pollsters that are polling the"},{"line_number":81,"context_line":"same host will share their workload between them."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"Alternatives"},{"line_number":84,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_f99f28b1","line":81,"in_reply_to":"baada198_8545e59c","updated":"2014-08-01 06:31:53.000000000","message":"With \u0027scoping\u0027 the discoveries I wanted to enable for example running multiple constrained discoveries. So you could run multiple compute agents per node and have them distribute workload between them.\n\nIn the future, I guess the discoveries could even be constrained by cell/region/etc... Or do you think this is not necessary and having one global group of agents is enough?","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"87e3b700d8f3cdbc20aac071ccf8107a055c100e","unresolved":false,"context_lines":[{"line_number":78,"context_line":"are constrained, like in the case of Compute Discovery, then the group name"},{"line_number":79,"context_line":"should reflect their scope. An example of this would be"},{"line_number":80,"context_line":"\u0027compute-\u003chostname\u003e-discovery\u0027. This only the pollsters that are polling the"},{"line_number":81,"context_line":"same host will share their workload between them."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"Alternatives"},{"line_number":84,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_49d91ff3","line":81,"in_reply_to":"baada198_f99f28b1","updated":"2014-08-01 09:57:23.000000000","message":"OK, that\u0027s an interesting example.\n\nAs it happens the compute agent is currently a kind of special case, in the sense that a single per-agent default discovery extension is used, instead of potentially multiple discovery agents being loaded for the pipeline sources relevant to this agent.\n\nSee:\n\n* https://github.com/openstack/ceilometer/blob/master/ceilometer/compute/manager.py#L28\n* https://github.com/openstack/ceilometer/blob/master/ceilometer/agent.py#L103\n\nIn which case, the discovery extension and the agent itself could alternatively be used as the \"identity\".\n\nHowever I guess in the most general-case, we\u0027d want to be able to use any discovery pattern in the compute agent.","commit_id":"231bb0076c309fbad61ea926b1a46295662b164e"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"d1bc541f6b809287102e96abb2d0e895c11d67dd","unresolved":false,"context_lines":[{"line_number":28,"context_line":"of resources to poll from its Discovery plugin (configured in the pipeline"},{"line_number":29,"context_line":"or a default one). This makes the discovery process a great place to implement"},{"line_number":30,"context_line":"the coordination and partitioning logic, while the pollsters themselves can"},{"line_number":31,"context_line":"remain in blissful ignorance of anything going on."},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"Proposed change"},{"line_number":34,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_a4bf3dfb","line":31,"updated":"2014-08-01 12:00:14.000000000","message":"/me loves blissful ignorance","commit_id":"943de797f67b8901c26c47779bb699237744e8aa"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"d1bc541f6b809287102e96abb2d0e895c11d67dd","unresolved":false,"context_lines":[{"line_number":44,"context_line":"  Central Agents that have the same set of resources)."},{"line_number":45,"context_line":"* They would use hashing to determine the set of resources assigned to the"},{"line_number":46,"context_line":"  Discovery\u0027s Central Agent."},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"**Determining the resources we\u0027re responsible for**"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"We have a list of resources and get a list of active agents from *tooz*. We"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_a4773d3a","line":47,"updated":"2014-08-01 12:00:14.000000000","message":"How and when does an agent leave a group? More importantly, how does tooz (and thus the other agents) know when a group has fallen off the map (for any reason)? I can guess that tooz has some mechanism for but it would be useful for that to be (lightly) described here. Otherwise the scalability of the system is well described, but not its resiliency.\n\nThe appears to be a bit on this below in the alternatives,as heartbeats.\n\nCan tooz have expiry on group memberships? If that\u0027s so what about membership periods which are polling cycle + 1 second?","commit_id":"943de797f67b8901c26c47779bb699237744e8aa"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"30b7c7afffb4bacffb54c73a017df736ebd2ad36","unresolved":false,"context_lines":[{"line_number":44,"context_line":"  Central Agents that have the same set of resources)."},{"line_number":45,"context_line":"* They would use hashing to determine the set of resources assigned to the"},{"line_number":46,"context_line":"  Discovery\u0027s Central Agent."},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"**Determining the resources we\u0027re responsible for**"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"We have a list of resources and get a list of active agents from *tooz*. We"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_7f71ee43","line":47,"in_reply_to":"baada198_a09f09b2","updated":"2014-08-04 12:39:32.000000000","message":"Eoghan explication is definitely right. You can read the doc online at http://tooz.rtfd.org/ btw :)","commit_id":"943de797f67b8901c26c47779bb699237744e8aa"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"ffa77cd27fc0166135d3b19781a198534cb84fbf","unresolved":false,"context_lines":[{"line_number":44,"context_line":"  Central Agents that have the same set of resources)."},{"line_number":45,"context_line":"* They would use hashing to determine the set of resources assigned to the"},{"line_number":46,"context_line":"  Discovery\u0027s Central Agent."},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"**Determining the resources we\u0027re responsible for**"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"We have a list of resources and get a list of active agents from *tooz*. We"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_a09f09b2","line":47,"in_reply_to":"baada198_a4773d3a","updated":"2014-08-02 13:14:07.000000000","message":"@cdent:\n\nThere\u0027s some basic tooz documentation on the semantics of group membership here:\n\n  https://github.com/stackforge/tooz/blob/master/doc/source/tutorial/group_membership.rst\n\nand also a simple example showing the callback-on-membership-change pattern: \n\n  https://github.com/stackforge/tooz/blob/master/examples/group_membership_watch.py\n\nThere also seems to be a mechanism for controlling the frequency of internal heartbeats:\n\n  https://github.com/stackforge/tooz/blob/master/doc/source/tutorial/coordinator.rst\n\n\"depending on your application, you may want to send the heartbeat at different moment or intervals.\"\n\nI\u0027m not sure there actually is a concept of a joining tooz group with membership limited upfront to a specific period, i.e. the operation for joining a group doesn\u0027t seem to provide that degree of freedom:\n\n  https://github.com/stackforge/tooz/blob/master/tooz/coordination.py#L205\n\nInstead I suspect the membership remains in force until either the member explicitly leaves, or is detected by the underying coordination protocol driver to have fallen off the map.","commit_id":"943de797f67b8901c26c47779bb699237744e8aa"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"d1bc541f6b809287102e96abb2d0e895c11d67dd","unresolved":false,"context_lines":[{"line_number":101,"context_line":"  Another approach would be to use distributed locking provided by *tooz*."},{"line_number":102,"context_line":"  Before a pollster would poll a resource, it\u0027d need to acquire its lock."},{"line_number":103,"context_line":"  Pollsters contend for the locks and whoever gets the lock, polls the"},{"line_number":104,"context_line":"  resource."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"Data model impact"},{"line_number":107,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_846b191e","line":104,"updated":"2014-08-01 12:00:14.000000000","message":"At first glance this seems the lightest in weight of all the proposals as it seems to require the least brains and there\u0027s no partitioning/bucketing. Thus manifests the most blissful ignorance. Such is the nature of contending rather the cooperating or coordinated systems.\n\nHowever I\u0027ve heard that this method has potential issues with overloading the services being polled in a kind of stampeding herd way. If that\u0027s the case it would be useful to explain here why this presents that problem and why the proposed solution does not.\n\nFinally: The underlying purpose of the coordination is to ensure that we do not record multiple samples of the same thing. This suggests that it could also be possible to handle this problem by instead of _preventing_ duplicates being created, we remove duplicates elsewhere in the stack. However, this is likely not pragmatic as it too has problems with overloading the services, but may be a fruitful venue of Gedankenexperimenting.","commit_id":"943de797f67b8901c26c47779bb699237744e8aa"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"d1bc541f6b809287102e96abb2d0e895c11d67dd","unresolved":false,"context_lines":[{"line_number":157,"context_line":"  nejc-saje"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"Other contributors:"},{"line_number":160,"context_line":"  nejc-saje"},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"Ongoing maintainer:"},{"line_number":163,"context_line":"  ceilometer team"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_a4681d22","line":160,"updated":"2014-08-01 12:00:14.000000000","message":"If it is cool with others I\u0027d like it get in on this.","commit_id":"943de797f67b8901c26c47779bb699237744e8aa"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"b6be807a2642ec5922d405a9d9fb2c8f55993997","unresolved":false,"context_lines":[{"line_number":120,"context_line":"        all_alarms \u003d get_all_alarms()"},{"line_number":121,"context_line":"        my_alarms \u003d partition_coordinator.get_my_subset(all_alarms)"},{"line_number":122,"context_line":"        for a in my_alarms:"},{"line_number":123,"context_line":"            evaluate(a)"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"or in the case of the central agent::"},{"line_number":126,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"baada198_d3911147","line":123,"updated":"2014-08-04 14:13:24.000000000","message":"You could just make a note that the change-over of the alarm partitioning cordination to a similar hash-based scheme will be tracked via a separate blueprint.","commit_id":"c6d4f65d80f6b4c714185674d68b97971d2ea5f8"}]}