)]}'
{"doc/source/test_plans/massively_distribute_rpc/environment_description.rst":[{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"9f436f4f_3af0aa67","line":49,"updated":"2017-08-09 20:43:55.000000000","message":"Will need to record the version of oslo.messaging for the synthetic tests.\n\nIn addition the versions of the various messaging components and libraries for each backend should be captured.  The list would include the following:\n\nfor rabbit backends involving the rabbitmq broker:\nRabbitMQ server version\nKombu python package version\namqp python package version\n\nfor AMQP 1.0 backend:\nQpid Dispatch Router version\npython-qpid-proton package version\npyngus python package version\n\nZeroMQ backend:\npyzmq python package version\nredis-server version\n\nKafka backend:\nKafka server version\nkafka-python package version\nJava Runtime version","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"9f436f4f_603a29ce","line":49,"in_reply_to":"9f436f4f_3af0aa67","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"4a1857ee2476c4a241bf092d929144652d2c2951","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"This section contains list of interfaces and network parameters.  In the"},{"line_number":24,"context_line":"context of a cloud massively distributed (e.g accross a WAN), the network links"},{"line_number":25,"context_line":"may present different caracteristics in terms of latency, bandwidth, packet"},{"line_number":26,"context_line":"loss. These caracteristics can be emulated (e.g `tc`_) or be the result of a"},{"line_number":27,"context_line":"real deployment over a large geographical area.  In any cases, link"},{"line_number":28,"context_line":"caracteristics must be described."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_7532e7a9","line":25,"range":{"start_line":25,"start_character":22,"end_line":25,"end_character":36},"updated":"2017-10-18 15:10:09.000000000","message":"sp. \u0027Characteristics\u0027","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"6cf01627150afbb00fe134a39d1a73ab8deaf48f","unresolved":false,"context_lines":[{"line_number":22,"context_line":""},{"line_number":23,"context_line":"This section contains list of interfaces and network parameters.  In the"},{"line_number":24,"context_line":"context of a cloud massively distributed (e.g accross a WAN), the network links"},{"line_number":25,"context_line":"may present different caracteristics in terms of latency, bandwidth, packet"},{"line_number":26,"context_line":"loss. These caracteristics can be emulated (e.g `tc`_) or be the result of a"},{"line_number":27,"context_line":"real deployment over a large geographical area.  In any cases, link"},{"line_number":28,"context_line":"caracteristics must be described."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_50eed288","line":25,"range":{"start_line":25,"start_character":22,"end_line":25,"end_character":36},"in_reply_to":"3f4b6375_7532e7a9","updated":"2017-10-18 19:25:05.000000000","message":"Done","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":5950,"name":"Ilya Shakhat","email":"shakhat@gmail.com","username":"shakhat"},"change_message_id":"adb3579b214b39292dafab6e6e8dbb6d7d63bdaa","unresolved":false,"context_lines":[{"line_number":109,"context_line":"Openstack version"},{"line_number":110,"context_line":"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"For the operationnal testings, OpenStack version must be specified."}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_17bfd061","line":112,"range":{"start_line":112,"start_character":8,"end_line":112,"end_character":20},"updated":"2017-10-23 14:16:10.000000000","message":"-\u003e operational","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"3902a16d6a066630abfee7a28b6078e4137f9913","unresolved":false,"context_lines":[{"line_number":109,"context_line":"Openstack version"},{"line_number":110,"context_line":"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"For the operationnal testings, OpenStack version must be specified."}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_459b7365","line":112,"range":{"start_line":112,"start_character":8,"end_line":112,"end_character":20},"in_reply_to":"3f4b6375_17bfd061","updated":"2017-10-31 22:16:09.000000000","message":"Done","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"}],"doc/source/test_plans/massively_distribute_rpc/plan.rst":[{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Synthetic testing"},{"line_number":43,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Test Dimensions"},{"line_number":46,"context_line":"---------------"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"The dimensions are identified parameters that can influence the overall"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_4bc6026a","line":45,"updated":"2017-08-09 20:43:55.000000000","message":"Looking at this from a scale perspective there\u0027s a dimension not explicitly covered here: queue distribution.\n\nI\u0027m thinking of two test objectives:\n\n1) how \u0027large\u0027 can we scale a single queue, in terms of the # of clients.  [When I say \u0027single queue\u0027 I\u0027m actually talking about a single \u0027Target\u0027 shared cloud-wide].\n\n2) to what density of separate queues (different \u0027Targets\u0027) can the message bus scale.  By distributing these queues we can see the effects of locality.\n\nSo I think perhaps we should have two dedicated test scenarios, one for \u0027massive queue\u0027 and one for \u0027distributed queues\u0027?","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"4de461f610dc5b44046d168e14c78b745f2e0c85","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Synthetic testing"},{"line_number":43,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Test Dimensions"},{"line_number":46,"context_line":"---------------"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"The dimensions are identified parameters that can influence the overall"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_c871c1e0","line":45,"in_reply_to":"9f436f4f_4bc6026a","updated":"2017-08-10 14:11:23.000000000","message":"I think it\u0027s a very good direction to organize the document !\n\nI may be wrong but, it seems that we are trying to evaluate the following use cases (by pushing them at the bus limit)\nhttps://wiki.openstack.org/wiki/Oslo/Messaging#Use_Cases\n\n1. Invoke Method on One of Multiple Servers\n-\u003e one single target, big number of clients : 1 single large queue\n\n2. Invoke Method on a Specific Server\n-\u003e big number of differents target : multiple queues\n\n3. Invoke Method on all of Multiple Servers\n-\u003e fanout related which can be splitted as you suggested : \n  - one single fanout (1 client, increase the number of servers)\n  - increase the number of fanouts  (repeat start 1 client + n servers using the same Target)\n\n-- \nWhat do you mean by : \"By distributing these queues we can see the effects of locality.\" ?\n\nMoreover in 1. are we interested in the way the bus is balancing the message distribution ?","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"2d2c6608803797c9fbc3e2a93d3b9bd80dd55622","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Synthetic testing"},{"line_number":43,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Test Dimensions"},{"line_number":46,"context_line":"---------------"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"The dimensions are identified parameters that can influence the overall"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_e3fe3a37","line":45,"in_reply_to":"9f436f4f_c871c1e0","updated":"2017-08-17 16:15:50.000000000","message":"I think we are trying to evaluate use cases #1 and #3 from that wiki - one single massively distributed queue.  That\u0027s exactly what I\u0027m referring to in my \"objective 1) massive queue\" above.\n\nI wasn\u0027t considering wiki use case #2 - the single server pattern - simply because   I think the single server will be the limiting factor rather than the message bus itself.\n\nBut there is an additional dimension of interest that isn\u0027t specifically called out by that wiki: scaling the # of different queues that are simultaneously active.  \n\nBy different queues I mean different targets, for example\n\nQueue001, Queue002, ... QueueNNN\n\nClients (both servers and callers) would be assigned to a particular queue and would only transfer messages across that queue.\n\nBy increasing the # of these (clients, queue) instances, we can gauge the ability of a message bus to scale the # of distinct queues that are active simultaneously.  That\u0027s what I\u0027m referring to as \"objective 2) distributed queues\".  But on second thought that really isn\u0027t the correct terminology.  Perhaps simply \u0027multiple queues\u0027 is clearer.\n\nLocality: ugh, my description doesn\u0027t really make sense, does it?  What I meant to say was if we distribute the servers and clients across the cloud rather than say concentrating the servers to one node we would enhance the locality effect.  But on second thought I don\u0027t think a \u0027locality effect\u0027 is something we should test for specifically.\n\nMessage distribution:  yes, capturing the total # of calls received per server would be an interesting data point for comparison.  But note that the wiki is a bit out of date - the API has relaxed the round-robin constraint to be \"best effort\".  Implementing a perfect balanced distribution had a negative impact on message throughput since it disables batching.","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Synthetic testing"},{"line_number":43,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Test Dimensions"},{"line_number":46,"context_line":"---------------"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"The dimensions are identified parameters that can influence the overall"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_60f7c9d6","line":45,"in_reply_to":"9f436f4f_e3fe3a37","updated":"2017-08-18 08:49:54.000000000","message":"I rephrase the test cases trying to take into account the above elements.","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"- Message payload size"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"This can be a fixed value. In case of dynamic values, the distribution of the"},{"line_number":54,"context_line":"values from which they are taken must be indicated."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"- Message generation rate (messages/sec)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_70c5333c","line":53,"updated":"2017-08-09 20:43:55.000000000","message":"Perhaps the easiest thing to do is integrate the sample message sizes used by oslo.messaging simulator into ombt2?\n\nWe\u0027d want to avoid any randomness if we do this.  In other words the sequence of payload sizes must be identical between separate test runs.","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"- Message payload size"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"This can be a fixed value. In case of dynamic values, the distribution of the"},{"line_number":54,"context_line":"values from which they are taken must be indicated."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"- Message generation rate (messages/sec)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_401de50d","line":53,"in_reply_to":"9f436f4f_70c5333c","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":55,"context_line":""},{"line_number":56,"context_line":"- Message generation rate (messages/sec)"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"This can be a fixed value. In case of dynamic values, the distribution of the"},{"line_number":59,"context_line":"values from which they are taken must be indicated."},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"- RPC Server capacity"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_b0cb9b51","line":58,"updated":"2017-08-09 20:43:55.000000000","message":"In general the message generation rate will dictate the # of producers to consumers (assuming consumers are bounded by CPU throughput).\n\nFor MDC I\u0027d favor many slow producers to few fast producers.","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"- RPC Server capacity"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"Number of servers per topic"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"- RPC Client load:"},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_3560bda5","line":63,"updated":"2017-08-09 20:43:55.000000000","message":"Instead of \u0027topic\u0027 we should refer to the oslo.messaging\u0027s API term for an address: \u0027Target\u0027.\n\nSorry should\u0027ve thought about that before!","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"- RPC Server capacity"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"Number of servers per topic"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"- RPC Client load:"},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_e0d17931","line":63,"in_reply_to":"9f436f4f_3560bda5","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":84,"context_line":"Test Case 1 : RPC Call performance metrics"},{"line_number":85,"context_line":"------------------------------------------"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Methodology"},{"line_number":88,"context_line":"^^^^^^^^^^^^"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":".. code-block:: none"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_704893cb","line":87,"updated":"2017-08-09 20:43:55.000000000","message":"This test is the \u0027large distributed single queue\u0027 test described above since it calls for a single topic (Target) used by all clients.\n\nWe should have a second test case that calls for a unique Target for every, say, 2 RPC servers to test queue density/locality.","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":84,"context_line":"Test Case 1 : RPC Call performance metrics"},{"line_number":85,"context_line":"------------------------------------------"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Methodology"},{"line_number":88,"context_line":"^^^^^^^^^^^^"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":".. code-block:: none"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_80b50dd0","line":87,"in_reply_to":"9f436f4f_704893cb","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":93,"context_line":"      * Provision a single RPC server on the default topic"},{"line_number":94,"context_line":"      * Provision a single RPC client"},{"line_number":95,"context_line":"      * RPC client issues calls to topic:"},{"line_number":96,"context_line":"        - full rate"},{"line_number":97,"context_line":"    Repeat:"},{"line_number":98,"context_line":"      * Add additional clients until RPC server CPU utilization reaches \u003e70%"},{"line_number":99,"context_line":"      * Provision another RPC server (default topic)"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_90009736","line":96,"updated":"2017-08-09 20:43:55.000000000","message":"We should avoid using 0 delay between RPC calls if it means fewer clients per server.  Perhaps a consistent delay between RPC calls should be used in order to increase the client/server ratio?","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":113,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":".. note:: The latency can correspond to different time :"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"    - client-side latency, the time taken in the execution flow of the sender to"},{"line_number":118,"context_line":"    make the RPC."},{"line_number":119,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_507d8faf","line":116,"updated":"2017-08-09 20:43:55.000000000","message":"For RPC calls we probably would only be interested in the round trip latency (t0\u003dRequest sent, t1\u003dReply received, latency \u003d t1 - t2).","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":113,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":".. note:: The latency can correspond to different time :"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"    - client-side latency, the time taken in the execution flow of the sender to"},{"line_number":118,"context_line":"    make the RPC."},{"line_number":119,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_c0bb15c2","line":116,"in_reply_to":"9f436f4f_507d8faf","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":121,"context_line":"    in the request."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Test Case 2 : RPC Cast performance metrics"},{"line_number":125,"context_line":"------------------------------------------"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"Same as above with RPC cast instead of RCP call"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_2ba17eaa","line":124,"updated":"2017-08-09 20:43:55.000000000","message":"Given time/resources, we could consider de-emphasizing this particular test since it will be accounted for in the RPC call test (Cast is call without reply).","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":121,"context_line":"    in the request."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Test Case 2 : RPC Cast performance metrics"},{"line_number":125,"context_line":"------------------------------------------"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"Same as above with RPC cast instead of RCP call"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_e0b619dd","line":124,"in_reply_to":"9f436f4f_2ba17eaa","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":124,"context_line":"Test Case 2 : RPC Cast performance metrics"},{"line_number":125,"context_line":"------------------------------------------"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"Same as above with RPC cast instead of RCP call"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":130,"context_line":"Priority  Value       Measurement Units  Description"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_d0129fb1","line":127,"updated":"2017-08-09 20:43:55.000000000","message":"\u0027RPC\u0027 not \u0027RCP\u0027 :)","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":124,"context_line":"Test Case 2 : RPC Cast performance metrics"},{"line_number":125,"context_line":"------------------------------------------"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"Same as above with RPC cast instead of RCP call"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":130,"context_line":"Priority  Value       Measurement Units  Description"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_00a8fd74","line":127,"in_reply_to":"9f436f4f_d0129fb1","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":134,"context_line":"2         Latency     ms                 The latency in message processing"},{"line_number":135,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Test Case 3 : RPC fanout perfomance metrics"},{"line_number":138,"context_line":"-------------------------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"Methodology"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_ebcb76a2","line":137,"updated":"2017-08-09 20:43:55.000000000","message":"RPC fanout is interesting because it uses a different model (pub/sub) than a queue.\n\nWe can perform the same \u0027large single fanout\u0027 and \u0027many independent fanouts\u0027 as we do for queues.","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":134,"context_line":"2         Latency     ms                 The latency in message processing"},{"line_number":135,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Test Case 3 : RPC fanout perfomance metrics"},{"line_number":138,"context_line":"-------------------------------------------"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"Methodology"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_009a3d6d","line":137,"in_reply_to":"9f436f4f_ebcb76a2","updated":"2017-08-18 08:49:54.000000000","message":"Done","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"a8692d4ba21807a6ca20ff0d6a2686c6c8e72090","unresolved":false,"context_lines":[{"line_number":156,"context_line":""},{"line_number":157,"context_line":"In case of fanout cast, no ack are sent to the sender. The latency will be the"},{"line_number":158,"context_line":"time interval between the message is sent and the message is received by all the"},{"line_number":159,"context_line":"servers."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"Test Case 4 : Resilience"},{"line_number":162,"context_line":"------------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_cbbaf2dc","line":159,"updated":"2017-08-09 20:43:55.000000000","message":"To be specific (and this would require work to get ombt2 to support this):\n\n1) sender puts sequence # and current time in message\n2) message is sent\n3) at each server:\n   arrival time is taken, latency is computed as the difference between send and generated time.\n   server sends the timestamps and message id to the ombt2 test controller.\n4) the test controller can compute the overall propagation delay over the fanout using the arrival times reported by the servers.","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"7cce949aa833f7b97f979ce3f47b39ae0025ddd4","unresolved":false,"context_lines":[{"line_number":156,"context_line":""},{"line_number":157,"context_line":"In case of fanout cast, no ack are sent to the sender. The latency will be the"},{"line_number":158,"context_line":"time interval between the message is sent and the message is received by all the"},{"line_number":159,"context_line":"servers."},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"Test Case 4 : Resilience"},{"line_number":162,"context_line":"------------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9f436f4f_209d4166","line":159,"in_reply_to":"9f436f4f_cbbaf2dc","updated":"2017-08-18 08:49:54.000000000","message":"Sure I kept this in my mind :)","commit_id":"e7718b246e7b368bac0292f4f60babd2833f5101"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"af7b9208cbad0d0eb91545d349e7dcd68438091b","unresolved":false,"context_lines":[{"line_number":89,"context_line":"In this test case clients are sending requests to the same Target. Servers are"},{"line_number":90,"context_line":"serving those requests. The goal of this test case is to evaluate how large a"},{"line_number":91,"context_line":"single distributed queue can be in terms of number of clients/servers."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Methodology"},{"line_number":94,"context_line":"^^^^^^^^^^^^"},{"line_number":95,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_a50bd027","line":92,"updated":"2017-08-29 20:19:30.000000000","message":"Maybe add \"RPC servers and clients should be instantiated evenly across all nodes in the test environment\"  or some such text so it\u0027s clear the intent is to push the messaging clients out to the edge of the cloud?","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":89,"context_line":"In this test case clients are sending requests to the same Target. Servers are"},{"line_number":90,"context_line":"serving those requests. The goal of this test case is to evaluate how large a"},{"line_number":91,"context_line":"single distributed queue can be in terms of number of clients/servers."},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Methodology"},{"line_number":94,"context_line":"^^^^^^^^^^^^"},{"line_number":95,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_00b1fde1","line":92,"in_reply_to":"7f287b81_a50bd027","updated":"2017-10-06 20:19:48.000000000","message":"Done","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"af7b9208cbad0d0eb91545d349e7dcd68438091b","unresolved":false,"context_lines":[{"line_number":112,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":113,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":114,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":115,"context_line":"1         Throughput          msg/sec            Number of messages per second"},{"line_number":116,"context_line":"2         Variance            msg/sec            Throughput variance over time"},{"line_number":117,"context_line":"2         Latency             ms                 The round-trip latency in message processing"},{"line_number":118,"context_line":"3         Number of messages  -                  The number of messages handled per server"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_2552403c","line":115,"updated":"2017-08-29 20:19:30.000000000","message":"Since this is round-trip RPC, we should define throughput as the number of calls per second (request/response exchanges) from the point of view of the RPC caller.","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":112,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":113,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":114,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":115,"context_line":"1         Throughput          msg/sec            Number of messages per second"},{"line_number":116,"context_line":"2         Variance            msg/sec            Throughput variance over time"},{"line_number":117,"context_line":"2         Latency             ms                 The round-trip latency in message processing"},{"line_number":118,"context_line":"3         Number of messages  -                  The number of messages handled per server"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_e0a2714b","line":115,"in_reply_to":"7f287b81_2552403c","updated":"2017-10-06 20:19:48.000000000","message":"Done","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"af7b9208cbad0d0eb91545d349e7dcd68438091b","unresolved":false,"context_lines":[{"line_number":120,"context_line":""},{"line_number":121,"context_line":".. note::"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"    This test case can be run for RPC call and RPC cast."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Test Case 2: Multiple distributed queues"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_7016c84b","line":123,"updated":"2017-08-29 20:19:30.000000000","message":"In the case of cast, throughput won\u0027t be accurate - sends are async so it will appear throughput is much higher than actual.\n\nLatency and # of messages of course still apply for casts.","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":120,"context_line":""},{"line_number":121,"context_line":".. note::"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"    This test case can be run for RPC call and RPC cast."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Test Case 2: Multiple distributed queues"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_608e81d1","line":123,"in_reply_to":"7f287b81_7016c84b","updated":"2017-10-06 20:19:48.000000000","message":"see my question on the next version","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"af7b9208cbad0d0eb91545d349e7dcd68438091b","unresolved":false,"context_lines":[{"line_number":143,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":144,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":145,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":146,"context_line":"1         Throughput          msg/sec            Number of messages per second"},{"line_number":147,"context_line":"2         Variance            msg/sec            Throughput variance over time"},{"line_number":148,"context_line":"2         Latency             ms                 The round-trip latency in message processing"},{"line_number":149,"context_line":"3         Number of messages  -                  The number of messages handled per server"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_f023d8a5","line":146,"updated":"2017-08-29 20:19:30.000000000","message":"Throughput - ditto round trip for RPC call, ignored for RPC cast.","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":143,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":144,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":145,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":146,"context_line":"1         Throughput          msg/sec            Number of messages per second"},{"line_number":147,"context_line":"2         Variance            msg/sec            Throughput variance over time"},{"line_number":148,"context_line":"2         Latency             ms                 The round-trip latency in message processing"},{"line_number":149,"context_line":"3         Number of messages  -                  The number of messages handled per server"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_a0ad8932","line":146,"in_reply_to":"7f287b81_f023d8a5","updated":"2017-10-06 20:19:48.000000000","message":"Done","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"af7b9208cbad0d0eb91545d349e7dcd68438091b","unresolved":false,"context_lines":[{"line_number":214,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":215,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":216,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":217,"context_line":"1         Throughput          msg/sec            Number of messages per second"},{"line_number":218,"context_line":"2         Variance            msg/sec            Throughput variance over time"},{"line_number":219,"context_line":"2         Latency             ms                 The latency of transferring a message"},{"line_number":220,"context_line":"3         Number of messages  -                  The number of messages handled per server"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_d06c5ccf","line":217,"updated":"2017-08-29 20:19:30.000000000","message":"Similar to casts I think throughput from the point of view of the client will be artificially low.  Probably should just track latency and message counts here as well.","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":214,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":215,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":216,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":217,"context_line":"1         Throughput          msg/sec            Number of messages per second"},{"line_number":218,"context_line":"2         Variance            msg/sec            Throughput variance over time"},{"line_number":219,"context_line":"2         Latency             ms                 The latency of transferring a message"},{"line_number":220,"context_line":"3         Number of messages  -                  The number of messages handled per server"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_bb493e33","line":217,"in_reply_to":"7f287b81_d06c5ccf","updated":"2017-10-06 20:19:48.000000000","message":"Done","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"334721872d52b2dd268a2b442f57972f9934c42e","unresolved":false,"context_lines":[{"line_number":230,"context_line":"observed. In a massively distributed case, communicating entities may fail more"},{"line_number":231,"context_line":"often (link down, hardware failure). The goal of this test case is to evaluate"},{"line_number":232,"context_line":"the resiliency of the messaging layer to failures."},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"Methodology"},{"line_number":235,"context_line":"^^^^^^^^^^^"},{"line_number":236,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_9f3ecebe","line":233,"updated":"2017-08-30 14:23:45.000000000","message":"This test will be based on RPC call traffic.  RPC cast traffic is sent \"least effort\" - cast messages are more likely to be dropped than calls since there is no return ACKs in the case of cast.","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":230,"context_line":"observed. In a massively distributed case, communicating entities may fail more"},{"line_number":231,"context_line":"often (link down, hardware failure). The goal of this test case is to evaluate"},{"line_number":232,"context_line":"the resiliency of the messaging layer to failures."},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"Methodology"},{"line_number":235,"context_line":"^^^^^^^^^^^"},{"line_number":236,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_bb201e4b","line":233,"in_reply_to":"7f287b81_9f3ecebe","updated":"2017-10-06 20:19:48.000000000","message":"done by adding this to the methodology section.","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"334721872d52b2dd268a2b442f57972f9934c42e","unresolved":false,"context_lines":[{"line_number":234,"context_line":"Methodology"},{"line_number":235,"context_line":"^^^^^^^^^^^"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"TODO"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"Metrics"},{"line_number":240,"context_line":"^^^^^^^"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_ba587830","line":237,"updated":"2017-08-30 14:23:45.000000000","message":"The messaging infrastructure must be configured in such a way as to ensure functionality can be preserved in the case of loss of any one messaging component.  E.g. three rabbit brokers in a cluster, two alternate paths across a router mesh, etc.\n\nEach messaging client must be configured with a fail-over address for re-connecting to the message bus should its primary connection fail (see the oslo.messaging documentation for `TransportURL` addresses)\n\nThe test environment is the same as that for Test Case 1 : One single large distributed queue, with the caveat that each process comprising the message bus maintains a steady state CPU load of approximately 50%.  In other words the test traffic should maintain a reasonable and consistent load on the message bus without overloading it.\n\n.. code-block:: none\n\n    Start:\n      * Provision the test environment as described above\n    Phase 1:\n      * reboot one component of the message bus (e.g. a single rabbit broker in the cluster)\n      * wait until the component recovers and the message bus returns to steady state\n    Repeat:\n      * Phase 1 for each component of the message bus\n    Phase 2:\n      * force the failure of one of the TCP connections linking two components of the message bus (e.g. the connection between two rabbit brokers in a cluster).\n      * wait 60 seconds\n      * restore the connection\n      * wait until message bus stabilizes\n    Repeat:\n      * Phase 2 for each TCP connection connecting any two message bus components.","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":234,"context_line":"Methodology"},{"line_number":235,"context_line":"^^^^^^^^^^^"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"TODO"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"Metrics"},{"line_number":240,"context_line":"^^^^^^^"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_5b17faee","line":237,"in_reply_to":"7f287b81_ba587830","updated":"2017-10-06 20:19:48.000000000","message":"Done","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"334721872d52b2dd268a2b442f57972f9934c42e","unresolved":false,"context_lines":[{"line_number":239,"context_line":"Metrics"},{"line_number":240,"context_line":"^^^^^^^"},{"line_number":241,"context_line":""},{"line_number":242,"context_line":"TODO"},{"line_number":243,"context_line":""},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"Common metrics to all test cases"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_3a8b6868","line":242,"updated":"2017-08-30 14:23:45.000000000","message":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\nPriority  Value               Measurement Units  Description\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n1         Call failures       -                 Total number of RPC call operations that failed\n2         Reboot recovery     seconds           The average time between a component reboot and recovery of the message bus\n2         Reconnect recovery  seconds           The average time between the restoration of an internal TCP connection and the recovery of the message bus\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":239,"context_line":"Metrics"},{"line_number":240,"context_line":"^^^^^^^"},{"line_number":241,"context_line":""},{"line_number":242,"context_line":"TODO"},{"line_number":243,"context_line":""},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"Common metrics to all test cases"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_7b1ab617","line":242,"in_reply_to":"7f287b81_3a8b6868","updated":"2017-10-06 20:19:48.000000000","message":"Done","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"334721872d52b2dd268a2b442f57972f9934c42e","unresolved":false,"context_lines":[{"line_number":314,"context_line":".. _Rally: https://github.com/openstack/rally"},{"line_number":315,"context_line":".. _Neutron scenarios: https://github.com/openstack/rally/tree/0aa2f58e43ff143a27880b5fa527b09c7670d5f7/samples/tasks/scenarios/neutron"},{"line_number":316,"context_line":".. _Nova scenarios: https://github.com/openstack/rally/tree/0aa2f58e43ff143a27880b5fa527b09c7670d5f7/samples/tasks/scenarios/nova"},{"line_number":317,"context_line":""},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Reports"},{"line_number":320,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_ffd83253","line":317,"updated":"2017-08-30 14:23:45.000000000","message":".. _TransportURL: https://docs.openstack.org/oslo.messaging/latest/reference/transport.html","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":314,"context_line":".. _Rally: https://github.com/openstack/rally"},{"line_number":315,"context_line":".. _Neutron scenarios: https://github.com/openstack/rally/tree/0aa2f58e43ff143a27880b5fa527b09c7670d5f7/samples/tasks/scenarios/neutron"},{"line_number":316,"context_line":".. _Nova scenarios: https://github.com/openstack/rally/tree/0aa2f58e43ff143a27880b5fa527b09c7670d5f7/samples/tasks/scenarios/nova"},{"line_number":317,"context_line":""},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Reports"},{"line_number":320,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f287b81_9b1d6211","line":317,"in_reply_to":"7f287b81_ffd83253","updated":"2017-10-06 20:19:48.000000000","message":"Done","commit_id":"b734e68cb452cdd0f189c2692212647defd03dd0"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"894ff675f8fa7bc9eb85534f4035aafd2f698c1b","unresolved":false,"context_lines":[{"line_number":129,"context_line":""},{"line_number":130,"context_line":"    This test case can be run for RPC call and RPC cast."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"    Note that throughput is less interesting in the case of cast"},{"line_number":133,"context_line":"    messages since it can be artificially high due to the lack of ack."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f287b81_96f417d1","line":132,"updated":"2017-09-04 10:58:01.000000000","message":"In the case of brokerless approach, I wonder  if the throughput will self-stabilize around a value that is dictated by the credit allowed to each publisher by the routing mesh (from   https://review.openstack.org/#/c/314603)","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"11b0e290f4606cd63e6d1dcc1bb04a82eaa621ac","unresolved":false,"context_lines":[{"line_number":129,"context_line":""},{"line_number":130,"context_line":"    This test case can be run for RPC call and RPC cast."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"    Note that throughput is less interesting in the case of cast"},{"line_number":133,"context_line":"    messages since it can be artificially high due to the lack of ack."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_684bd0af","line":132,"in_reply_to":"7f287b81_96f417d1","updated":"2017-09-26 15:06:32.000000000","message":"Yes at some point it will stabilize: the routers limit the amount of internal message buffering to prevent memory exhaustion, specifically in the case of a fast producer + slow consumer.  This will result in back pressure of the producer.\n\nThat note is vague.  What I meant to say is that we can\u0027t determine the duration of the test from just the producer\u0027s point of view when calculating throughput, since messages will be in flight when the producer finishes sending the last cast.\n\nFor a more accurate non-fanout cast throughput ombt2 will need to compute the test duration using the time the client sends the first cast to the arrival time of the last cast to the server.\n\nI could be over thinking this :) ...","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":129,"context_line":""},{"line_number":130,"context_line":"    This test case can be run for RPC call and RPC cast."},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"    Note that throughput is less interesting in the case of cast"},{"line_number":133,"context_line":"    messages since it can be artificially high due to the lack of ack."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_cfb1a3e3","line":132,"in_reply_to":"7f515b1d_684bd0af","updated":"2017-10-06 20:19:48.000000000","message":"Let\u0027s keep the note as it was.\nI agree with you about the throughput calculation, if we can have it this would be great but it\u0027s probably not the highest priority.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"894ff675f8fa7bc9eb85534f4035aafd2f698c1b","unresolved":false,"context_lines":[{"line_number":298,"context_line":"      * Provision the test environment as described above"},{"line_number":299,"context_line":"    Phase 1:"},{"line_number":300,"context_line":"      * reboot one component of the message bus (e.g. a single rabbit broker in the cluster)"},{"line_number":301,"context_line":"      * wait until the component recovers and the message bus returns to steady state"},{"line_number":302,"context_line":"    Repeat:"},{"line_number":303,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":304,"context_line":"    Phase 2:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f287b81_f6a91b90","line":301,"updated":"2017-09-04 10:58:01.000000000","message":"It\u0027s unclear to me how we can evaluate the time to come back to a steady state.\nMaybe we could wait a fixed amount of time (reasonnably not to high, not to low neither).","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"11b0e290f4606cd63e6d1dcc1bb04a82eaa621ac","unresolved":false,"context_lines":[{"line_number":298,"context_line":"      * Provision the test environment as described above"},{"line_number":299,"context_line":"    Phase 1:"},{"line_number":300,"context_line":"      * reboot one component of the message bus (e.g. a single rabbit broker in the cluster)"},{"line_number":301,"context_line":"      * wait until the component recovers and the message bus returns to steady state"},{"line_number":302,"context_line":"    Repeat:"},{"line_number":303,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":304,"context_line":"    Phase 2:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_c3c6259b","line":301,"in_reply_to":"7f287b81_f6a91b90","updated":"2017-09-26 15:06:32.000000000","message":"Yeah, the term \"steady state\" is a bit vague here.\n\nIn the concrete case for qdrouterd, I would define \u0027return to steady state\u0027 to be the point were the rebooted router shows up in the route table of at least one of the neighboring routers.  The \u0027qdstat -b \u003cip addr\u003e -n\u0027 command lists all known routers from the point of view of the router at \u003cip addr\u003e.  Or perhaps when all tcp connections are re-established?\n\nFor rabbit maybe there\u0027s a way we can determine when the rebooted broker has re-joined the cluster?\n\nJust some ideas.  In the end I think we\u0027ll have to document how we consider a component to be \u0027recovered\u0027 for each type of message bus and be consistent when testing that type of bus.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":298,"context_line":"      * Provision the test environment as described above"},{"line_number":299,"context_line":"    Phase 1:"},{"line_number":300,"context_line":"      * reboot one component of the message bus (e.g. a single rabbit broker in the cluster)"},{"line_number":301,"context_line":"      * wait until the component recovers and the message bus returns to steady state"},{"line_number":302,"context_line":"    Repeat:"},{"line_number":303,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":304,"context_line":"    Phase 2:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_f4331612","line":301,"in_reply_to":"7f515b1d_c3c6259b","updated":"2017-10-06 20:19:48.000000000","message":"I\u0027ve added a note about this.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"894ff675f8fa7bc9eb85534f4035aafd2f698c1b","unresolved":false,"context_lines":[{"line_number":302,"context_line":"    Repeat:"},{"line_number":303,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":304,"context_line":"    Phase 2:"},{"line_number":305,"context_line":"      * force the failure of one of the TCP connections linking two components of the message bus (e.g. the connection between two rabbit brokers in a cluster)."},{"line_number":306,"context_line":"      * wait 60 seconds"},{"line_number":307,"context_line":"      * restore the connection"},{"line_number":308,"context_line":"      * wait until message bus stabilizes"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f287b81_d6e05ffd","line":305,"range":{"start_line":305,"start_character":0,"end_line":305,"end_character":160},"updated":"2017-09-04 10:58:01.000000000","message":"I guess it incudes also the links between the clients and the broker/router agents.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"11b0e290f4606cd63e6d1dcc1bb04a82eaa621ac","unresolved":false,"context_lines":[{"line_number":302,"context_line":"    Repeat:"},{"line_number":303,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":304,"context_line":"    Phase 2:"},{"line_number":305,"context_line":"      * force the failure of one of the TCP connections linking two components of the message bus (e.g. the connection between two rabbit brokers in a cluster)."},{"line_number":306,"context_line":"      * wait 60 seconds"},{"line_number":307,"context_line":"      * restore the connection"},{"line_number":308,"context_line":"      * wait until message bus stabilizes"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_03b79df8","line":305,"range":{"start_line":305,"start_character":0,"end_line":305,"end_character":160},"in_reply_to":"7f287b81_d6e05ffd","updated":"2017-09-26 15:06:32.000000000","message":"I could - honestly I was only considering the internal message bus connections.\n\nIn the case of client connections the reconnect logic is handled by the oslo.messaging driver itself, not the message bus.  So I had assumed that wasn\u0027t strictly a message bus test, but it would be useful to gauge what happens when a client moves from one component of the message bus to another (which is how oslo.messaging handles reconnect)","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":302,"context_line":"    Repeat:"},{"line_number":303,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":304,"context_line":"    Phase 2:"},{"line_number":305,"context_line":"      * force the failure of one of the TCP connections linking two components of the message bus (e.g. the connection between two rabbit brokers in a cluster)."},{"line_number":306,"context_line":"      * wait 60 seconds"},{"line_number":307,"context_line":"      * restore the connection"},{"line_number":308,"context_line":"      * wait until message bus stabilizes"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_6f0d0f34","line":305,"range":{"start_line":305,"start_character":0,"end_line":305,"end_character":160},"in_reply_to":"7f515b1d_03b79df8","updated":"2017-10-06 20:19:48.000000000","message":"I\u0027ve added a third phase in the test to test the fail-over mecanism of oslo.messaging. It\u0027s probably low priority compared to phase 1 \u0026 2.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"894ff675f8fa7bc9eb85534f4035aafd2f698c1b","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Metrics"},{"line_number":313,"context_line":"^^^^^^^"},{"line_number":314,"context_line":""},{"line_number":315,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":316,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":317,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f287b81_56446fb7","line":314,"updated":"2017-09-04 10:58:01.000000000","message":"From the oslo.messaging perspective, I guess failures may introduce some requeues on the sending part and duplicate arrivals from the receiver part. I saw that the latter is transparently handled by oslo.messaging, at least under \u0027normal\u0027 operations (https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/_drivers/amqp.py#L100). If we are more aggressive maybe we can start to see duplicate messages coming at the application level. My feeling is that we could track all the message id sent and received to evaluate this.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"11b0e290f4606cd63e6d1dcc1bb04a82eaa621ac","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Metrics"},{"line_number":313,"context_line":"^^^^^^^"},{"line_number":314,"context_line":""},{"line_number":315,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":316,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":317,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_63a89176","line":314,"in_reply_to":"7f287b81_56446fb7","updated":"2017-09-26 15:06:32.000000000","message":"That duplicate detection logic only works in the case of a single consumer for a given address.  If there are multiple consumers for the address a re-transmitted message may arrive at a consumer that did not receive the first transmit. So the duplication is not detected.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Metrics"},{"line_number":313,"context_line":"^^^^^^^"},{"line_number":314,"context_line":""},{"line_number":315,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":316,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":317,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_8fae0be5","line":314,"in_reply_to":"7f515b1d_63a89176","updated":"2017-10-06 20:19:48.000000000","message":"I\u0027d love to evaluate the message duplications :).\n- not related to this doc but - do you know if they are some guidelines for the project team developpers to handle messages in a idempotent way ? (and thus handle messages duplication gracefully)","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"86e1297aeafd2d4e66aaf84eacbf3716cc77ddb0","unresolved":false,"context_lines":[{"line_number":311,"context_line":""},{"line_number":312,"context_line":"Metrics"},{"line_number":313,"context_line":"^^^^^^^"},{"line_number":314,"context_line":""},{"line_number":315,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":316,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":317,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_70f7075b","line":314,"in_reply_to":"7f515b1d_8fae0be5","updated":"2017-10-09 13:45:46.000000000","message":"Unfortunately not from an oslo.messaging\u0027s user\u0027s point of view since o.m. doesn\u0027t track the state of the application.\n\nHandling duplication of non-idempotent events is left up to the application, esp. if some sort of coordination is required to manage state across a distributed application.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"894ff675f8fa7bc9eb85534f4035aafd2f698c1b","unresolved":false,"context_lines":[{"line_number":315,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":316,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":317,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":318,"context_line":"1         Call failures       -                  Total number of RPC call"},{"line_number":319,"context_line":"                                                 operations that failed"},{"line_number":320,"context_line":"2         Reboot recovery     seconds            The average time between a"},{"line_number":321,"context_line":"                                                 component reboot and recovery"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f287b81_3678c3f6","line":318,"updated":"2017-09-04 10:58:01.000000000","message":"Is there a way to differentiate between the failure causes ? \n(unreachable server, max retries ...)","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"11b0e290f4606cd63e6d1dcc1bb04a82eaa621ac","unresolved":false,"context_lines":[{"line_number":315,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":316,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":317,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":318,"context_line":"1         Call failures       -                  Total number of RPC call"},{"line_number":319,"context_line":"                                                 operations that failed"},{"line_number":320,"context_line":"2         Reboot recovery     seconds            The average time between a"},{"line_number":321,"context_line":"                                                 component reboot and recovery"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_4315959b","line":318,"in_reply_to":"7f287b81_3678c3f6","updated":"2017-09-26 15:06:32.000000000","message":"There are different types of exceptions raised, and each exception contains an error text.  We could have ombt2 count each one separately.","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"e5fa86c188988f626212d4e2915262292c70db10","unresolved":false,"context_lines":[{"line_number":315,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":316,"context_line":"Priority  Value               Measurement Units  Description"},{"line_number":317,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":318,"context_line":"1         Call failures       -                  Total number of RPC call"},{"line_number":319,"context_line":"                                                 operations that failed"},{"line_number":320,"context_line":"2         Reboot recovery     seconds            The average time between a"},{"line_number":321,"context_line":"                                                 component reboot and recovery"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_6f92afa1","line":318,"in_reply_to":"7f515b1d_4315959b","updated":"2017-10-06 20:19:48.000000000","message":"This would be great !","commit_id":"9a89e3da34349f923af0017e8c3833977952ffd8"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"4a1857ee2476c4a241bf092d929144652d2c2951","unresolved":false,"context_lines":[{"line_number":14,"context_line":"  massively distributed cloud addresses the use case of the Fog or Edge"},{"line_number":15,"context_line":"  computing where the services are geographically distributed accross a large"},{"line_number":16,"context_line":"  area. For the time being, OpenStack inter-service communications rely on"},{"line_number":17,"context_line":"  oslo_messaging which defines a common abstraction to different instanciations"},{"line_number":18,"context_line":"  of the message bus. Historically broker-based implementations (e.g RabbitMQ,"},{"line_number":19,"context_line":"  QPid) competed with brokerless based implementations (e.g ZeroMQ), but with"},{"line_number":20,"context_line":"  the advent of AMQP1.0 in oslo_messaging, `alternative non-broker`_ messaging"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_b57cdfb5","line":17,"range":{"start_line":17,"start_character":65,"end_line":17,"end_character":79},"updated":"2017-10-18 15:10:09.000000000","message":"sp. \u0027instantiations\u0027","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"6cf01627150afbb00fe134a39d1a73ab8deaf48f","unresolved":false,"context_lines":[{"line_number":14,"context_line":"  massively distributed cloud addresses the use case of the Fog or Edge"},{"line_number":15,"context_line":"  computing where the services are geographically distributed accross a large"},{"line_number":16,"context_line":"  area. For the time being, OpenStack inter-service communications rely on"},{"line_number":17,"context_line":"  oslo_messaging which defines a common abstraction to different instanciations"},{"line_number":18,"context_line":"  of the message bus. Historically broker-based implementations (e.g RabbitMQ,"},{"line_number":19,"context_line":"  QPid) competed with brokerless based implementations (e.g ZeroMQ), but with"},{"line_number":20,"context_line":"  the advent of AMQP1.0 in oslo_messaging, `alternative non-broker`_ messaging"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_90bc4abc","line":17,"range":{"start_line":17,"start_character":65,"end_line":17,"end_character":79},"in_reply_to":"3f4b6375_b57cdfb5","updated":"2017-10-18 19:25:05.000000000","message":"Done","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"4a1857ee2476c4a241bf092d929144652d2c2951","unresolved":false,"context_lines":[{"line_number":24,"context_line":"  The test plan takes place in the context of a prospective effort to evaluate"},{"line_number":25,"context_line":"  the distribution of the messaging bus using emerging solutions (e.g qpid"},{"line_number":26,"context_line":"  dispatch router) or established ones (e.g Zero-MQ) compared to the"},{"line_number":27,"context_line":"  traditionnal and centralized solutions (e.g RabbitMQ). Finally the scope of"},{"line_number":28,"context_line":"  the test plan is RPC communication between OpenStack services, thus"},{"line_number":29,"context_line":"  notification is out of the scope of range."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_353fcfd7","line":27,"range":{"start_line":27,"start_character":2,"end_line":27,"end_character":14},"updated":"2017-10-18 15:10:09.000000000","message":"sp. \u0027traditional\u0027","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"6cf01627150afbb00fe134a39d1a73ab8deaf48f","unresolved":false,"context_lines":[{"line_number":24,"context_line":"  The test plan takes place in the context of a prospective effort to evaluate"},{"line_number":25,"context_line":"  the distribution of the messaging bus using emerging solutions (e.g qpid"},{"line_number":26,"context_line":"  dispatch router) or established ones (e.g Zero-MQ) compared to the"},{"line_number":27,"context_line":"  traditionnal and centralized solutions (e.g RabbitMQ). Finally the scope of"},{"line_number":28,"context_line":"  the test plan is RPC communication between OpenStack services, thus"},{"line_number":29,"context_line":"  notification is out of the scope of range."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_50b6d2db","line":27,"range":{"start_line":27,"start_character":2,"end_line":27,"end_character":14},"in_reply_to":"3f4b6375_353fcfd7","updated":"2017-10-18 19:25:05.000000000","message":"Done","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"4a1857ee2476c4a241bf092d929144652d2c2951","unresolved":false,"context_lines":[{"line_number":76,"context_line":"      * Provision another RPC server on T"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"List of performance metrics"},{"line_number":80,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"The following metrics are recorded for each repetition."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_f09ee552","line":79,"updated":"2017-10-18 15:10:09.000000000","message":"Now that ombt2 measures latency and throughput on both the RPC client and the RPC server perhaps we should be explicit as to which \u0027end\u0027 of the RPC transaction a particular metric should be captured?\n\nSpecifically, in the case of RPC *call* tests, throughput and latency should be measured from the RPC client (the caller).  For cast tests the latency and throughput should be measured from the RPC server (since the client does not block for ack).","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"6cf01627150afbb00fe134a39d1a73ab8deaf48f","unresolved":false,"context_lines":[{"line_number":76,"context_line":"      * Provision another RPC server on T"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"List of performance metrics"},{"line_number":80,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"The following metrics are recorded for each repetition."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_10697a24","line":79,"in_reply_to":"3f4b6375_f09ee552","updated":"2017-10-18 19:25:05.000000000","message":"Done","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"4a1857ee2476c4a241bf092d929144652d2c2951","unresolved":false,"context_lines":[{"line_number":100,"context_line":""},{"line_number":101,"context_line":"    This test case can be run for RPC call and RPC cast."},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"    Note that in the case of cast messages, measuring the throughput on the"},{"line_number":104,"context_line":"    caller side isn\u0027t accurate and can result in artificially high values."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_70e1d5d0","line":103,"updated":"2017-10-18 15:10:09.000000000","message":"This note can be removed if the above description of measurements is added.","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"6cf01627150afbb00fe134a39d1a73ab8deaf48f","unresolved":false,"context_lines":[{"line_number":100,"context_line":""},{"line_number":101,"context_line":"    This test case can be run for RPC call and RPC cast."},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"    Note that in the case of cast messages, measuring the throughput on the"},{"line_number":104,"context_line":"    caller side isn\u0027t accurate and can result in artificially high values."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_f075befb","line":103,"in_reply_to":"3f4b6375_70e1d5d0","updated":"2017-10-18 19:25:05.000000000","message":"Done","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"4a1857ee2476c4a241bf092d929144652d2c2951","unresolved":false,"context_lines":[{"line_number":271,"context_line":"distributed queue, with the caveat that each process comprising the message bus"},{"line_number":272,"context_line":"maintains a steady state CPU load of approximately 50%.  In other words the"},{"line_number":273,"context_line":"test traffic should maintain a reasonable and consistent load on the message"},{"line_number":274,"context_line":"bus without overloading it. Additionnaly test will be based on RPC call"},{"line_number":275,"context_line":"traffic.  RPC cast traffic is sent \"least effort\" - cast messages are more"},{"line_number":276,"context_line":"likely to be dropped than calls since there is no return ACKs in the case of"},{"line_number":277,"context_line":"cast."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_10274115","line":274,"range":{"start_line":274,"start_character":28,"end_line":274,"end_character":40},"updated":"2017-10-18 15:10:09.000000000","message":"sp. \u0027additionally\u0027","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"6cf01627150afbb00fe134a39d1a73ab8deaf48f","unresolved":false,"context_lines":[{"line_number":271,"context_line":"distributed queue, with the caveat that each process comprising the message bus"},{"line_number":272,"context_line":"maintains a steady state CPU load of approximately 50%.  In other words the"},{"line_number":273,"context_line":"test traffic should maintain a reasonable and consistent load on the message"},{"line_number":274,"context_line":"bus without overloading it. Additionnaly test will be based on RPC call"},{"line_number":275,"context_line":"traffic.  RPC cast traffic is sent \"least effort\" - cast messages are more"},{"line_number":276,"context_line":"likely to be dropped than calls since there is no return ACKs in the case of"},{"line_number":277,"context_line":"cast."}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_90668a51","line":274,"range":{"start_line":274,"start_character":28,"end_line":274,"end_character":40},"in_reply_to":"3f4b6375_10274115","updated":"2017-10-18 19:25:05.000000000","message":"Done","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":8770,"name":"Ken Giusti","email":"kgiusti@gmail.com","username":"kgiusti"},"change_message_id":"4a1857ee2476c4a241bf092d929144652d2c2951","unresolved":false,"context_lines":[{"line_number":343,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":344,"context_line":""},{"line_number":345,"context_line":""},{"line_number":346,"context_line":"Test Case 6 : Operationnal testing"},{"line_number":347,"context_line":"----------------------------------"},{"line_number":348,"context_line":""},{"line_number":349,"context_line":"Description"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_707d55e6","line":346,"range":{"start_line":346,"start_character":14,"end_line":346,"end_character":26},"updated":"2017-10-18 15:10:09.000000000","message":"sp. \u0027operational\u0027","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"6cf01627150afbb00fe134a39d1a73ab8deaf48f","unresolved":false,"context_lines":[{"line_number":343,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":344,"context_line":""},{"line_number":345,"context_line":""},{"line_number":346,"context_line":"Test Case 6 : Operationnal testing"},{"line_number":347,"context_line":"----------------------------------"},{"line_number":348,"context_line":""},{"line_number":349,"context_line":"Description"}],"source_content_type":"text/x-rst","patch_set":6,"id":"3f4b6375_705b0e87","line":346,"range":{"start_line":346,"start_character":14,"end_line":346,"end_character":26},"in_reply_to":"3f4b6375_707d55e6","updated":"2017-10-18 19:25:05.000000000","message":"Done","commit_id":"366fd4c6157491b1ff3bbd184e4fe3088a979e72"},{"author":{"_account_id":20523,"name":"Andy Smith","email":"ansmith@redhat.com","username":"ajssmith"},"change_message_id":"76b126ef1388318d4be20bb929d4b6ed16475df7","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":".. include:: environment_description.rst"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Test Case 1 : One single large distributed queue"},{"line_number":55,"context_line":"------------------------------------------------"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Description"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_8a6d96cd","line":54,"updated":"2017-10-25 14:00:59.000000000","message":"Should we consider replacing \u0027queue\u0027 with \u0027target\u0027\n\ne.g. dissociate the queue for direct messaging backends","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"3902a16d6a066630abfee7a28b6078e4137f9913","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":".. include:: environment_description.rst"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"Test Case 1 : One single large distributed queue"},{"line_number":55,"context_line":"------------------------------------------------"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Description"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_25d077f7","line":54,"in_reply_to":"3f4b6375_8a6d96cd","updated":"2017-10-31 22:16:09.000000000","message":"I also think that target is more appropriate.","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":20523,"name":"Andy Smith","email":"ansmith@redhat.com","username":"ajssmith"},"change_message_id":"76b126ef1388318d4be20bb929d4b6ed16475df7","unresolved":false,"context_lines":[{"line_number":108,"context_line":"      dividing the total number of messages by the time interval between the"},{"line_number":109,"context_line":"      first message sent by a client and the last message received by a server."},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"Test Case 2: Multiple distributed queues"},{"line_number":112,"context_line":"----------------------------------------"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"Description"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_ca85ce5b","line":111,"updated":"2017-10-25 14:00:59.000000000","message":"see above","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"3902a16d6a066630abfee7a28b6078e4137f9913","unresolved":false,"context_lines":[{"line_number":108,"context_line":"      dividing the total number of messages by the time interval between the"},{"line_number":109,"context_line":"      first message sent by a client and the last message received by a server."},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"Test Case 2: Multiple distributed queues"},{"line_number":112,"context_line":"----------------------------------------"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"Description"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_05cb3b4a","line":111,"in_reply_to":"3f4b6375_ca85ce5b","updated":"2017-10-31 22:16:09.000000000","message":"Done","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":5950,"name":"Ilya Shakhat","email":"shakhat@gmail.com","username":"shakhat"},"change_message_id":"adb3579b214b39292dafab6e6e8dbb6d7d63bdaa","unresolved":false,"context_lines":[{"line_number":291,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"    Phase 2:"},{"line_number":294,"context_line":"      * force the failure of one of the TCP connections linking two components of the message bus (e.g. the connection between two rabbit brokers in a cluster)."},{"line_number":295,"context_line":"      * wait 60 seconds"},{"line_number":296,"context_line":"      * restore the connection"},{"line_number":297,"context_line":"      * wait until message bus stabilizes"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_496ed095","line":294,"updated":"2017-10-23 14:16:10.000000000","message":"please wrap the long line; in the generated docs it is too long and requires horizontal scrolling","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"3902a16d6a066630abfee7a28b6078e4137f9913","unresolved":false,"context_lines":[{"line_number":291,"context_line":"      * Phase 1 for each component of the message bus"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"    Phase 2:"},{"line_number":294,"context_line":"      * force the failure of one of the TCP connections linking two components of the message bus (e.g. the connection between two rabbit brokers in a cluster)."},{"line_number":295,"context_line":"      * wait 60 seconds"},{"line_number":296,"context_line":"      * restore the connection"},{"line_number":297,"context_line":"      * wait until message bus stabilizes"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_25f9977f","line":294,"in_reply_to":"3f4b6375_496ed095","updated":"2017-10-31 22:16:09.000000000","message":"Done","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":5950,"name":"Ilya Shakhat","email":"shakhat@gmail.com","username":"shakhat"},"change_message_id":"adb3579b214b39292dafab6e6e8dbb6d7d63bdaa","unresolved":false,"context_lines":[{"line_number":300,"context_line":""},{"line_number":301,"context_line":"    Phase 3:"},{"line_number":302,"context_line":"      * force the failure of one of the TCP connections linking one client and its connected endpoint"},{"line_number":303,"context_line":"      * wait until client reconnect using one fail-over address"},{"line_number":304,"context_line":"      * restore the connection"},{"line_number":305,"context_line":"    Repeat:"},{"line_number":306,"context_line":"      * Phase 3 for each TCP connection connecting one client and the bus"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_a92b042e","line":303,"range":{"start_line":303,"start_character":26,"end_line":303,"end_character":35},"updated":"2017-10-23 14:16:10.000000000","message":"-\u003e reconnects","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"3902a16d6a066630abfee7a28b6078e4137f9913","unresolved":false,"context_lines":[{"line_number":300,"context_line":""},{"line_number":301,"context_line":"    Phase 3:"},{"line_number":302,"context_line":"      * force the failure of one of the TCP connections linking one client and its connected endpoint"},{"line_number":303,"context_line":"      * wait until client reconnect using one fail-over address"},{"line_number":304,"context_line":"      * restore the connection"},{"line_number":305,"context_line":"    Repeat:"},{"line_number":306,"context_line":"      * Phase 3 for each TCP connection connecting one client and the bus"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_45025371","line":303,"range":{"start_line":303,"start_character":26,"end_line":303,"end_character":35},"in_reply_to":"3f4b6375_a92b042e","updated":"2017-10-31 22:16:09.000000000","message":"Done","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":20523,"name":"Andy Smith","email":"ansmith@redhat.com","username":"ajssmith"},"change_message_id":"76b126ef1388318d4be20bb929d4b6ed16475df7","unresolved":false,"context_lines":[{"line_number":353,"context_line":"Description"},{"line_number":354,"context_line":"^^^^^^^^^^^"},{"line_number":355,"context_line":""},{"line_number":356,"context_line":"Operational testing intends to evaluate the correct behaviour of a running"},{"line_number":357,"context_line":"OpenStack on top of a specific deployment of the messaging middleware."},{"line_number":358,"context_line":"This test case aims to measure the correct behaviour of OpenStack"},{"line_number":359,"context_line":"under WAN at messaging plane level. It relies on `rally`_ that runs"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_e571db45","line":356,"updated":"2017-10-25 14:00:59.000000000","message":"This section needs more detail as the deployment of the messaging middleware could differ greatly. For example, a rabbit cluster should be connected via reliable LAN links implying it is centralized (e.g. the cluster does not operate across the WAN). Meanwhile, the dispatch-router can be naturally deployed across the WAN for distribution. \n\nShould we add a WAN test case scenario above? The WAN scenario introduces concepts of locality and concurrency in the messaging bus that should be measurable and will provide the basis for the operational testing for fog/edge scenarios.","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"3902a16d6a066630abfee7a28b6078e4137f9913","unresolved":false,"context_lines":[{"line_number":353,"context_line":"Description"},{"line_number":354,"context_line":"^^^^^^^^^^^"},{"line_number":355,"context_line":""},{"line_number":356,"context_line":"Operational testing intends to evaluate the correct behaviour of a running"},{"line_number":357,"context_line":"OpenStack on top of a specific deployment of the messaging middleware."},{"line_number":358,"context_line":"This test case aims to measure the correct behaviour of OpenStack"},{"line_number":359,"context_line":"under WAN at messaging plane level. It relies on `rally`_ that runs"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_258877c4","line":356,"in_reply_to":"3f4b6375_e571db45","updated":"2017-10-31 22:16:09.000000000","message":"Yes this section is probably too vague. \nI think the initial idea was to test a deployment of the message bus through test cases 1-5 and then map this deployment with an OpenStack running and see what happens.\nFor example, if we mimic cell based deployment (without cells) using a hierarchy of qpid-dispatch routers. we could test this hierarchy of routers and see the effects of locality through test1-5 (e.g having the messages processed by each servers we could check if the provenance was local). In a second step we could map this hierarchy of routers with an OpenStack deployment.\n\nI propose to keep this section  as it is and think together how to implement it (esp. what topology of routers/openstack services is relevant)","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":5950,"name":"Ilya Shakhat","email":"shakhat@gmail.com","username":"shakhat"},"change_message_id":"adb3579b214b39292dafab6e6e8dbb6d7d63bdaa","unresolved":false,"context_lines":[{"line_number":360,"context_line":"loads on the current OpenStack. Then Rally reports can be used to get"},{"line_number":361,"context_line":"time of operations executions and percent of failure to evaluate"},{"line_number":362,"context_line":"OpenStack. The chosen Rally scenarios and those known to be intensive on the"},{"line_number":363,"context_line":"messaging layer (e.g `Neutron scenarios`_ and `Nova_scenarios`_)."},{"line_number":364,"context_line":""},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"List of performance metrics"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_89ccc86a","line":363,"range":{"start_line":363,"start_character":47,"end_line":363,"end_character":61},"updated":"2017-10-23 14:16:10.000000000","message":"`Nova_scenarios` -\u003e `Nova scenarios` (no underscore, exact spelling as in references)","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"},{"author":{"_account_id":21470,"name":"Matthieu Simonin","email":"matthieu.simonin@inria.fr","username":"msimonin"},"change_message_id":"3902a16d6a066630abfee7a28b6078e4137f9913","unresolved":false,"context_lines":[{"line_number":360,"context_line":"loads on the current OpenStack. Then Rally reports can be used to get"},{"line_number":361,"context_line":"time of operations executions and percent of failure to evaluate"},{"line_number":362,"context_line":"OpenStack. The chosen Rally scenarios and those known to be intensive on the"},{"line_number":363,"context_line":"messaging layer (e.g `Neutron scenarios`_ and `Nova_scenarios`_)."},{"line_number":364,"context_line":""},{"line_number":365,"context_line":""},{"line_number":366,"context_line":"List of performance metrics"}],"source_content_type":"text/x-rst","patch_set":7,"id":"3f4b6375_65b78f88","line":363,"range":{"start_line":363,"start_character":47,"end_line":363,"end_character":61},"in_reply_to":"3f4b6375_89ccc86a","updated":"2017-10-31 22:16:09.000000000","message":"Done","commit_id":"3194c059674aa6f11f50f1f4764b06070aa3a5dc"}]}