)]}'
{"doc/arch-design/source/design-storage/design-storage-arch.rst":[{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":184,"context_line":"Performance"},{"line_number":185,"context_line":"  Performance of block based storage is typically measured in the maximum read"},{"line_number":186,"context_line":"  and write operations to non-contiguous storage locations per second. This"},{"line_number":187,"context_line":"  measurement typically applies to SAN, hard drives and solid state drives."},{"line_number":188,"context_line":"  While IOPS can be broadly measured and is not an official benchmark, many"},{"line_number":189,"context_line":"  vendors like to used by vendors to communicate performance levels. Since"},{"line_number":190,"context_line":"  there are no real standards for measuring IOPS, vendor test results may vary,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_1d7038a5","line":187,"range":{"start_line":187,"start_character":40,"end_line":187,"end_character":51},"updated":"2017-05-16 08:44:48.000000000","message":"hard drives, and *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":186,"context_line":"  and write operations to non-contiguous storage locations per second. This"},{"line_number":187,"context_line":"  measurement typically applies to SAN, hard drives and solid state drives."},{"line_number":188,"context_line":"  While IOPS can be broadly measured and is not an official benchmark, many"},{"line_number":189,"context_line":"  vendors like to used by vendors to communicate performance levels. Since"},{"line_number":190,"context_line":"  there are no real standards for measuring IOPS, vendor test results may vary,"},{"line_number":191,"context_line":"  sometimes wildly. Howver, along with transfer rate which measures the speed"},{"line_number":192,"context_line":"  that data can be transferred contiguous storage locations, IOPS can be used"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_dd69b0be","line":189,"range":{"start_line":189,"start_character":10,"end_line":189,"end_character":25},"updated":"2017-05-16 08:44:48.000000000","message":"like to be used by *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":188,"context_line":"  While IOPS can be broadly measured and is not an official benchmark, many"},{"line_number":189,"context_line":"  vendors like to used by vendors to communicate performance levels. Since"},{"line_number":190,"context_line":"  there are no real standards for measuring IOPS, vendor test results may vary,"},{"line_number":191,"context_line":"  sometimes wildly. Howver, along with transfer rate which measures the speed"},{"line_number":192,"context_line":"  that data can be transferred contiguous storage locations, IOPS can be used"},{"line_number":193,"context_line":"  in a performance evaluation. Typically transfer rate is represented by a"},{"line_number":194,"context_line":"  bytes per second calculation but IOPS is measured by an integer."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_9d6328dd","line":191,"range":{"start_line":191,"start_character":20,"end_line":191,"end_character":26},"updated":"2017-05-16 08:44:48.000000000","message":"However *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":190,"context_line":"  there are no real standards for measuring IOPS, vendor test results may vary,"},{"line_number":191,"context_line":"  sometimes wildly. Howver, along with transfer rate which measures the speed"},{"line_number":192,"context_line":"  that data can be transferred contiguous storage locations, IOPS can be used"},{"line_number":193,"context_line":"  in a performance evaluation. Typically transfer rate is represented by a"},{"line_number":194,"context_line":"  bytes per second calculation but IOPS is measured by an integer."},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"To calculate IOPS for a single drive you could use:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_5d9b60c8","line":193,"range":{"start_line":193,"start_character":31,"end_line":193,"end_character":40},"updated":"2017-05-16 08:44:48.000000000","message":"Typically, transfer *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":201,"context_line":"  IOPS \u003d 1/(.00299 + .0047)"},{"line_number":202,"context_line":"  IOPS \u003d 130"},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"To calculate maximum IOPS for a disk array"},{"line_number":205,"context_line":"  Maximum Read IOPS:"},{"line_number":206,"context_line":"  In order to accurately calculate maximum read IOPS for a disk array simply"},{"line_number":207,"context_line":"  multiply the IOPS for each disk by the maximum read or write IOPS per disk."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_1da5d804","line":204,"range":{"start_line":204,"start_character":37,"end_line":204,"end_character":42},"updated":"2017-05-16 08:44:48.000000000","message":"array: *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":203,"context_line":""},{"line_number":204,"context_line":"To calculate maximum IOPS for a disk array"},{"line_number":205,"context_line":"  Maximum Read IOPS:"},{"line_number":206,"context_line":"  In order to accurately calculate maximum read IOPS for a disk array simply"},{"line_number":207,"context_line":"  multiply the IOPS for each disk by the maximum read or write IOPS per disk."},{"line_number":208,"context_line":"  maxReadIOPS \u003d nDisks * diskMaxIOPS"},{"line_number":209,"context_line":"  For example, 15 10K Spinning Disks would be measured the following way:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_7db5fc30","line":206,"range":{"start_line":206,"start_character":64,"end_line":206,"end_character":76},"updated":"2017-05-16 08:44:48.000000000","message":"array, multiply* (delete \u0027simply\u0027)","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":209,"context_line":"  For example, 15 10K Spinning Disks would be measured the following way:"},{"line_number":210,"context_line":"  maxReadIOPS \u003d 15 * 130 maxReadIOPS \u003d 1950"},{"line_number":211,"context_line":""},{"line_number":212,"context_line":"Maximum Write IOPS per Array"},{"line_number":213,"context_line":"  Determining the maximum *write* IOPS is a little different because most"},{"line_number":214,"context_line":"  administrators configure disk replication using RAID and since the RAID"},{"line_number":215,"context_line":"  controller requires IOPS itself, there is a write penalty. The severity of"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_ddae50e6","line":212,"range":{"start_line":212,"start_character":23,"end_line":212,"end_character":28},"updated":"2017-05-16 08:44:48.000000000","message":"array: *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":209,"context_line":"  For example, 15 10K Spinning Disks would be measured the following way:"},{"line_number":210,"context_line":"  maxReadIOPS \u003d 15 * 130 maxReadIOPS \u003d 1950"},{"line_number":211,"context_line":""},{"line_number":212,"context_line":"Maximum Write IOPS per Array"},{"line_number":213,"context_line":"  Determining the maximum *write* IOPS is a little different because most"},{"line_number":214,"context_line":"  administrators configure disk replication using RAID and since the RAID"},{"line_number":215,"context_line":"  controller requires IOPS itself, there is a write penalty. The severity of"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_9db8c82a","line":212,"range":{"start_line":212,"start_character":8,"end_line":212,"end_character":13},"updated":"2017-05-16 08:44:48.000000000","message":"write*","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":217,"context_line":""},{"line_number":218,"context_line":"  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":219,"context_line":"  Raid Type   Penalty"},{"line_number":220,"context_line":"  ----------- ----------"},{"line_number":221,"context_line":"  1 2"},{"line_number":222,"context_line":"  5 4"},{"line_number":223,"context_line":"  10  2"},{"line_number":224,"context_line":"  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Note: Raid 5 has the worst penalty (has the most cross disk writes)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_1d7ef86a","line":223,"range":{"start_line":220,"start_character":0,"end_line":223,"end_character":7},"updated":"2017-05-16 08:44:48.000000000","message":"this the formatting concerns you had?","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":223,"context_line":"  10  2"},{"line_number":224,"context_line":"  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Note: Raid 5 has the worst penalty (has the most cross disk writes)"},{"line_number":227,"context_line":"  Therefore, using the above examples, a 15 disk array using RAID 5 is capable"},{"line_number":228,"context_line":"  of 1950 read IOPS however, we need to add the penalty when determining the"},{"line_number":229,"context_line":"  *write* IOPS:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_dd977090","line":226,"range":{"start_line":226,"start_character":0,"end_line":226,"end_character":5},"updated":"2017-05-16 08:44:48.000000000","message":".. note:: *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":223,"context_line":"  10  2"},{"line_number":224,"context_line":"  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Note: Raid 5 has the worst penalty (has the most cross disk writes)"},{"line_number":227,"context_line":"  Therefore, using the above examples, a 15 disk array using RAID 5 is capable"},{"line_number":228,"context_line":"  of 1950 read IOPS however, we need to add the penalty when determining the"},{"line_number":229,"context_line":"  *write* IOPS:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_9d91e89c","line":226,"range":{"start_line":226,"start_character":60,"end_line":226,"end_character":67},"updated":"2017-05-16 08:44:48.000000000","message":"writes). (missing a full stop here)","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":10607,"name":"Alexandra Settle","email":"a.settle@outlook.com","username":"asettle"},"change_message_id":"26afbbb255eef6b47b7bb5ac71e07f22b542e735","unresolved":false,"context_lines":[{"line_number":224,"context_line":"  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"Note: Raid 5 has the worst penalty (has the most cross disk writes)"},{"line_number":227,"context_line":"  Therefore, using the above examples, a 15 disk array using RAID 5 is capable"},{"line_number":228,"context_line":"  of 1950 read IOPS however, we need to add the penalty when determining the"},{"line_number":229,"context_line":"  *write* IOPS:"},{"line_number":230,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1f013ff3_7d8e1c79","line":227,"range":{"start_line":227,"start_character":13,"end_line":227,"end_character":18},"updated":"2017-05-16 08:44:48.000000000","message":"when using the *","commit_id":"923fb690dd89fc4eeed04c5b84f1d30c72e523ae"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":186,"context_line":"  and write operations to non-contiguous storage locations per second. This"},{"line_number":187,"context_line":"  measurement typically applies to SAN, hard drives, and solid state drives."},{"line_number":188,"context_line":"  While IOPS can be broadly measured and is not an official benchmark, many"},{"line_number":189,"context_line":"  vendors like to be used by vendors to communicate performance levels. Since"},{"line_number":190,"context_line":"  there are no real standards for measuring IOPS, vendor test results may vary,"},{"line_number":191,"context_line":"  sometimes wildly. However, along with transfer rate which measures the speed"},{"line_number":192,"context_line":"  that data can be transferred contiguous storage locations, IOPS can be used"}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_779022ee","line":189,"range":{"start_line":189,"start_character":2,"end_line":189,"end_character":9},"updated":"2017-06-02 05:19:22.000000000","message":"vectors","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":189,"context_line":"  vendors like to be used by vendors to communicate performance levels. Since"},{"line_number":190,"context_line":"  there are no real standards for measuring IOPS, vendor test results may vary,"},{"line_number":191,"context_line":"  sometimes wildly. However, along with transfer rate which measures the speed"},{"line_number":192,"context_line":"  that data can be transferred contiguous storage locations, IOPS can be used"},{"line_number":193,"context_line":"  in a performance evaluation. Typically, transfer rate is represented by a"},{"line_number":194,"context_line":"  bytes per second calculation but IOPS is measured by an integer."},{"line_number":195,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_d7ba4e6c","line":192,"range":{"start_line":192,"start_character":31,"end_line":192,"end_character":41},"updated":"2017-06-02 05:19:22.000000000","message":"to contiguous","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":196,"context_line":"To calculate IOPS for a single drive you could use:"},{"line_number":197,"context_line":"  IOPS \u003d 1 / (AverageLatency + AverageSeekTime)"},{"line_number":198,"context_line":"  For example:"},{"line_number":199,"context_line":"  Average Latency for Single Disko \u003d 2.99ms or .00299 seconds"},{"line_number":200,"context_line":"  Average Seek Time for Single Disk \u003d 4.7ms or .0047 seconds"},{"line_number":201,"context_line":"  IOPS \u003d 1/(.00299 + .0047)"},{"line_number":202,"context_line":"  IOPS \u003d 130"}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_97b0568c","line":199,"range":{"start_line":199,"start_character":29,"end_line":199,"end_character":34},"updated":"2017-06-02 05:19:22.000000000","message":"Disk","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":242,"context_line":"  In an HDD, data transfer is sequential. The actual read/write head \"seeks\" a"},{"line_number":243,"context_line":"  point in the hard drive to execute the operation. Seek time is significant."},{"line_number":244,"context_line":"  Transfer rate can also be influenced by file system fragmentation and the"},{"line_number":245,"context_line":"  layout. Finally, the mechanical nature of hard disks also hascertain"},{"line_number":246,"context_line":"  performance limitations."},{"line_number":247,"context_line":""},{"line_number":248,"context_line":"  In an SSD, data transfer is *not* sequential; it is random so it is faster."}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_777ee2af","line":245,"range":{"start_line":245,"start_character":60,"end_line":245,"end_character":70},"updated":"2017-06-02 05:19:22.000000000","message":"has certain","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":497,"context_line":"Replication"},{"line_number":498,"context_line":"-----------"},{"line_number":499,"context_line":""},{"line_number":500,"context_line":"Because each replica in Object Storage functions independently, and clients"},{"line_number":501,"context_line":"require only a majority of nodes responding to consider an operation"},{"line_number":502,"context_line":"successful as transient failures, like network partitions, can quickly cause"},{"line_number":503,"context_line":"replicas to diverge. These differences are eventually reconciled by"},{"line_number":504,"context_line":"asynchronous, peer-to-peer replicator processes. The replicator processes"},{"line_number":505,"context_line":"traverse their local filesystems, concurrently performing operations in a"},{"line_number":506,"context_line":"manner that balances load across physical disks."}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_3a784f8e","line":503,"range":{"start_line":500,"start_character":0,"end_line":503,"end_character":20},"updated":"2017-06-02 05:19:22.000000000","message":"Replicas in Object Storage function independently, and clients only require a majority of nodes to respond to a request in order for an operation to be considered successful. Thus, transient failures like network partitions can quickly cause replicas to diverge.","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":515,"context_line":"Every deleted record or file in the system is marked by a tombstone, so that"},{"line_number":516,"context_line":"deletions can be replicated alongside creations. The replication process cleans"},{"line_number":517,"context_line":"up tombstones after a time period known as the consistency window. The"},{"line_number":518,"context_line":"consistency window encompasses replication duration and how long transient"},{"line_number":519,"context_line":"failure can remove a node from the cluster. Tombstone cleanup must be tied to"},{"line_number":520,"context_line":"replication to reach replica convergence."},{"line_number":521,"context_line":""}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_3a860f68","line":518,"range":{"start_line":518,"start_character":56,"end_line":518,"end_character":64},"updated":"2017-06-02 05:19:22.000000000","message":"the length of time a","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":520,"context_line":"replication to reach replica convergence."},{"line_number":521,"context_line":""},{"line_number":522,"context_line":"If a replicator detects that a remote drive has failed, the replicator uses the"},{"line_number":523,"context_line":"``get_more_nodes`` interface for the ring to choose an alternate node with"},{"line_number":524,"context_line":"which to synchronize. The replicator can maintain desired levels of replication"},{"line_number":525,"context_line":"in the face of disk failures, though some replicas may not be in an immediately"},{"line_number":526,"context_line":"usable location."}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_9a963b93","line":523,"range":{"start_line":523,"start_character":55,"end_line":523,"end_character":64},"updated":"2017-06-02 05:19:22.000000000","message":"alternative","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":528,"context_line":".. note::"},{"line_number":529,"context_line":""},{"line_number":530,"context_line":"   The replicator does not maintain desired levels of replication when other"},{"line_number":531,"context_line":"   failures, such as entire node failures, occur because most failure are"},{"line_number":532,"context_line":"   transient."},{"line_number":533,"context_line":""},{"line_number":534,"context_line":"   Replication is an area of active development, and likely rife with potential"},{"line_number":535,"context_line":"   improvements to speed and correctness."}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_3ad4cf47","line":532,"range":{"start_line":531,"start_character":3,"end_line":532,"end_character":13},"updated":"2017-06-02 05:19:22.000000000","message":"other failures occur, such as entire node failures, because most failures are transient.","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"},{"author":{"_account_id":12686,"name":"Brian Moss","email":"kallimachos@gmail.com","username":"bmoss"},"change_message_id":"095398723d3dae6cc3034eae7ec8e9599216c767","unresolved":false,"context_lines":[{"line_number":531,"context_line":"   failures, such as entire node failures, occur because most failure are"},{"line_number":532,"context_line":"   transient."},{"line_number":533,"context_line":""},{"line_number":534,"context_line":"   Replication is an area of active development, and likely rife with potential"},{"line_number":535,"context_line":"   improvements to speed and correctness."},{"line_number":536,"context_line":""},{"line_number":537,"context_line":"There are two major classes of replicator: the db replicator, which replicates"},{"line_number":538,"context_line":"accounts and containers, and the object replicator, which replicates object"}],"source_content_type":"text/x-rst","patch_set":12,"id":"df140735_bac7bf96","line":535,"range":{"start_line":534,"start_character":49,"end_line":535,"end_character":41},"updated":"2017-06-02 05:19:22.000000000","message":"and implementation details are likely to change over time.","commit_id":"7e379432ef3e70d7d7262ee323ac6ccf35674347"}]}