)]}'
{"doc/source/associated_projects.rst":[{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"165e9efb80bb33b03f286a5347c76c1647459d6e","unresolved":false,"context_lines":[{"line_number":86,"context_line":"* `Swiftsync \u003chttps://github.com/stackforge/swiftsync\u003e`_ - A massive syncer between two swift clusters."},{"line_number":87,"context_line":"* `Swiftbrowser \u003chttps://github.com/cschwede/django-swiftbrowser\u003e`_ - Simple web app to access Openstack Swift."},{"line_number":88,"context_line":"* `Swift-account-stats \u003chttps://github.com/enovance/swift-account-stats\u003e`_ - Swift-account-stats is a tool to report statistics on Swift usage at tenant and global levels."},{"line_number":89,"context_line":"* `PyECLib \u003chttps://bitbucket.org/kmgreen2/pyeclib\u003e`_ - Erasure Code library used by Swift"}],"source_content_type":"text/x-rst","patch_set":6,"id":"baada198_8bddb6e2","line":89,"updated":"2014-07-14 14:55:11.000000000","message":"I will add a liberasurecode reference here.","commit_id":"117aa81aa6f0911bc6d92c084944e6d57e4a47fa"}],"doc/source/overview_erasure_code.rst":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"dce21af6c78370d126da6f68967c82681d076341","unresolved":false,"context_lines":[{"line_number":29,"context_line":""},{"line_number":30,"context_line":"At a high level EC works by using a specific scheme to break up a single data buffer"},{"line_number":31,"context_line":"into several smaller data buffers then, depending on the scheme, performing some encoding"},{"line_number":32,"context_line":"operation on that data in odder to generate addition information.  So you end up with more"},{"line_number":33,"context_line":"data than you started with and that extra data is often called \"parity\".  Note that there are"},{"line_number":34,"context_line":"many, many different encoding techniques that vary both in how they organize and manipulate"},{"line_number":35,"context_line":"the data as well by what means they use to calculate parity.  For example, one scheme might"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_b8d30037","line":32,"updated":"2014-07-11 02:11:59.000000000","message":"s/odder/order and s/addtion/additional","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a155f93a2501b90d0cfa38a4085972f7bd040a49","unresolved":false,"context_lines":[{"line_number":29,"context_line":""},{"line_number":30,"context_line":"At a high level EC works by using a specific scheme to break up a single data buffer"},{"line_number":31,"context_line":"into several smaller data buffers then, depending on the scheme, performing some encoding"},{"line_number":32,"context_line":"operation on that data in odder to generate addition information.  So you end up with more"},{"line_number":33,"context_line":"data than you started with and that extra data is often called \"parity\".  Note that there are"},{"line_number":34,"context_line":"many, many different encoding techniques that vary both in how they organize and manipulate"},{"line_number":35,"context_line":"the data as well by what means they use to calculate parity.  For example, one scheme might"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_29afa4e4","line":32,"in_reply_to":"baada198_b8d30037","updated":"2014-07-11 03:44:14.000000000","message":"Done","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"dce21af6c78370d126da6f68967c82681d076341","unresolved":false,"context_lines":[{"line_number":62,"context_line":"The Swift code base doesn\u0027t include any of the algorithms necessary to perform the actual"},{"line_number":63,"context_line":"encoding and decoding of data; that is left to an external library.  The Storage Policies"},{"line_number":64,"context_line":"architecture is leveraged to allow EC on a per container basis and the object rings still"},{"line_number":65,"context_line":"provide for placement of EC data fragments.  Although there are several code paths that are"},{"line_number":66,"context_line":"unique to an operation associated with an EC policy, an external dependency to an Erasure Code"},{"line_number":67,"context_line":"library is what Swift counts on to perform the low level EC functions.  The use of an external"},{"line_number":68,"context_line":"library allows for maximum flexibility as there are a significant number of options out there,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_78ade887","line":65,"updated":"2014-07-11 02:11:59.000000000","message":"s/for/the","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a155f93a2501b90d0cfa38a4085972f7bd040a49","unresolved":false,"context_lines":[{"line_number":62,"context_line":"The Swift code base doesn\u0027t include any of the algorithms necessary to perform the actual"},{"line_number":63,"context_line":"encoding and decoding of data; that is left to an external library.  The Storage Policies"},{"line_number":64,"context_line":"architecture is leveraged to allow EC on a per container basis and the object rings still"},{"line_number":65,"context_line":"provide for placement of EC data fragments.  Although there are several code paths that are"},{"line_number":66,"context_line":"unique to an operation associated with an EC policy, an external dependency to an Erasure Code"},{"line_number":67,"context_line":"library is what Swift counts on to perform the low level EC functions.  The use of an external"},{"line_number":68,"context_line":"library allows for maximum flexibility as there are a significant number of options out there,"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_a982d44f","line":65,"in_reply_to":"baada198_78ade887","updated":"2014-07-11 03:44:14.000000000","message":"actually for is what I meant :)  \"for the\" works also though","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"dce21af6c78370d126da6f68967c82681d076341","unresolved":false,"context_lines":[{"line_number":128,"context_line":"Under the Hood section."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"--------------"},{"line_number":131,"context_line":"Reconstruction"},{"line_number":132,"context_line":"--------------"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"The basic framework of how replication works is also used with an EC policy a few notable"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_b81220b2","line":131,"updated":"2014-07-11 02:11:59.000000000","message":"Reconstruction or  do you mean replication here?","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a155f93a2501b90d0cfa38a4085972f7bd040a49","unresolved":false,"context_lines":[{"line_number":128,"context_line":"Under the Hood section."},{"line_number":129,"context_line":""},{"line_number":130,"context_line":"--------------"},{"line_number":131,"context_line":"Reconstruction"},{"line_number":132,"context_line":"--------------"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"The basic framework of how replication works is also used with an EC policy a few notable"}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_c985485a","line":131,"in_reply_to":"baada198_b81220b2","updated":"2014-07-11 03:44:14.000000000","message":"No, Reconstruction.  I\u0027ll make the intro a little clearer because I start of talking about replication and don\u0027t mention reconstruction.  Take another look after I push the next one and see if it makes more sense.  Thanks!","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"dce21af6c78370d126da6f68967c82681d076341","unresolved":false,"context_lines":[{"line_number":212,"context_line":"paying special attention to using the correct terms consistently, it is very easy to get confused"},{"line_number":213,"context_line":"in a hurry!"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"* segment: not to be confused with SLO/DLO use of the work, in EC we call a segment a series of"},{"line_number":216,"context_line":"consecutive HTTP chunks buffered up before performing an EC operation."},{"line_number":217,"context_line":"* fragment: data and parity \u0027fragments\u0027 are generated when erasure coding transformation is applied"},{"line_number":218,"context_line":"to a segment."}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_18d5ccb8","line":215,"updated":"2014-07-11 02:11:59.000000000","message":"s/work/word/","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"a155f93a2501b90d0cfa38a4085972f7bd040a49","unresolved":false,"context_lines":[{"line_number":212,"context_line":"paying special attention to using the correct terms consistently, it is very easy to get confused"},{"line_number":213,"context_line":"in a hurry!"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"* segment: not to be confused with SLO/DLO use of the work, in EC we call a segment a series of"},{"line_number":216,"context_line":"consecutive HTTP chunks buffered up before performing an EC operation."},{"line_number":217,"context_line":"* fragment: data and parity \u0027fragments\u0027 are generated when erasure coding transformation is applied"},{"line_number":218,"context_line":"to a segment."}],"source_content_type":"text/x-rst","patch_set":2,"id":"baada198_49913899","line":215,"in_reply_to":"baada198_18d5ccb8","updated":"2014-07-11 03:44:14.000000000","message":"Done","commit_id":"4d18a755bdd0813854fa64dd482c140dc448303a"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":35,"context_line":"the data as well by what means they use to calculate parity.  For example, one scheme might"},{"line_number":36,"context_line":"rely on Galois Field Arithmetic while others may work with only XOR. The number of variations"},{"line_number":37,"context_line":"and details about their differences are well beyond the scope of this introduction, but we\u0027ll"},{"line_number":38,"context_line":"talk more about of few of them when we get into the implementation of EC in Swift."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"--------------------------------"},{"line_number":41,"context_line":"Overview of EC Support in Swift"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_40a2d44b","line":38,"updated":"2014-07-11 16:15:41.000000000","message":"a few of them","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":35,"context_line":"the data as well by what means they use to calculate parity.  For example, one scheme might"},{"line_number":36,"context_line":"rely on Galois Field Arithmetic while others may work with only XOR. The number of variations"},{"line_number":37,"context_line":"and details about their differences are well beyond the scope of this introduction, but we\u0027ll"},{"line_number":38,"context_line":"talk more about of few of them when we get into the implementation of EC in Swift."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"--------------------------------"},{"line_number":41,"context_line":"Overview of EC Support in Swift"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_f5d1dd57","line":38,"in_reply_to":"baada198_40a2d44b","updated":"2014-07-11 19:38:15.000000000","message":"Done","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":50,"context_line":"details on Storage Policies.  Because support is implemented as a Storage Policy, all of"},{"line_number":51,"context_line":"the storage devices associated with your cluster\u0027s EC capability can be isolated.  It is"},{"line_number":52,"context_line":"entirely possible to share devices between storage policies, but for EC it may make more"},{"line_number":53,"context_line":"sense to not only use separate devices but possibly even entire nodes dedicated for EC."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":56,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_403e54db","line":53,"updated":"2014-07-11 16:15:41.000000000","message":"This last sentence is an important one that could affect how people perceive configuration/deployment of clusters, but no reasoning is given. Not sure this is the correct place, but it seems important to explain why it makes sense to separate devices and nodes for EC and at least provide a link here","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":50,"context_line":"details on Storage Policies.  Because support is implemented as a Storage Policy, all of"},{"line_number":51,"context_line":"the storage devices associated with your cluster\u0027s EC capability can be isolated.  It is"},{"line_number":52,"context_line":"entirely possible to share devices between storage policies, but for EC it may make more"},{"line_number":53,"context_line":"sense to not only use separate devices but possibly even entire nodes dedicated for EC."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":56,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_d5d5b9e5","line":53,"in_reply_to":"baada198_403e54db","updated":"2014-07-11 19:38:15.000000000","message":"Yeah, good point.  Take a look at what I added and feel free to comment/edit on other scenarios that may be better to callout as examples","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":104,"context_line":"#. The proxy server streams in an object and buffers up some amount of data (configurable)"},{"line_number":105,"context_line":"#. The proxy server calls on PyECLib to encode the data into fragments"},{"line_number":106,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":107,"context_line":"#. Rinse and repeat until the client is done sending data"},{"line_number":108,"context_line":"#. The client is notified of completion when a quorum is met (configurable)."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"The GET flow looks like this:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_0e072d19","line":107,"updated":"2014-07-11 16:15:41.000000000","message":"This is an idiom that might not be understood by people whose English is not their first language","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":104,"context_line":"#. The proxy server streams in an object and buffers up some amount of data (configurable)"},{"line_number":105,"context_line":"#. The proxy server calls on PyECLib to encode the data into fragments"},{"line_number":106,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":107,"context_line":"#. Rinse and repeat until the client is done sending data"},{"line_number":108,"context_line":"#. The client is notified of completion when a quorum is met (configurable)."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"The GET flow looks like this:"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_35fc3572","line":107,"in_reply_to":"baada198_0e072d19","updated":"2014-07-11 19:38:15.000000000","message":"heh, good point","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":135,"context_line":"few notable exceptions:"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* Because EC does actually replicate partitions, it needs to operate at a finer granularity"},{"line_number":138,"context_line":"than what is provided with rsync, therefore using an EC policy requires the use of ssync"},{"line_number":139,"context_line":"* Once a pair of nodes has determined the need to replace a missing object fragment, instead"},{"line_number":140,"context_line":"of pushing over a copy, like replication would do, the reconstructor has to read in enough"},{"line_number":141,"context_line":"surviving fragments from other nodes and perform a local reconstruction before it has the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_ae8ac161","line":138,"updated":"2014-07-11 16:15:41.000000000","message":"This is confusing..why does it replicate partitions? I guess I would also need to better understand the differences between rsync and ssync before reading this...","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":135,"context_line":"few notable exceptions:"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* Because EC does actually replicate partitions, it needs to operate at a finer granularity"},{"line_number":138,"context_line":"than what is provided with rsync, therefore using an EC policy requires the use of ssync"},{"line_number":139,"context_line":"* Once a pair of nodes has determined the need to replace a missing object fragment, instead"},{"line_number":140,"context_line":"of pushing over a copy, like replication would do, the reconstructor has to read in enough"},{"line_number":141,"context_line":"surviving fragments from other nodes and perform a local reconstruction before it has the"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_15ce512e","line":138,"in_reply_to":"baada198_ae8ac161","updated":"2014-07-11 19:38:15.000000000","message":"sorry, left out the word \"not\" which I guess changes things a bit :)\n\nAlso reworded a bit as from the end users perspective we may not appear to be using \u0027ssync\u0027 in the final implementation.  We\u0027re just re-using a ton of what\u0027s there.  Once the reconstructor takes more shape this section will get an overhaul","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":180,"context_line":"* ec_num_parity_fragments:  the total number of fragments that will be comprised of parity"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"When PyECLib encodes an object, it will break it into N fragments however during configuration"},{"line_number":183,"context_line":"what\u0027s important is how many of those data and how many are parity.  So in the example above,"},{"line_number":184,"context_line":"PyECLib will actually break an object in 14 different fragments, 10 of them will be made up of"},{"line_number":185,"context_line":"actual object data and 4 of them will be made of parity data (calculations depending on ec_type)."},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_aefd61b3","line":183,"updated":"2014-07-11 16:15:41.000000000","message":"...those are data and how....","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":180,"context_line":"* ec_num_parity_fragments:  the total number of fragments that will be comprised of parity"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"When PyECLib encodes an object, it will break it into N fragments however during configuration"},{"line_number":183,"context_line":"what\u0027s important is how many of those data and how many are parity.  So in the example above,"},{"line_number":184,"context_line":"PyECLib will actually break an object in 14 different fragments, 10 of them will be made up of"},{"line_number":185,"context_line":"actual object data and 4 of them will be made of parity data (calculations depending on ec_type)."},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_35b43581","line":183,"in_reply_to":"baada198_aefd61b3","updated":"2014-07-11 19:38:15.000000000","message":"Done","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":195,"context_line":"A common usage of EC is to migrate less commonly accessed data from a more expensive but"},{"line_number":196,"context_line":"lower latency policy such as replication.  When an application determines that it wants to"},{"line_number":197,"context_line":"move data from a replication policy to an EC policy, it simply needs to move the data from"},{"line_number":198,"context_line":"the EC container to a different container that was created with the target durability policy."},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"--------------"},{"line_number":201,"context_line":"Under the Hood"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_cea295ad","line":198,"updated":"2014-07-11 16:15:41.000000000","message":"does swift provide an automated way moving data from one container to another? if so, this is a good place to mention it...","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":195,"context_line":"A common usage of EC is to migrate less commonly accessed data from a more expensive but"},{"line_number":196,"context_line":"lower latency policy such as replication.  When an application determines that it wants to"},{"line_number":197,"context_line":"move data from a replication policy to an EC policy, it simply needs to move the data from"},{"line_number":198,"context_line":"the EC container to a different container that was created with the target durability policy."},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"--------------"},{"line_number":201,"context_line":"Under the Hood"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_957501cd","line":198,"in_reply_to":"baada198_cea295ad","updated":"2014-07-11 19:38:15.000000000","message":"good question, I don\u0027t know","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":201,"context_line":"Under the Hood"},{"line_number":202,"context_line":"--------------"},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"Now that we\u0027ve explained a little about EC support in Swift are and how to configure/use it,"},{"line_number":205,"context_line":"let\u0027s explore how EC fits in at the nuts-n-bolts level."},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"Terminology"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_8e733d4d","line":204,"updated":"2014-07-11 16:15:41.000000000","message":"remove \u0027are\u0027\n\"... EC support in swift and how to...\"","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":201,"context_line":"Under the Hood"},{"line_number":202,"context_line":"--------------"},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"Now that we\u0027ve explained a little about EC support in Swift are and how to configure/use it,"},{"line_number":205,"context_line":"let\u0027s explore how EC fits in at the nuts-n-bolts level."},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"Terminology"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_f54b9d84","line":204,"in_reply_to":"baada198_8e733d4d","updated":"2014-07-11 19:38:15.000000000","message":"Done","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"6305b03c1334756bc2e56896dbec3622d587d353","unresolved":false,"context_lines":[{"line_number":228,"context_line":"SLO/DLO middleware, the segments of your large object will be erasure coded (if being stored"},{"line_number":229,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":230,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":231,"context_line":"the actual object, not the locations of all of its fragments."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":"On Disk Storage"},{"line_number":234,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_ae43a1ad","line":231,"updated":"2014-07-11 16:15:41.000000000","message":"this last sentence is confusing. how could i get endpoint information about the actual object if the object was divided up in fragments?","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"702411e1db3e0e71d104d43149e5d34854073009","unresolved":false,"context_lines":[{"line_number":228,"context_line":"SLO/DLO middleware, the segments of your large object will be erasure coded (if being stored"},{"line_number":229,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":230,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":231,"context_line":"the actual object, not the locations of all of its fragments."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":"On Disk Storage"},{"line_number":234,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_353995f2","line":231,"in_reply_to":"baada198_ae43a1ad","updated":"2014-07-11 19:38:15.000000000","message":"hmm, the point is that the fact that its broken up is transparent to middleware.  Maybe I\u0027ll just state that and see how that reads","commit_id":"e921dfe4d0b2bf6a7bd3c658872360e481f3e5f7"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a34814c3acda1ca6d5e634de0515a94fe906fbf6","unresolved":false,"context_lines":[{"line_number":35,"context_line":"the data as well by what means they use to calculate parity.  For example, one scheme might"},{"line_number":36,"context_line":"rely on Galois Field Arithmetic while others may work with only XOR. The number of variations"},{"line_number":37,"context_line":"and details about their differences are well beyond the scope of this introduction, but we\u0027ll"},{"line_number":38,"context_line":"talk more about of few of them when we get into the implementation of EC in Swift."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"--------------------------------"},{"line_number":41,"context_line":"Overview of EC Support in Swift"}],"source_content_type":"text/x-rst","patch_set":5,"id":"baada198_9ff09792","line":38,"updated":"2014-07-14 12:21:58.000000000","message":"The of has come back again: .s/of few of them/a few of them/","commit_id":"245b7e74b65a6363d5a60f81235228e3bfe02533"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a34814c3acda1ca6d5e634de0515a94fe906fbf6","unresolved":false,"context_lines":[{"line_number":50,"context_line":"details on Storage Policies.  Because support is implemented as a Storage Policy, all of"},{"line_number":51,"context_line":"the storage devices associated with your cluster\u0027s EC capability can be isolated.  It is"},{"line_number":52,"context_line":"entirely possible to share devices between storage policies, but for EC it may make more"},{"line_number":53,"context_line":"sense to not only use separate devices but possibly even entire nodes dedicated for EC."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":56,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"}],"source_content_type":"text/x-rst","patch_set":5,"id":"baada198_6ab14346","line":53,"updated":"2014-07-14 12:21:58.000000000","message":"I personally like the examples you seemed to have removed from the last patchset. Have you changed and uploaded an other version of the patchet, removing already fixed changes?","commit_id":"245b7e74b65a6363d5a60f81235228e3bfe02533"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a34814c3acda1ca6d5e634de0515a94fe906fbf6","unresolved":false,"context_lines":[{"line_number":134,"context_line":"The basic framework of reconstruction is very similar to that of replication with a"},{"line_number":135,"context_line":"few notable exceptions:"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"* Because EC does actually replicate partitions, it needs to operate at a finer granularity"},{"line_number":138,"context_line":"than what is provided with rsync, therefore using an EC policy requires the use of ssync"},{"line_number":139,"context_line":"* Once a pair of nodes has determined the need to replace a missing object fragment, instead"},{"line_number":140,"context_line":"of pushing over a copy, like replication would do, the reconstructor has to read in enough"}],"source_content_type":"text/x-rst","patch_set":5,"id":"baada198_ca65efaf","line":137,"updated":"2014-07-14 12:21:58.000000000","message":"does not","commit_id":"245b7e74b65a6363d5a60f81235228e3bfe02533"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a34814c3acda1ca6d5e634de0515a94fe906fbf6","unresolved":false,"context_lines":[{"line_number":180,"context_line":"* ec_num_parity_fragments:  the total number of fragments that will be comprised of parity"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"When PyECLib encodes an object, it will break it into N fragments however during configuration"},{"line_number":183,"context_line":"what\u0027s important is how many of those data and how many are parity.  So in the example above,"},{"line_number":184,"context_line":"PyECLib will actually break an object in 14 different fragments, 10 of them will be made up of"},{"line_number":185,"context_line":"actual object data and 4 of them will be made of parity data (calculations depending on ec_type)."},{"line_number":186,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"baada198_aa499b61","line":183,"updated":"2014-07-14 12:21:58.000000000","message":"The are is missing agian: s/those datathose are data/","commit_id":"245b7e74b65a6363d5a60f81235228e3bfe02533"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a34814c3acda1ca6d5e634de0515a94fe906fbf6","unresolved":false,"context_lines":[{"line_number":201,"context_line":"Under the Hood"},{"line_number":202,"context_line":"--------------"},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"Now that we\u0027ve explained a little about EC support in Swift are and how to configure/use it,"},{"line_number":205,"context_line":"let\u0027s explore how EC fits in at the nuts-n-bolts level."},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"Terminology"}],"source_content_type":"text/x-rst","patch_set":5,"id":"baada198_0aefe71e","line":204,"updated":"2014-07-14 12:21:58.000000000","message":"Remove \u0027are\u0027 s/swift are and/swift and/\n\nAgain I think this the return of something that was previously fixed.","commit_id":"245b7e74b65a6363d5a60f81235228e3bfe02533"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"dc96b4bef96b9508afa129c5800f1bfa8664107e","unresolved":false,"context_lines":[{"line_number":228,"context_line":"SLO/DLO middleware, the segments of your large object will be erasure coded (if being stored"},{"line_number":229,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":230,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":231,"context_line":"the actual object, not the locations of all of its fragments."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":"On Disk Storage"},{"line_number":234,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"baada198_b3b681f7","line":231,"updated":"2014-07-14 05:55:24.000000000","message":"With respect to list_endpoints: what does it means the endpoint information for the actual object? I mean when erasure coding is active, than the object is divided into chunks stored on different object servers. Where is the actual object in this case?","commit_id":"245b7e74b65a6363d5a60f81235228e3bfe02533"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"de5c91ce95afc8406eb2777c6f6ac4e6d0957782","unresolved":false,"context_lines":[{"line_number":228,"context_line":"SLO/DLO middleware, the segments of your large object will be erasure coded (if being stored"},{"line_number":229,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":230,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":231,"context_line":"the actual object, not the locations of all of its fragments."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":"On Disk Storage"},{"line_number":234,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":5,"id":"baada198_bf471ba2","line":231,"in_reply_to":"baada198_b3b681f7","updated":"2014-07-14 12:09:24.000000000","message":"The actual object is fragmented throughout the cluster however the proxy has knowledge of the fragments and will find them/reassemble them transparently to any middleware","commit_id":"245b7e74b65a6363d5a60f81235228e3bfe02533"},{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"165e9efb80bb33b03f286a5347c76c1647459d6e","unresolved":false,"context_lines":[{"line_number":33,"context_line":"data than you started with and that extra data is often called \"parity\".  Note that there are"},{"line_number":34,"context_line":"many, many different encoding techniques that vary both in how they organize and manipulate"},{"line_number":35,"context_line":"the data as well by what means they use to calculate parity.  For example, one scheme might"},{"line_number":36,"context_line":"rely on `Galois Field Arithmetic \u003chttps://www.google.com/url?sa\u003dt\u0026rct\u003dj\u0026q\u003d\u0026esrc\u003ds\u0026source\u003dweb\u0026cd\u003d1\u0026cad\u003drja\u0026uact\u003d8\u0026ved\u003d0CB8QFjAA\u0026url\u003dhttp%3A%2F%2Fwww.ssrc.ucsc.edu%2FPapers%2Fplank-fast13.pdf\u0026ei\u003dFMbDU6fwF4iwoQTM8ICQCA\u0026usg\u003dAFQjCNHJu_ZsW8OkA9_zsjNvnRzy5YZXYA\u0026bvm\u003dbv.70810081,d.cGU\u003e`_ while others may work with only XOR. The number of variations"},{"line_number":37,"context_line":"and details about their differences are well beyond the scope of this introduction, but we\u0027ll"},{"line_number":38,"context_line":"talk more about a few of them when we get into the implementation of EC in Swift."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":6,"id":"baada198_4b9e3e08","line":36,"updated":"2014-07-14 14:55:11.000000000","message":"http://www.ssrc.ucsc.edu/Papers/plank-fast13.pdf","commit_id":"117aa81aa6f0911bc6d92c084944e6d57e4a47fa"},{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"165e9efb80bb33b03f286a5347c76c1647459d6e","unresolved":false,"context_lines":[{"line_number":52,"context_line":"entirely possible to share devices between storage policies, but for EC it may make more"},{"line_number":53,"context_line":"sense to not only use separate devices but possibly even entire nodes dedicated for EC."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Wich direction one chooses depends on why the EC policy is being deployed.  If, for"},{"line_number":56,"context_line":"example, there is a production replication policy in place already and the goal is to add"},{"line_number":57,"context_line":"a cold storage tier such that the existing nodes performing replication are impacted as"},{"line_number":58,"context_line":"little as possible, adding a new set of nodes dedicated to EC might make the most sense"}],"source_content_type":"text/x-rst","patch_set":6,"id":"baada198_6b6f82fc","line":55,"updated":"2014-07-14 14:55:11.000000000","message":"Which","commit_id":"117aa81aa6f0911bc6d92c084944e6d57e4a47fa"},{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"165e9efb80bb33b03f286a5347c76c1647459d6e","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Layout of existing infrastructure"},{"line_number":67,"context_line":"* Cost of adding a dedicated EC nodes (or just dedicated EC devices)"},{"line_number":68,"context_line":"* Intended ussage model(s)"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":71,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"}],"source_content_type":"text/x-rst","patch_set":6,"id":"baada198_6bb86275","line":68,"updated":"2014-07-14 14:55:11.000000000","message":"usage","commit_id":"117aa81aa6f0911bc6d92c084944e6d57e4a47fa"},{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"165e9efb80bb33b03f286a5347c76c1647459d6e","unresolved":false,"context_lines":[{"line_number":95,"context_line":"* Jerasure at https://bitbucket.org/jimplank/jerasure)"},{"line_number":96,"context_line":"* GFComplete at https://bitbucket.org/jimplank/gf-complete)"},{"line_number":97,"context_line":"* Intel(R) ISA-L at https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version"},{"line_number":98,"context_line":"* Or write your own!"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"PyECLib itself therefore allows for not only choice, but further extensibility as well. PyECLib also"},{"line_number":101,"context_line":"comes with a handy utility to help determine the best algorithm to use based on the equipment that"}],"source_content_type":"text/x-rst","patch_set":6,"id":"baada198_2b374a00","line":98,"updated":"2014-07-14 14:55:11.000000000","message":"Will add xor codes reference (part of liberasurecode now) and push another draft.","commit_id":"117aa81aa6f0911bc6d92c084944e6d57e4a47fa"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"361c147acbd643c54c6e46745684cc5c2f25247d","unresolved":false,"context_lines":[{"line_number":33,"context_line":"data than you started with and that extra data is often called \"parity\".  Note that there are"},{"line_number":34,"context_line":"many, many different encoding techniques that vary both in how they organize and manipulate"},{"line_number":35,"context_line":"the data as well by what means they use to calculate parity.  For example, one scheme might"},{"line_number":36,"context_line":"rely on `Galois Field Arithmetic \u003chttps://www.google.com/url?sa\u003dt\u0026rct\u003dj\u0026q\u003d\u0026esrc\u003ds\u0026source\u003dweb\u0026cd\u003d1\u0026cad\u003drja\u0026uact\u003d8\u0026ved\u003d0CB8QFjAA\u0026url\u003dhttp%3A%2F%2Fwww.ssrc.ucsc.edu%2FPapers%2Fplank-fast13.pdf\u0026ei\u003dFMbDU6fwF4iwoQTM8ICQCA\u0026usg\u003dAFQjCNHJu_ZsW8OkA9_zsjNvnRzy5YZXYA\u0026bvm\u003dbv.70810081,d.cGU\u003e`_ while others may work with only XOR. The number of variations"},{"line_number":37,"context_line":"and details about their differences are well beyond the scope of this introduction, but we\u0027ll"},{"line_number":38,"context_line":"talk more about a few of them when we get into the implementation of EC in Swift."},{"line_number":39,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"baada198_c2a9ba25","line":36,"updated":"2014-07-17 15:24:05.000000000","message":"Like Tushar said in the last patch set, the link to Galois Field Arithmetic you gave is a google search link rather then the real link which is:\n\n  http://www.ssrc.ucsc.edu/Papers/plank-fast13.pdf","commit_id":"29dd6653f97b6d1906028da795c723159228c553"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"bde2e98ca6e468d18b4e636e40ba50ca43bf478c","unresolved":false,"context_lines":[{"line_number":1,"context_line":"*** NOTE:  EC related docs are WIP and may/may not reflect the current state"},{"line_number":2,"context_line":"of the codebase on the feature/ec branch.  They will continue to be a WIP"},{"line_number":3,"context_line":"until we get closer and closer to code complete."},{"line_number":4,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_a6c77363","line":1,"updated":"2014-07-17 17:42:50.000000000","message":"Not actually the right note syntax","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"bde2e98ca6e468d18b4e636e40ba50ca43bf478c","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Layout of existing infrastructure"},{"line_number":67,"context_line":"* Cost of adding a dedicated EC nodes (or just dedicated EC devices)"},{"line_number":68,"context_line":"* Intended usage model(s)"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":71,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_a32e85a3","line":68,"updated":"2014-07-17 17:42:50.000000000","message":"Nit for later: This is a confusingly-worded paragraph.","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Layout of existing infrastructure"},{"line_number":67,"context_line":"* Cost of adding a dedicated EC nodes (or just dedicated EC devices)"},{"line_number":68,"context_line":"* Intended usage model(s)"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":71,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_409bb465","line":68,"in_reply_to":"baada198_a32e85a3","updated":"2014-07-18 19:41:22.000000000","message":"added todo","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"bde2e98ca6e468d18b4e636e40ba50ca43bf478c","unresolved":false,"context_lines":[{"line_number":68,"context_line":"* Intended usage model(s)"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":71,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"},{"line_number":72,"context_line":"transparent to users of the cluster.  Note that it is possible to have multiple EC containers"},{"line_number":73,"context_line":"each with different EC policies that have different EC parameters just as it\u0027s possible to"},{"line_number":74,"context_line":"have a cluster that has multiple different replication policies with different"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_c300d904","line":71,"updated":"2014-07-17 17:42:50.000000000","message":"Nit for later: \"policy. However,\"\n\nor reword the sentence","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":68,"context_line":"* Intended usage model(s)"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"EC support impacts many of the code paths and background operations for data stored in a"},{"line_number":71,"context_line":"container that was created with an EC policy however, as mentioned above, this is all"},{"line_number":72,"context_line":"transparent to users of the cluster.  Note that it is possible to have multiple EC containers"},{"line_number":73,"context_line":"each with different EC policies that have different EC parameters just as it\u0027s possible to"},{"line_number":74,"context_line":"have a cluster that has multiple different replication policies with different"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_60a3589e","line":71,"in_reply_to":"baada198_c300d904","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":85,"context_line":""},{"line_number":86,"context_line":"---------------------------------------"},{"line_number":87,"context_line":"PyECLib:  External Erasure Code Library"},{"line_number":88,"context_line":"---------------------------------------"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"PyECLib is a Python Erasure Coding Library originally designed and written as part of the"},{"line_number":91,"context_line":"effort to add EC support to the Swift project, however it is an independent project.  The"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_5c8f2dcd","line":88,"updated":"2014-07-18 10:27:02.000000000","message":"suggestion: maybe move this section to end (where there is already a placeholder for pyeclib) and just put a link in paragraph above.","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":85,"context_line":""},{"line_number":86,"context_line":"---------------------------------------"},{"line_number":87,"context_line":"PyECLib:  External Erasure Code Library"},{"line_number":88,"context_line":"---------------------------------------"},{"line_number":89,"context_line":""},{"line_number":90,"context_line":"PyECLib is a Python Erasure Coding Library originally designed and written as part of the"},{"line_number":91,"context_line":"effort to add EC support to the Swift project, however it is an independent project.  The"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_a0d730ed","line":88,"in_reply_to":"baada198_5c8f2dcd","updated":"2014-07-18 19:41:22.000000000","message":"Intent was that this beginning part is an intro and towards the end is more detail, once the latter shapes up if it turns out they\u0027re repeating each other then I\u0027d agree.","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":92,"context_line":"library provides a well-defined and simple Python interface and internally implements a"},{"line_number":93,"context_line":"plug-in architecture allowing it to take advantage of many well-known C libraries such as:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"* Jerasure at https://bitbucket.org/jimplank/jerasure)"},{"line_number":96,"context_line":"* GFComplete at https://bitbucket.org/jimplank/gf-complete)"},{"line_number":97,"context_line":"* Intel(R) ISA-L at https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version"},{"line_number":98,"context_line":"* Or write your own!"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_1c269516","line":95,"updated":"2014-07-18 10:27:02.000000000","message":"nit: delete trailing )","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":92,"context_line":"library provides a well-defined and simple Python interface and internally implements a"},{"line_number":93,"context_line":"plug-in architecture allowing it to take advantage of many well-known C libraries such as:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"* Jerasure at https://bitbucket.org/jimplank/jerasure)"},{"line_number":96,"context_line":"* GFComplete at https://bitbucket.org/jimplank/gf-complete)"},{"line_number":97,"context_line":"* Intel(R) ISA-L at https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version"},{"line_number":98,"context_line":"* Or write your own!"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_e004c87f","line":95,"in_reply_to":"baada198_1c269516","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":93,"context_line":"plug-in architecture allowing it to take advantage of many well-known C libraries such as:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"* Jerasure at https://bitbucket.org/jimplank/jerasure)"},{"line_number":96,"context_line":"* GFComplete at https://bitbucket.org/jimplank/gf-complete)"},{"line_number":97,"context_line":"* Intel(R) ISA-L at https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version"},{"line_number":98,"context_line":"* Or write your own!"},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_3c235906","line":96,"updated":"2014-07-18 10:27:02.000000000","message":"ditto","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":93,"context_line":"plug-in architecture allowing it to take advantage of many well-known C libraries such as:"},{"line_number":94,"context_line":""},{"line_number":95,"context_line":"* Jerasure at https://bitbucket.org/jimplank/jerasure)"},{"line_number":96,"context_line":"* GFComplete at https://bitbucket.org/jimplank/gf-complete)"},{"line_number":97,"context_line":"* Intel(R) ISA-L at https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version"},{"line_number":98,"context_line":"* Or write your own!"},{"line_number":99,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_000a9c89","line":96,"in_reply_to":"baada198_3c235906","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":114,"context_line":""},{"line_number":115,"context_line":"The PUT flow looks like this:"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"#. The proxy server streams in an object and buffers up some amount of data (configurable)"},{"line_number":118,"context_line":"#. The proxy server calls on PyECLib to encode the data into fragments"},{"line_number":119,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":120,"context_line":"#. Repeat until the client is done sending data"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_b704e215","line":117,"updated":"2014-07-18 10:27:02.000000000","message":"was there a term for the configurable \u0027amount of data\u0027? I know its not \u0027chunk\u0027 :)","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":114,"context_line":""},{"line_number":115,"context_line":"The PUT flow looks like this:"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"#. The proxy server streams in an object and buffers up some amount of data (configurable)"},{"line_number":118,"context_line":"#. The proxy server calls on PyECLib to encode the data into fragments"},{"line_number":119,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":120,"context_line":"#. Repeat until the client is done sending data"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_60f8b87c","line":117,"in_reply_to":"baada198_b704e215","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":119,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":120,"context_line":"#. Repeat until the client is done sending data"},{"line_number":121,"context_line":"#. The client is notified of completion when a quorum is met (configurable)."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"The GET flow looks like this:"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"#. The proxy server makes simultaneous requests to some number of participating nodes*"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_f70ada1e","line":122,"updated":"2014-07-18 10:27:02.000000000","message":"Somewhere it would be useful to point out the key differences w.r.t. replication . e.g. point 3 above, different fragments go to different nodes","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":119,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":120,"context_line":"#. Repeat until the client is done sending data"},{"line_number":121,"context_line":"#. The client is notified of completion when a quorum is met (configurable)."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"The GET flow looks like this:"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"#. The proxy server makes simultaneous requests to some number of participating nodes*"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_ee0af9d0","line":122,"in_reply_to":"baada198_f70ada1e","updated":"2014-07-18 19:41:22.000000000","message":"good call, not quite sure how to do that right now but I think once we flesh out the \u0027under the hood section\u0027 we can come back up and do a better job summarizing this.","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":122,"context_line":""},{"line_number":123,"context_line":"The GET flow looks like this:"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"#. The proxy server makes simultaneous requests to some number of participating nodes*"},{"line_number":126,"context_line":"#. As soon as the proxy has the fragments it needs, it calls on PyECLib to decode the data"},{"line_number":127,"context_line":"#. The proxy streams the decoded data it has back to the client"},{"line_number":128,"context_line":"#. Repeat until the proxy is done sending data back to the client"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_5c73ad15","line":125,"updated":"2014-07-18 10:27:02.000000000","message":"nit: delete * at end of line","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":122,"context_line":""},{"line_number":123,"context_line":"The GET flow looks like this:"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"#. The proxy server makes simultaneous requests to some number of participating nodes*"},{"line_number":126,"context_line":"#. As soon as the proxy has the fragments it needs, it calls on PyECLib to decode the data"},{"line_number":127,"context_line":"#. The proxy streams the decoded data it has back to the client"},{"line_number":128,"context_line":"#. Repeat until the proxy is done sending data back to the client"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_ee63d9ae","line":125,"in_reply_to":"baada198_5c73ad15","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"bde2e98ca6e468d18b4e636e40ba50ca43bf478c","unresolved":false,"context_lines":[{"line_number":138,"context_line":"explosion in the number of actual files stored in each node\u0027s local file system.  Although"},{"line_number":139,"context_line":"it is true that more files will be stored (because an object is broken into pieces), the"},{"line_number":140,"context_line":"implementation works to minimize this where possible, more details are available in the"},{"line_number":141,"context_line":"Under the Hood section."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"--------------"},{"line_number":144,"context_line":"Reconstruction"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_e6e88b18","line":141,"updated":"2014-07-17 17:42:50.000000000","message":"add a section, even just a placeholder on handoffs","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":138,"context_line":"explosion in the number of actual files stored in each node\u0027s local file system.  Although"},{"line_number":139,"context_line":"it is true that more files will be stored (because an object is broken into pieces), the"},{"line_number":140,"context_line":"implementation works to minimize this where possible, more details are available in the"},{"line_number":141,"context_line":"Under the Hood section."},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"--------------"},{"line_number":144,"context_line":"Reconstruction"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_4ef925c7","line":141,"in_reply_to":"baada198_e6e88b18","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":142,"context_line":""},{"line_number":143,"context_line":"--------------"},{"line_number":144,"context_line":"Reconstruction"},{"line_number":145,"context_line":"--------------"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"The basic framework of reconstruction is very similar to that of replication with a"},{"line_number":148,"context_line":"few notable exceptions:"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_972b5eea","line":145,"updated":"2014-07-18 10:27:02.000000000","message":"So if I am understanding correctly, replication is replaced by reconstruction? If so, maybe a sentence to make that clear.","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":142,"context_line":""},{"line_number":143,"context_line":"--------------"},{"line_number":144,"context_line":"Reconstruction"},{"line_number":145,"context_line":"--------------"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"The basic framework of reconstruction is very similar to that of replication with a"},{"line_number":148,"context_line":"few notable exceptions:"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_2e115104","line":145,"in_reply_to":"baada198_972b5eea","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":148,"context_line":"few notable exceptions:"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* Because EC does not actually replicate partitions, it needs to operate at a finer granularity"},{"line_number":151,"context_line":"* Because EC does actually replicate partitions, it needs to operate at a finer granularity than what is provided with rsync, therefore using an EC policy leverages much of ssync for the  than what is provided with rsync, therefore using an EC policy requires the use of ssync basic mechanism of discovering what work needs to be done."},{"line_number":152,"context_line":"* Once a pair of nodes has determined the need to replace a missing object fragment, instead of pushing over a copy, like replication would do, the reconstructor has to read in enough surviving fragments from other nodes and perform a local reconstruction before it has the correct data to push to the other node."},{"line_number":153,"context_line":""},{"line_number":154,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_575e2614","line":151,"updated":"2014-07-18 10:27:02.000000000","message":"mangled text in 5 lines above","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":148,"context_line":"few notable exceptions:"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"* Because EC does not actually replicate partitions, it needs to operate at a finer granularity"},{"line_number":151,"context_line":"* Because EC does actually replicate partitions, it needs to operate at a finer granularity than what is provided with rsync, therefore using an EC policy leverages much of ssync for the  than what is provided with rsync, therefore using an EC policy requires the use of ssync basic mechanism of discovering what work needs to be done."},{"line_number":152,"context_line":"* Once a pair of nodes has determined the need to replace a missing object fragment, instead of pushing over a copy, like replication would do, the reconstructor has to read in enough surviving fragments from other nodes and perform a local reconstruction before it has the correct data to push to the other node."},{"line_number":153,"context_line":""},{"line_number":154,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_ae42c1a8","line":151,"in_reply_to":"baada198_575e2614","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":169,"context_line":"----------------------------"},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"To use an EC policy, the administrator simply needs to define an EC policy and setup the"},{"line_number":172,"context_line":"associated object ring.  An example of how an EC policy can be setup is shown below::"},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"        [storage-policy:2]"},{"line_number":175,"context_line":"        name \u003d deepfreeze10-4"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_97e99e1c","line":172,"updated":"2014-07-18 10:27:02.000000000","message":"s/setup/setup in swift.conf/","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":169,"context_line":"----------------------------"},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"To use an EC policy, the administrator simply needs to define an EC policy and setup the"},{"line_number":172,"context_line":"associated object ring.  An example of how an EC policy can be setup is shown below::"},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"        [storage-policy:2]"},{"line_number":175,"context_line":"        name \u003d deepfreeze10-4"}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_d7b7811f","line":172,"in_reply_to":"baada198_97e99e1c","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":182,"context_line":""},{"line_number":183,"context_line":"* name: this is a standard storage policy parameter. See :doc:`overview_policies` for details."},{"line_number":184,"context_line":"* type: set this to \u0027erasure_coding\u0027 to indicate that this is an EC policy"},{"line_number":185,"context_line":"* ec_type: set this value according to the available options in selected the PyECLib back-end. This specifies the EC scheme that is to be used.  For example the option shown here selects Vandermonde Reed-Solomon encoding while an option of \u0027flat_xor_3\u0027 would select Flat-XOR based HD combination codes.  See the `PyECLib \u003chttps://bitbucket.org/kmgreen2/pyeclib\u003e`_ page for full details."},{"line_number":186,"context_line":"* ec_num_data_fragments:  the total number of fragments that will be comprised of data"},{"line_number":187,"context_line":"* ec_num_parity_fragments:  the total number of fragments that will be comprised of parity"},{"line_number":188,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_77b7aa21","line":185,"updated":"2014-07-18 10:27:02.000000000","message":"typo \u0027in selected the\u0027","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":182,"context_line":""},{"line_number":183,"context_line":"* name: this is a standard storage policy parameter. See :doc:`overview_policies` for details."},{"line_number":184,"context_line":"* type: set this to \u0027erasure_coding\u0027 to indicate that this is an EC policy"},{"line_number":185,"context_line":"* ec_type: set this value according to the available options in selected the PyECLib back-end. This specifies the EC scheme that is to be used.  For example the option shown here selects Vandermonde Reed-Solomon encoding while an option of \u0027flat_xor_3\u0027 would select Flat-XOR based HD combination codes.  See the `PyECLib \u003chttps://bitbucket.org/kmgreen2/pyeclib\u003e`_ page for full details."},{"line_number":186,"context_line":"* ec_num_data_fragments:  the total number of fragments that will be comprised of data"},{"line_number":187,"context_line":"* ec_num_parity_fragments:  the total number of fragments that will be comprised of parity"},{"line_number":188,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_b7512ddb","line":185,"in_reply_to":"baada198_77b7aa21","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":233,"context_line":"SLO/DLO middleware, the segments of your large object will be erasure coded (if being stored"},{"line_number":234,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":235,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":236,"context_line":"the actual object, not the locations of all of its fragments. The fact that an object is the"},{"line_number":237,"context_line":"actual object, not the locations of all of its fragments.  The fact that an object is actually"},{"line_number":238,"context_line":"fragmented within the cluster is handled at the proxy, middleware is unaware of what\u0027s happening"},{"line_number":239,"context_line":"behind the scenes."}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_9788be23","line":236,"updated":"2014-07-18 10:27:02.000000000","message":"won\u0027t list_endpoints return list of all 14 archives, equivalent to the 3 replica locations?","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":233,"context_line":"SLO/DLO middleware, the segments of your large object will be erasure coded (if being stored"},{"line_number":234,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":235,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":236,"context_line":"the actual object, not the locations of all of its fragments. The fact that an object is the"},{"line_number":237,"context_line":"actual object, not the locations of all of its fragments.  The fact that an object is actually"},{"line_number":238,"context_line":"fragmented within the cluster is handled at the proxy, middleware is unaware of what\u0027s happening"},{"line_number":239,"context_line":"behind the scenes."}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_5728d132","line":236,"in_reply_to":"baada198_9788be23","updated":"2014-07-18 19:41:22.000000000","message":"yeah, I\u0027m going to remove all this for now because what we do in LE needs a bit more discussion, not quite ready for the user docs just yet.  Will discuss in the EC spec, thanks","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07a735fea21e5c5914496d86937d5b6795c57215","unresolved":false,"context_lines":[{"line_number":234,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":235,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":236,"context_line":"the actual object, not the locations of all of its fragments. The fact that an object is the"},{"line_number":237,"context_line":"actual object, not the locations of all of its fragments.  The fact that an object is actually"},{"line_number":238,"context_line":"fragmented within the cluster is handled at the proxy, middleware is unaware of what\u0027s happening"},{"line_number":239,"context_line":"behind the scenes."},{"line_number":240,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_57c4665f","line":237,"updated":"2014-07-18 10:27:02.000000000","message":"I couldn\u0027t makes sense of the sentence \u0027The fact that an object is the\nactual object, not the locations of all of its fragments.\u0027","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"694745c061b0f11dedb0246721cd0df14fcd5dd6","unresolved":false,"context_lines":[{"line_number":234,"context_line":"in an EC policy container of course) in the same manner as any other object would be.  The"},{"line_number":235,"context_line":"same thing goes for middleware like List Endpoints, you will get the endpoint information for"},{"line_number":236,"context_line":"the actual object, not the locations of all of its fragments. The fact that an object is the"},{"line_number":237,"context_line":"actual object, not the locations of all of its fragments.  The fact that an object is actually"},{"line_number":238,"context_line":"fragmented within the cluster is handled at the proxy, middleware is unaware of what\u0027s happening"},{"line_number":239,"context_line":"behind the scenes."},{"line_number":240,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"baada198_77207549","line":237,"in_reply_to":"baada198_57c4665f","updated":"2014-07-18 19:41:22.000000000","message":"Done","commit_id":"afa53499c48007ca4e6c0d2f1340bfd2284cf2d2"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"ed3b94ba6973a0b0057c16c0104008423c69e834","unresolved":false,"context_lines":[{"line_number":121,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":122,"context_line":"#. Repeat until the client is done sending data"},{"line_number":123,"context_line":"#. The client is notified of completion when a quorum is met (configurable)."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"The GET flow looks like this:"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"#. The proxy server makes simultaneous requests to some number of participating nodes"}],"source_content_type":"text/x-rst","patch_set":9,"id":"baada198_c7a848dc","line":124,"updated":"2014-07-22 09:29:12.000000000","message":"With respect to PUT. I am curious how you will solve the object length issues. Currently the storage  nodes updates the container DB with the correct object length after it completed reading the data from the socket. In case of EC, what component will update the original object length? \nThe same about Etag. You told me once that you will cover it.\nIs there any spec or documentation with the proposed solution for this? I am curious how you resolved it.","commit_id":"fbbf775058bc410a26a37270718994bcfe9ca168"},{"author":{"_account_id":7479,"name":"paul luse","email":"paul.e.luse@intel.com","username":"peluse"},"change_message_id":"3623f6029c90fc3ea27992ca0673ce3374f3f400","unresolved":false,"context_lines":[{"line_number":121,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":122,"context_line":"#. Repeat until the client is done sending data"},{"line_number":123,"context_line":"#. The client is notified of completion when a quorum is met (configurable)."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"The GET flow looks like this:"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"#. The proxy server makes simultaneous requests to some number of participating nodes"}],"source_content_type":"text/x-rst","patch_set":9,"id":"baada198_e5799d83","line":124,"in_reply_to":"baada198_c7a848dc","updated":"2014-07-22 21:23:56.000000000","message":"Gil:  these are just the user docs, for details like this please refer to either https://review.openstack.org/#/c/106103/ or to Trello at https://trello.com/b/LlvIFIQs/swift-erasure-codes\n\nYou won\u0027t necessarily find all the answers to all your questions, but those are the places to ask to make sure we cover all the bases in the design.  Thanks!!!","commit_id":"fbbf775058bc410a26a37270718994bcfe9ca168"},{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"eecaaff779ac83a8bdc176c841059cbae98202b8","unresolved":false,"context_lines":[{"line_number":121,"context_line":"#. The proxy streams the encoded fragments out to the storage nodes based on ring locations"},{"line_number":122,"context_line":"#. Repeat until the client is done sending data"},{"line_number":123,"context_line":"#. The client is notified of completion when a quorum is met (configurable)."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"The GET flow looks like this:"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"#. The proxy server makes simultaneous requests to some number of participating nodes"}],"source_content_type":"text/x-rst","patch_set":9,"id":"baada198_340d9b36","line":124,"in_reply_to":"baada198_c7a848dc","updated":"2014-07-22 14:51:49.000000000","message":"The original object length and etag will be calculated by the proxy and sent to the object server via a \"trailer\" at the end of the data stream.  I will propose a patch for generic trailer support soon (payload trailer support would be useful in other use cases, such as encryption).  The container update will still be done at the object server.","commit_id":"fbbf775058bc410a26a37270718994bcfe9ca168"}]}