)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"844f63994ec2a2b92648265167dbf421a32ecb5c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"151e11de_5b60d74f","updated":"2026-04-13 17:33:15.000000000","message":"Interesting... I want to make sure I understand the idea though.","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"29529b3ebcac40c02f7d6f673a79b0b924e6afe7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"fb2e583d_fc08ef37","updated":"2026-04-29 23:06:18.000000000","message":"Did an initial read-through -- excited to see how it goes!","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"01d7c698fab58dcbcafe214fdbe44e40ed83951e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"98557050_0ea008ee","updated":"2026-04-27 07:54:56.000000000","message":"Step 1 Prepare encoding matrixes, done\nStep 2 enccoding\nstep 3: decoding\nstep 4 reconstruct","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"d57199c59c868934bf2466c068fc5032a11750dc","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"a150c81d_6c63ccee","updated":"2026-05-02 16:50:00.000000000","message":"Encoding is ongoing, it passes the iterations over v, but need to check that global parity is ok .\nNeed to add local paratiy.\nTests done using : \ngdb --args ./test/.libs/liberasurecode_rs_isal_stress_test 11 20 3 0 1","commit_id":"3c709c156caaed26c26b22f19da47fa3334ec4cd"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"851b35965f332f76219cb733390fb2e990c959b4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"5f6ed834_f16ed914","updated":"2026-05-05 20:22:41.000000000","message":"Encoding is ongoing but not yet validated, discovered a method in libisal that updates parities so sweeted for our needs)","commit_id":"5c27bd95625ade53d29b2aa4fb464c3476a2d8b6"}],"higher-lrc.md":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"844f63994ec2a2b92648265167dbf421a32ecb5c","unresolved":true,"context_lines":[{"line_number":1,"context_line":"# Higher LRC"},{"line_number":2,"context_line":""},{"line_number":3,"context_line":"Using large values of `k` and `m` for RS or LRC is complex in practice.  "},{"line_number":4,"context_line":"Two main issues appear:"},{"line_number":5,"context_line":""},{"line_number":6,"context_line":"1. **High socket fan-out**: reconstruction may require up to `k` parallel sockets."}],"source_content_type":"text/x-markdown","patch_set":1,"id":"a3aa3c4d_e48e9552","line":3,"updated":"2026-04-13 17:33:15.000000000","message":"I think it\u0027s worth fleshing out *why* you might want larger values of `k` and `m` -- it is mainly about reducing storage overhead with something like a `120+6` policy like you\u0027d called out on https://review.opendev.org/c/openstack/liberasurecode/+/959280? Or maybe increasing performance by getting more drives involved?\n\nPart of me wonders if it would be better to architect the storage system to have two encodings. Like, a client-facing API server that encodes with a 16+1 policy and sends frags to 17 storage servers, which in turn uses a 32+1 policy (or something; gets limited by your disks-per-server) to distribute across disks within the server. Then you can still withstand the loss of a whole server, and within a server it\u0027s easy to rebuild given the loss of a drive. Even losing multiple drives within a server should be OK as long as the other servers as OK.\n\n**Or**... make sure API\u003c-\u003estorage server communication uses HTTP/2 so we can store/retrieve multiple frags from multiple disks over the one socket... hmmm 🤔","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"00aaeec715ded5327ab69094155a692870319481","unresolved":true,"context_lines":[{"line_number":1,"context_line":"# Higher LRC"},{"line_number":2,"context_line":""},{"line_number":3,"context_line":"Using large values of `k` and `m` for RS or LRC is complex in practice.  "},{"line_number":4,"context_line":"Two main issues appear:"},{"line_number":5,"context_line":""},{"line_number":6,"context_line":"1. **High socket fan-out**: reconstruction may require up to `k` parallel sockets."}],"source_content_type":"text/x-markdown","patch_set":1,"id":"287afcd3_3d0e89d1","line":3,"in_reply_to":"52e20fe0_52155012","updated":"2026-04-14 01:04:14.000000000","message":"\u003e I don\u0027t know about the use of HTTP2 to get that but this is interesting.\n\nMainly just trying to think of ways to multiplex the connection.","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"963d784ba2f83996035f6ff9f72abbc35e8b67d9","unresolved":true,"context_lines":[{"line_number":1,"context_line":"# Higher LRC"},{"line_number":2,"context_line":""},{"line_number":3,"context_line":"Using large values of `k` and `m` for RS or LRC is complex in practice.  "},{"line_number":4,"context_line":"Two main issues appear:"},{"line_number":5,"context_line":""},{"line_number":6,"context_line":"1. **High socket fan-out**: reconstruction may require up to `k` parallel sockets."}],"source_content_type":"text/x-markdown","patch_set":1,"id":"52e20fe0_52155012","line":3,"in_reply_to":"a3aa3c4d_e48e9552","updated":"2026-04-13 18:13:37.000000000","message":"Yes the idea is to go for large factor and get same protection for a little overhead. \nI don\u0027t know about the use of HTTP2 to get that but this is interesting.\nI didnt understand well you scheme about 16+1 but in my opinion it offers less protection (data is less protected in time window before moving to another policy)","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"844f63994ec2a2b92648265167dbf421a32ecb5c","unresolved":true,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"The current chunking path is stream-oriented.  "},{"line_number":12,"context_line":"For each buffer, data is split into `k` chunks, and each chunk is written immediately.  "},{"line_number":13,"context_line":"Because data is striped across `k` fragments per step, reads often need `k` sources."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"## Current encoding scheme"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-markdown","patch_set":1,"id":"0886fe32_230c7bdb","line":13,"range":{"start_line":13,"start_character":55,"end_line":13,"end_character":83},"updated":"2026-04-13 17:33:15.000000000","message":"By \"read\" do you mean \"decode\"? Won\u0027t that *always* require `k` sources?","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"963d784ba2f83996035f6ff9f72abbc35e8b67d9","unresolved":true,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"The current chunking path is stream-oriented.  "},{"line_number":12,"context_line":"For each buffer, data is split into `k` chunks, and each chunk is written immediately.  "},{"line_number":13,"context_line":"Because data is striped across `k` fragments per step, reads often need `k` sources."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"## Current encoding scheme"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-markdown","patch_set":1,"id":"024c279c_2aa06eee","line":13,"range":{"start_line":13,"start_character":55,"end_line":13,"end_character":83},"in_reply_to":"0886fe32_230c7bdb","updated":"2026-04-13 18:13:37.000000000","message":"Yes decoding needs k source even we use pass through mode. and could lead to heavy resource usage (as we open k connections at time).","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"844f63994ec2a2b92648265167dbf421a32ecb5c","unresolved":true,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":"## Proposed idea: post-upload re-encoding with hierarchical chunking"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"After upload, re-encode the object into a higher storage policy (for example, larger `k`) when full object size is known."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Let:"},{"line_number":31,"context_line":""}],"source_content_type":"text/x-markdown","patch_set":1,"id":"80108979_d5ff4cdd","line":28,"updated":"2026-04-13 17:33:15.000000000","message":"OK, so the proposal is mainly about streamlining a re-encode from a `u+?` to `k+?` policy, is that right?\n\nThe storage system will be responsible for doing the re-encode, though, yeah? So from liberasurecode\u0027s perspective, it\u0027s mainly about\n\n1. providing APIs to split a fragment into `v` smaller fragments,\n2. being able to reconstruct/decode with a mix of large and small fragments, and\n2. doing something sensible with parities.\n\nWill the smaller fragments generally still live on the same server, or get distributed elsewhere? How will we know how many fragments to look for on decode?","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"963d784ba2f83996035f6ff9f72abbc35e8b67d9","unresolved":true,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":"## Proposed idea: post-upload re-encoding with hierarchical chunking"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"After upload, re-encode the object into a higher storage policy (for example, larger `k`) when full object size is known."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Let:"},{"line_number":31,"context_line":""}],"source_content_type":"text/x-markdown","patch_set":1,"id":"d18fd4dd_c8dec3a0","line":28,"in_reply_to":"80108979_d5ff4cdd","updated":"2026-04-13 18:13:37.000000000","message":"Fragment will have same length, except we reorganize data in order to have at each time only u connections at time.\n\nTo give example with k \u003d 60 ,m \u003d 4\nCurrently we work with a buffer of size let say 1MB (not sure of value and correct me if I am wrong)\n\nwe read data and divide it into k small parts then each part gos into corresponding chunk, parity go to parity chunks.\nWe do this process further and further until we end the stream input.\n\n\nThis means that there is an interleave in data contained in a chunk:\n\nwe work by buffer that we divide into k until the whole object is done\n\nProposed schema:\nu \u003d 6\nv\u003d 10\n\nWe know in advance the object size and object is already encoded in repli x 3 for example), so wil will work as if the object is divided into v parts.\nFor each part will do a classical encoding (RS or LRC) rs(u,m), we generate m parities.\nOnce one part is finished we move to next part , but the input here will be the part + global parities, so we will update the global parities.\nFor reading we will use only pass_throuhg mode.\nNow when reading, we will only the first u chunks and when done we move to next u chunks and so on.\n\nMain idea is that global parities are like accumulators and encoding matrix change at each new part.","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"d79332628d5a3e9230bb44965566701269dc81ff","unresolved":true,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":"## Proposed idea: post-upload re-encoding with hierarchical chunking"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"After upload, re-encode the object into a higher storage policy (for example, larger `k`) when full object size is known."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Let:"},{"line_number":31,"context_line":""}],"source_content_type":"text/x-markdown","patch_set":1,"id":"9b6043d4_0131751c","line":28,"in_reply_to":"9eb86f8b_b9a0d8a4","updated":"2026-04-14 09:49:58.000000000","message":"I agree with your description for current behavior in swift and our backend aslo.\n\nYour description applies for any RS(k,m).:\n\nWhat happens if k is high is that we need to contact k servers (disks) , this generate high pressure but also high speed of data.\nIn you described schema, we agree that one fragment doesn\u0027t contain a consecutive part of object.\n\nNow a simple idea would be for a whole object of length n:\nobj\u003dxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nobj is split into k chunks , each of size n//k + for example (add padding etc)\nwe obtain k chunks and each one is a consecutive bytes of original object.\n\nchunk1: xxxxxxxxxxxxxxxxxxxx\nchunk2: xxxxxxxxxxxxxxxxxxxx\nchunkk : xxxxxxxxxxxxxxxxxxxx\nwe conmpute parities and now for reading object we can stream chunk1 and when finished stream chunk2 and so on. (There is no mixing)\n\nThis idea is simple but could have low throughput as we read one disk at time.\nThis doesn\u0027t need any change from liberasure, it could be done on our backend side.\n\nHow to read from some disk at a time ? \nif k is higher we don\u0027t want to read from them in parallel but we want to read from some of them at a time, there where we introduce k \u003d u .v and reencode into rs(k,m).\n\nDuring reeconde:\nwe split whole object into v fragments:\nwe obtain v chunks and each one is a consecutive bytes of original object.\n\nchunk1: xxxxxxxxxxxxxxxxxxxx\nchunk2: xxxxxxxxxxxxxxxxxxxx\nchunkv : xxxxxxxxxxxxxxxxxxxx\n\n1) we process chunk1 first in a policy rs(u,m), obtain u chunks + m parities\n2) During processing of chunk2 in a rs(u,m) policy we will read m old parities and update them. Note that we will change encoding matrix here\n3) We repeat step 2 from all v chunks and obtained parities are the final ones\n\nthe final result is k chunks  and m global parities.\n\nRead:\nWhen reading we know that first u chunks contain first part of object (we can use data path), when we finish from first u chunsk we move to second group of u chunks and so on.","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"00aaeec715ded5327ab69094155a692870319481","unresolved":true,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":"## Proposed idea: post-upload re-encoding with hierarchical chunking"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"After upload, re-encode the object into a higher storage policy (for example, larger `k`) when full object size is known."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Let:"},{"line_number":31,"context_line":""}],"source_content_type":"text/x-markdown","patch_set":1,"id":"9eb86f8b_b9a0d8a4","line":28,"in_reply_to":"d18fd4dd_c8dec3a0","updated":"2026-04-14 01:04:14.000000000","message":"OK, I\u0027m still trying to understand. FWIW, my context is pretty much entirely Swift, which works like this:\n\n- Client starts sending some object data, say 1GiB worth\n- In a loop, the API server\n  - reads 1MiB from the client\n  - encodes it according to some scheme -- say, RS(8, 4) so we get 12 fragments of ~128KiB (+80 bytes for liberasurecode\u0027s fragment header) each\n  - streams to a storage server\n- Storage server writes down the 1024 concatenated fragments (a fragment archive) as a single ~128MB file on an individual disk\n- If any part of a fragment archive becomes corrupted, we delete (well, quarantine, but nearly the same effect) the whole thing and rebuild using 8 good fragment archives, reconstructing one fragment at a time -- very similar to what happens if there\u0027s a total disk failure\n- For any read, we contact storage servers until we get back 8 responses and take one fragment\u0027s worth from each at a time to decode and send to the client\n\nIt sounds like you\u0027re proposing doing something like:\n\n- Client starts sending some object data, say 1GiB worth\n- In a loop, the API server\n  - reads 1MiB buffer from the client\n  - breaks buffer into 10 parts, each ~102KiB\n\n... but then I\u0027ve lost the thread. There\u0027s something involving encoding each part according to RS(6, 4) to get 10 fragments of ~17KiB each and combining the resulting parity frags -- but I\u0027m still trying to work out the details. Things like\n\n- How are the parities combined? Like, repeatedly xor? Or is it something more complicated?\n- How do we use these combined parities to decode in the face of failures?\n- How many individual storage servers get written to? Seems like it must be \u003ck if we\u0027re trying to avoid the high fan-out ...\n- How much gets written to each storage server? Does it vary depending on whether it\u0027s receiving data fragments or parity fragments?\n- Does the storage server write to a single disk, or does it split up the stream across multiple?","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"844f63994ec2a2b92648265167dbf421a32ecb5c","unresolved":true,"context_lines":[{"line_number":36,"context_line":"1. Split the full object into `u` large segments."},{"line_number":37,"context_line":"2. For each segment, process buffers and split each into `v` chunks."},{"line_number":38,"context_line":"3. Encode segment-by-segment (`segment1`, `segment2`, ..., `segmentu`)."},{"line_number":39,"context_line":"4. Maintain accumulators during segment encoding to compute **global parity** across all segments."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"## Potential advantages"},{"line_number":42,"context_line":""}],"source_content_type":"text/x-markdown","patch_set":1,"id":"4099307f_43ef08d9","line":39,"updated":"2026-04-13 17:33:15.000000000","message":"So what\u0027s the plan for parities in all this? Will they be large or small fragments? If large, will they be able to be split like the data frags?","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"844f63994ec2a2b92648265167dbf421a32ecb5c","unresolved":true,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"## Recommendation"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Start as an **optional mode in LRC** to validate behavior and performance quickly.  "},{"line_number":58,"context_line":"If it proves stable and broadly useful (including RS), extract it into a **separate backend** with shared primitives."}],"source_content_type":"text/x-markdown","patch_set":1,"id":"f621dbcb_3149274f","line":57,"range":{"start_line":57,"start_character":40,"end_line":57,"end_character":81},"updated":"2026-04-13 17:33:15.000000000","message":"Big fan of this to determine viability -- to what degree do we need to get anything merged upstream to enable this validation?","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"963d784ba2f83996035f6ff9f72abbc35e8b67d9","unresolved":true,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"## Recommendation"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Start as an **optional mode in LRC** to validate behavior and performance quickly.  "},{"line_number":58,"context_line":"If it proves stable and broadly useful (including RS), extract it into a **separate backend** with shared primitives."}],"source_content_type":"text/x-markdown","patch_set":1,"id":"35a9af7e_7012726c","line":57,"range":{"start_line":57,"start_character":40,"end_line":57,"end_character":81},"in_reply_to":"f621dbcb_3149274f","updated":"2026-04-13 18:13:37.000000000","message":"Ok if plan is interesting, we could do as option of LRC","commit_id":"fd29982e1eacbc039e29122dcbf50591baa9630e"}],"src/backends/isa-l/isa_l_rs_vand_iter.c":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"29529b3ebcac40c02f7d6f673a79b0b924e6afe7","unresolved":false,"context_lines":[{"line_number":122,"context_line":"done:"},{"line_number":123,"context_line":"    free(V_inv_k);"},{"line_number":124,"context_line":"    free(V);"},{"line_number":125,"context_line":"    return ret;"},{"line_number":126,"context_line":"}"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"/*"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"15ad325f_e16fb30f","line":125,"updated":"2026-04-29 23:06:18.000000000","message":"Right; between `build_vandermonde_system` and `build_global_matrix`, it looks a lot like `isa_l_rs_vand_inv`\u0027s `gen_encoding_matrix` -- good 👍","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"6ee168881212dd525d450cc3dcc47a20212c978d","unresolved":false,"context_lines":[{"line_number":122,"context_line":"done:"},{"line_number":123,"context_line":"    free(V_inv_k);"},{"line_number":124,"context_line":"    free(V);"},{"line_number":125,"context_line":"    return ret;"},{"line_number":126,"context_line":"}"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"/*"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"cad57eb3_34692a11","line":125,"in_reply_to":"15ad325f_e16fb30f","updated":"2026-04-30 12:15:11.000000000","message":"Yes we could renamed them gen_encoding matrix but it is to show that there is a big matrix that can be usufell at any moment and small matrixes to use","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"29529b3ebcac40c02f7d6f673a79b0b924e6afe7","unresolved":true,"context_lines":[{"line_number":139,"context_line":" *   to the current iteration, both for the data (identity block) and"},{"line_number":140,"context_line":" *   the parity rows."},{"line_number":141,"context_line":" */"},{"line_number":142,"context_line":"static int refresh_iter_matrix(isa_l_iter_desc *id, int iter)"},{"line_number":143,"context_line":"{"},{"line_number":144,"context_line":"    if (iter \u003c 1 || iter \u003e id-\u003ev)"},{"line_number":145,"context_line":"        return -1;"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"ac940c42_579e0347","line":142,"updated":"2026-04-29 23:06:18.000000000","message":"And *this* is where the new magic lives, right? I need to look through it a little more carefully...","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"6ee168881212dd525d450cc3dcc47a20212c978d","unresolved":true,"context_lines":[{"line_number":139,"context_line":" *   to the current iteration, both for the data (identity block) and"},{"line_number":140,"context_line":" *   the parity rows."},{"line_number":141,"context_line":" */"},{"line_number":142,"context_line":"static int refresh_iter_matrix(isa_l_iter_desc *id, int iter)"},{"line_number":143,"context_line":"{"},{"line_number":144,"context_line":"    if (iter \u003c 1 || iter \u003e id-\u003ev)"},{"line_number":145,"context_line":"        return -1;"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"84d4777d_72fc8afa","line":142,"in_reply_to":"ac940c42_579e0347","updated":"2026-04-30 12:15:11.000000000","message":"right, it is as if a big matrix is decomposed of small matrixes","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"29529b3ebcac40c02f7d6f673a79b0b924e6afe7","unresolved":true,"context_lines":[{"line_number":186,"context_line":" *   data   : array of u data fragment pointers (segment for this iter)"},{"line_number":187,"context_line":" *   parity : array of m parity fragment pointers (accumulated)"},{"line_number":188,"context_line":" *   len    : byte length of each fragment"},{"line_number":189,"context_line":" *   iter   : current iteration index (1-based)"},{"line_number":190,"context_line":" *"},{"line_number":191,"context_line":" *   The caller drives the loop over iter \u003d 1..v, passing the"},{"line_number":192,"context_line":" *   appropriate data slice each time.  Parity buffers are XOR-"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"29b93798_f736c724","line":189,"updated":"2026-04-29 23:06:18.000000000","message":"This gets pulled off `desc_in` (which should probably be documented)\n\nBut the caller is still responsible for changing it (incrementing it?) between calls, yeah? How\u0027s it going to get at it when `liberasurecode_instance_create` is handing out integer instance descriptors rather than instance structs?\n\nOr is the caller supposed to create `1..v` instances, each with a different `iter`, and rotate through them?\n\nWould it make sense for the caller to pass larger buffers (all `1..v` slices that would get passed currently), and have liberasurecode be responsible for managing `iter`? Or is it that we\u0027re trying to keep the caller from needing to have all `k` data frags in their head at once?\n\nAre the intermediate parity values useful/interesting? Or will the caller ultimately only really care about the XOR-accumulated values after processing all `1..v` slices?","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":38169,"name":"Lhoussain AIT ASSOU","display_name":"aitassou","email":"lhoussain.ait-assou@ovhcloud.com","username":"aitassou"},"change_message_id":"6ee168881212dd525d450cc3dcc47a20212c978d","unresolved":true,"context_lines":[{"line_number":186,"context_line":" *   data   : array of u data fragment pointers (segment for this iter)"},{"line_number":187,"context_line":" *   parity : array of m parity fragment pointers (accumulated)"},{"line_number":188,"context_line":" *   len    : byte length of each fragment"},{"line_number":189,"context_line":" *   iter   : current iteration index (1-based)"},{"line_number":190,"context_line":" *"},{"line_number":191,"context_line":" *   The caller drives the loop over iter \u003d 1..v, passing the"},{"line_number":192,"context_line":" *   appropriate data slice each time.  Parity buffers are XOR-"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"5e7d9e56_ade487c9","line":189,"in_reply_to":"29b93798_f736c724","updated":"2026-04-30 12:15:11.000000000","message":"Good questions, for the moment I put some iter info in liberasurecode_instance_create instance, so to change iter we need to recreate instance.\n\"Or is it that we\u0027re trying to keep the caller from needing to have all k data frags in their head at once \" Yes right\nI had another choice is to put iter un enccoding intferace but this breaks interface.\nSo in next iterations we will create an api method set_iteration in order to have one instance but to change iteration.\n\"Are the intermediate parity values useful/interesting\" intermediate value are not useful, only final parities matter.\nBut I have idea to have lrc_flag that make intermediate partity as local parity for a subgroup. The reason for that is to have some diversity during decoding otherwise we need to do only pass_through. \nExammple of values to show this:  k \u003d 140, u\u003d 20, v \u003d 7, l \u003d 7, m \u003d 7 + 3, overhead \u003d 1.07","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"29529b3ebcac40c02f7d6f673a79b0b924e6afe7","unresolved":false,"context_lines":[{"line_number":262,"context_line":"    id-\u003ebase.field \u003d sym.type; \\"},{"line_number":263,"context_line":"    if (!id-\u003ebase.field) goto fail;"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"    LOAD_SYM(ec_encode_data,   \"ec_encode_data\",   enc)"},{"line_number":266,"context_line":"    LOAD_SYM(ec_init_tables,   \"ec_init_tables\",   init)"},{"line_number":267,"context_line":"    LOAD_SYM(gf_invert_matrix, \"gf_invert_matrix\", inv)"},{"line_number":268,"context_line":"    LOAD_SYM(gf_mul,           \"gf_mul\",           mul)"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"7593bd06_a63db51b","line":265,"updated":"2026-04-29 23:06:18.000000000","message":"Ooh, I think I like this idiom -- maybe I\u0027ll look at doing something similar for the other backends...","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"29529b3ebcac40c02f7d6f673a79b0b924e6afe7","unresolved":false,"context_lines":[{"line_number":323,"context_line":"    .EXIT             \u003d isa_l_rs_vand_iter_exit,"},{"line_number":324,"context_line":"    .ISSYSTEMATIC     \u003d 1,"},{"line_number":325,"context_line":"    .ENCODE           \u003d isa_l_rs_vand_iter_encode,          /* full-matrix path   */"},{"line_number":326,"context_line":"    .DECODE           \u003d isa_l_decode,"},{"line_number":327,"context_line":"    .FRAGSNEEDED      \u003d isa_l_min_fragments,"},{"line_number":328,"context_line":"    .RECONSTRUCT      \u003d isa_l_reconstruct,"},{"line_number":329,"context_line":"    .ELEMENTSIZE      \u003d isa_l_element_size,"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"dba8d032_715e7c2e","line":326,"updated":"2026-04-29 23:06:18.000000000","message":"Right, so this\u0027ll change in step 3, presumably.","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"29529b3ebcac40c02f7d6f673a79b0b924e6afe7","unresolved":false,"context_lines":[{"line_number":325,"context_line":"    .ENCODE           \u003d isa_l_rs_vand_iter_encode,          /* full-matrix path   */"},{"line_number":326,"context_line":"    .DECODE           \u003d isa_l_decode,"},{"line_number":327,"context_line":"    .FRAGSNEEDED      \u003d isa_l_min_fragments,"},{"line_number":328,"context_line":"    .RECONSTRUCT      \u003d isa_l_reconstruct,"},{"line_number":329,"context_line":"    .ELEMENTSIZE      \u003d isa_l_element_size,"},{"line_number":330,"context_line":"    .ISCOMPATIBLEWITH \u003d isa_l_rs_vand_iter_is_compatible_with,"},{"line_number":331,"context_line":"    .GETMETADATASIZE  \u003d get_backend_metadata_size_zero,"}],"source_content_type":"text/x-csrc","patch_set":2,"id":"6a6acfc8_1b9eee5b","line":328,"updated":"2026-04-29 23:06:18.000000000","message":"And this in step 4.","commit_id":"0e25dc9f91af58ff6e68aab1a0578299820fda22"}]}