)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"594aafdc29f45c6c6a4a7e35454872c1c1a740d2","unresolved":true,"context_lines":[{"line_number":23,"context_line":"- Drop MD5 tracking in RingReader. It was brittle at best anyway, and"},{"line_number":24,"context_line":"  nothing uses it. YAGNI"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"  Drive-by: Fix size/raw_size attributes when loading only metadata."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"- Add the ability to seek within RingReaders, though you need to know"},{"line_number":29,"context_line":"  what you\u0027re doing and only seek to flush points."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":17,"id":"9e1c4bf1_15e92926","line":26,"range":{"start_line":26,"start_character":2,"end_line":26,"end_character":68},"updated":"2022-11-16 01:37:02.000000000","message":"maybe this drive by could be down the end fo the commit message. But that\u0027s definitely a NIT.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ce0562ff0eca4d9936c96ffb3ec8f2feb52a5c74","unresolved":true,"context_lines":[{"line_number":23,"context_line":"- Drop MD5 tracking in RingReader. It was brittle at best anyway, and"},{"line_number":24,"context_line":"  nothing uses it. YAGNI"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"  Drive-by: Fix size/raw_size attributes when loading only metadata."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"- Add the ability to seek within RingReaders, though you need to know"},{"line_number":29,"context_line":"  what you\u0027re doing and only seek to flush points."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":17,"id":"b4402d4c_0876a269","line":26,"range":{"start_line":26,"start_character":2,"end_line":26,"end_character":68},"in_reply_to":"9e1c4bf1_15e92926","updated":"2025-05-20 20:29:35.000000000","message":"Made this part of a broader `Clean up some ring metadata handling` bullet point (which is part of why it fit with the MD5 tracking in my mind)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":23,"context_line":"- Drop MD5 tracking in RingReader. It was brittle at best anyway, and"},{"line_number":24,"context_line":"  nothing uses it. YAGNI"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"  Drive-by: Fix size/raw_size attributes when loading only metadata."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"- Add the ability to seek within RingReaders, though you need to know"},{"line_number":29,"context_line":"  what you\u0027re doing and only seek to flush points."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":17,"id":"039383b4_2f06f65c","line":26,"range":{"start_line":26,"start_character":2,"end_line":26,"end_character":68},"in_reply_to":"b4402d4c_0876a269","updated":"2025-06-13 21:04:18.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"842e5ddd44f0ff7edc9c9b27fcb2b05bcd9608bc","unresolved":true,"context_lines":[{"line_number":28,"context_line":"- Add the ability to seek within RingReaders, though you need to know"},{"line_number":29,"context_line":"  what you\u0027re doing and only seek to flush points."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"- Let RingData and RingBuilder objects change how wide their"},{"line_number":32,"context_line":"  replica2part2dev_id arrays are. Add a dev_id_bytes key to serialized"},{"line_number":33,"context_line":"  ring metadata."},{"line_number":34,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":30,"id":"d251ed6b_995c108a","line":31,"range":{"start_line":31,"start_character":2,"end_line":31,"end_character":54},"updated":"2025-05-15 04:04:09.000000000","message":"Commit message needs updating; only `RingBuilder`s can do this now.","commit_id":"c2cdd7495b93cd1b7a7fccf90f4a636dfcdab183"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ce0562ff0eca4d9936c96ffb3ec8f2feb52a5c74","unresolved":false,"context_lines":[{"line_number":28,"context_line":"- Add the ability to seek within RingReaders, though you need to know"},{"line_number":29,"context_line":"  what you\u0027re doing and only seek to flush points."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"- Let RingData and RingBuilder objects change how wide their"},{"line_number":32,"context_line":"  replica2part2dev_id arrays are. Add a dev_id_bytes key to serialized"},{"line_number":33,"context_line":"  ring metadata."},{"line_number":34,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":30,"id":"3e0b78c3_85f12556","line":31,"range":{"start_line":31,"start_character":2,"end_line":31,"end_character":54},"in_reply_to":"d251ed6b_995c108a","updated":"2025-05-20 20:29:35.000000000","message":"Done","commit_id":"c2cdd7495b93cd1b7a7fccf90f4a636dfcdab183"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"842e5ddd44f0ff7edc9c9b27fcb2b05bcd9608bc","unresolved":true,"context_lines":[{"line_number":36,"context_line":"  2, and there really isn\u0027t any benefit in having rings with"},{"line_number":37,"context_line":"  dev_id_bytes of 1. Sure save some space for clusters with \u003c 256 devices."},{"line_number":38,"context_line":"  But 2 byte dev_ids have worked fine for years. Further, this makes"},{"line_number":39,"context_line":"  rollout and any potential rollback simpler."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"- swift-ring-builder version subcommand added, which takes a ring. This"},{"line_number":42,"context_line":"  enables Ops to see the serialization format of a ring on disk:"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":30,"id":"5da88679_675c27ed","line":39,"updated":"2025-05-15 04:04:09.000000000","message":"If I\u0027m working on the commit message anyway, I might clean this up, too. Probably combine it with the previous bullet and just talk about how `dev_id_bytes` can be either 2 or 4 -- even mentioning the possibility of 1 seems like it\u0027d just be confusing the issue.","commit_id":"c2cdd7495b93cd1b7a7fccf90f4a636dfcdab183"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ce0562ff0eca4d9936c96ffb3ec8f2feb52a5c74","unresolved":false,"context_lines":[{"line_number":36,"context_line":"  2, and there really isn\u0027t any benefit in having rings with"},{"line_number":37,"context_line":"  dev_id_bytes of 1. Sure save some space for clusters with \u003c 256 devices."},{"line_number":38,"context_line":"  But 2 byte dev_ids have worked fine for years. Further, this makes"},{"line_number":39,"context_line":"  rollout and any potential rollback simpler."},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"- swift-ring-builder version subcommand added, which takes a ring. This"},{"line_number":42,"context_line":"  enables Ops to see the serialization format of a ring on disk:"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":30,"id":"d0634ac0_2c92f60d","line":39,"in_reply_to":"5da88679_675c27ed","updated":"2025-05-20 20:29:35.000000000","message":"Done","commit_id":"c2cdd7495b93cd1b7a7fccf90f4a636dfcdab183"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c9b68e79237f7f7698652e1af6d54c43da7fdde1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"01cc5db0_a8c5f3e9","updated":"2022-03-20 23:33:01.000000000","message":"This is really awesome work Tim! Love seeing it all encapsulated, and great work working with zlib to make things work much cleaner.\n\nQuestion inline about why we\u0027re encoding the blob length when we have an index that can/could tell us this information.","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"f565d17f_c205fe2b","updated":"2022-04-23 05:36:19.000000000","message":"Still needs more tests, too.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b938e76532bc222772ace8612d57f2317d4df0cd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"33f732ff_70f8c5af","updated":"2022-04-23 01:06:14.000000000","message":"This is going to have some obvious flake8 issues, but I want to sanity-check that the rest of the tests pass when defaulting to v2.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4c93df14740d4387c6754da5c9ea5914282c0ebf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"adaf27df_4d908f53","updated":"2022-05-17 06:25:59.000000000","message":"Added a follow that adds a bunch more test coverage. At least a start to the areas you mentioned in this patchset https://review.opendev.org/c/openstack/swift/+/842040\n\nFeel free to sqaush it into this one.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"48cba41492652b5d76421ff3e931b62cf020ab8e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"4cf3e3d8_93560081","updated":"2022-05-17 06:32:42.000000000","message":"In adding some test coverage there are a few areas that needed to be fixed that it found.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"071c867e_95982de1","updated":"2022-05-13 06:40:27.000000000","message":"Need to take a kid to swimming lessions, so pushing up where I\u0027ve got to in the review. Will get back to this on Monday (or the weekend maybe?). ","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"357c90f3903834ee1797f4742e6ff026de707982","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"bffbc7eb_0cbf9fa5","updated":"2022-06-30 02:05:07.000000000","message":"Nice, I\u0027ve also added another test follow up which just covers it a little more: https://review.opendev.org/c/openstack/swift/+/848203 ","commit_id":"00bdb0a26cf52bce1305f86a617662e7bb696655"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"85ec3c0711637b260ad4df830a782c502b7b59ef","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"ed9b3875_b65026ad","updated":"2022-07-11 22:17:27.000000000","message":"Changes:\n\n- Can\u0027t nest sections\n- replica_count and dev_id_bytes are properly exposed when reading with metadata_only\u003dTrue","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"227d72b0c53ee36a1a6725586bcfeddd13995720","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"dee8e186_f3568e37","updated":"2022-07-20 10:24:42.000000000","message":"Love, some comments inline, then I think I\u0027m ready to land this one!","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"423efd73dd66e6db4a97d67aa5a0b8b15d86326f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"3664e073_603f3d1f","updated":"2022-07-25 07:34:28.000000000","message":"I really like it, esp the test deomonstration of using the index.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"af38442e_9c1a405d","updated":"2022-07-27 22:25:32.000000000","message":"I\u0027m still trying to understand everything going on here\n\nI\u0027m surprised/worried about the density/complexity of the code/comments in the new io module ... but i haven\u0027t looked at the test_io module yet: it might shine light into all the scary edges cases (but the test:code ratio seems light)","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"594aafdc29f45c6c6a4a7e35454872c1c1a740d2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"b4b221ac_7f10ca45","updated":"2022-11-16 01:37:02.000000000","message":"I still really like it. It\u0027s missing some Docs, but I added that to https://review.opendev.org/c/openstack/swift/+/864494 Feel free to sqaush it in or whatever.\n\nusing 13 and 18 as contants to make it more readable and the use of functions over lamdas in the codec map could make things more readable, but not a blocker in my opinion. So +2 from me!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"549ab580_c9687148","updated":"2023-03-29 00:40:00.000000000","message":"ok, I don\u0027t think I have anything bad to say - the on-disk blob format makes sense, the index structure is sufficiently abstract, it\u0027s all pretty well tested, there\u0027s some docstrings.  Is far as the \"public\" ring interface - it does all seem to be pretty well abstracted behind that new format_version kwarg, so I\u0027m sure swift deployers build systems could easily adopt.\n\nMost of my reservations are around the internal interfaces and various layers between file-systems and Builder and RingData objects.  Thinking about overseer/ring-server and unified-builder/ringv3 and all the cool sections we\u0027ll be adding in ringv2 it migth be useful to invest in polishing these abstractions and interfaces - we don\u0027t have to do that RIGHT NOW, but maybe there\u0027s a few things we\u0027ll agree we want to  can straighten out - I\u0027m looking forward to your thoughts when you get back!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3f069ba1d9a2ad8e270b40503f37ad0ae9a5806f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"3f7963e2_ef46852e","updated":"2023-07-26 00:03:30.000000000","message":"I think we already keep builders backwards-compat, i just need to check tests","commit_id":"1ff766bcff07c6a038a19518cb569418abe32cbd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"caaf9e65b8497cec6328d8c81a464d43c230fd56","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":21,"id":"e848d3ea_1c927eec","updated":"2025-01-08 19:52:26.000000000","message":"The main gist of my comments seemed to suggest that my last review came away feeling that the code worked but I was unhappy about the \"dual\" SeekableZlibReader/Writer vs RingReader/Writer nature of the new serialization classes.  I think my recommendation would be try to stay away from making the RingReader/Writer ever act like a pure seekable-zlib-reader/writer and provide *only* the higher-order \"write_section\" or \"write_index\" methods with \"no chance\" of someone getting \"under the hood\" and using \"_open_section\" or \"_write_blob\" \"incorrectly\".  But as long as the on-disk format doesn\u0027t have to change that could all theoretically be follow-up (if even necessary).\n\nI think the *main* feedback I most wanted to provide for getting this ready to merge was for the \"introduce ringv2\" patch let\u0027s try to keep \"improve ringv1\" changed limited to a prefactor - with the hope of highlighting if the primitive/interface/abstractions we want to add as a \"foundation\" for ring reading/writing really don\u0027t make sense as a clean-up/improvement to ringv1 on their own until you introduce ringv2; then maybe they aren\u0027t the right abstractions for maintaining ringv1 and should just belong to ringv2 [de]serialization?\n\nThe last comment suggests some confusion about if \"upgrading to new swift\" with a small ring even on just your control node would prevent a rollback as soon as your builder changes because of the auto device_id resize in the builder - that would be bad and is confusing because we don\u0027t version builders like we do rings.  But maybe that doesn\u0027t happen (I think builders can stay 2-byte-width until they get their 65536 device and have to expand to make room for the none-dev and then rebalance/write-ring start failing until you --format-version\u003d2).  Somehow this may (or may not?) already be how this change behaves so probably we need some more unittests.","commit_id":"f7e05f4b57c3701f1232c361db111486a1b13907"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a6566844d2ef7139db296d2328bd10aa2dc546b5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"70ca9b16_346ad9f5","updated":"2025-02-04 20:59:16.000000000","message":"I was hoping that the extra level of classes would help draw a box around where I want help manipulating generic gzip streams (with atomic writes using temp files, changing compression levels, etc) vs wanting to define a new file format *on top of* gzip (with writing named sections, automatically updating and writing an index, etc) but maybe it should all just be one big ball o\u0027 mud.","commit_id":"338e28fee15c574a5b45f83b81c2f68fce164614"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6814294fdd41644fa2eb6bb6faf72e42e9561b7b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"1b8808a0_d1803055","updated":"2025-03-20 20:39:35.000000000","message":"I\u0027ve encountered some ergonomic issues with testing this in prod - it would be really nice to be able to explicitly write v2 rings with wide devices (same build version as v1 ring) and also ask a ring file (or builder for that matter) what it\u0027s dev_id_bytes is.","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"887f705893fe9e54225bba243b6c23f8c81ec776","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"68842388_025e0d8b","updated":"2025-03-21 08:29:19.000000000","message":"Maybe something like: https://review.opendev.org/c/openstack/swift/+/945192 \n\nThis is a patch to sqaush so we can play around with how we want to output to look.","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"7ee3b2bb_f621b2d9","updated":"2025-03-26 19:12:40.000000000","message":"neither a Ring nor RingData object should ever change the replica2part2dev_id array, so they don\u0027t have a good reason for their handling of the replica_count and dev_id_bytes properties to be in-consistent\n\n945648: DRY out array resize/normalize | https://review.opendev.org/c/openstack/swift/+/945648\n\nImplementing both of these properties consistently implemented as run-of-the-mill  \"calculated once and cached as under-attrs\" properties make them super obvious and easier to reason about (i.e. replica2part2dev_id table is always SoT, and since it\u0027s never modified for the life-time of this object it should be cached)\n\nThe Builder does not share these requirements:\n\n1) it doesn\u0027t have `_replica2part2dev_id` attribute - it\u0027s called `_replica2part2dev` instead.\n2) it likes to set `_replica2part2dev \u003d None` to indicate it\u0027s never been rebalanced, and so it can\u0027t derive dev_id_bytes consistently from `replica2part2dev` - so during initial rebalance it has to use `len(self.devs)`\n3) Builders have to be able to change both replica_count and dev_id_bytes - both of which should alter the structure of the replica2part2dev table - so it\u0027s less obvious when the array should be authoritative.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cc2db0a28617560f498d54857747e9591d06abe5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"8365518c_cc543ba7","updated":"2025-04-17 15:36:10.000000000","message":"some notes as I work through the code more and discover things; i\u0027m a ways off from any working alternative, but I think I can see an abstraction of the ring.io module that actually provides some benefit at organizing the existing pickle and json+arrays serialization codecs while also making it easier to add a new extensible sections+index codec.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"c717e9d26994ef10a53925d411577f00fddb23cd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":32,"id":"2e71d9b3_ee80376a","updated":"2025-05-20 19:11:43.000000000","message":"recheck","commit_id":"eed0cde6a9478395346020db0cfe0a0cbaffcb97"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"08d68a7ae55f7a38ad0f401a01242ded5d81471b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":36,"id":"e87b3881_7b9c8237","updated":"2025-06-09 01:31:48.000000000","message":"@clay.gerrard@gmail.com, do you still have some diffs kicking around for me to consider?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"65f8a329_f11e62a3","updated":"2025-06-13 21:04:18.000000000","message":"I wanted to share a WIP alternative ring.io abstraction for discussion:\n\n952613: Refactor ring.io | https://review.opendev.org/c/openstack/swift/+/952613\n\n... I could in theory spend MORE time to polish it as a truly viable alternative strategy; but I think that would compete directly with attention to review/polish/refine/merge THIS change.  Instead I think it\u0027d be more fruitful/valuable to try and find some consensus on what sort of abstraction boundaries we want in ring.io and commit to working together towards a common implementation that we all find sufficiently agreeable.\n\nAFAIK this *could* merge as-is w/o any ill effects to clusters currently running ringv1 and any remaining warts with ringv2 are restricted to the implementation rather than format and so could be addressed as follow-ups.  While waiting on feedback for the alternative ring.io abstraction; I\u0027ll work through my remaining comments and see how many of them I can address w/o significant refactor, e.g.\n\n * disallow writing format\u003d0\n * fix traceback(s) with swift-ring-builder object.builder version\n * move Builder\u0027s array resize to happen less frequently\n * docstring cleanup\n * test gaps\n \nThanks for your patience, I think I see a path towards merging the ringv2 format - I\u0027ll appreciate your collaboration as we try to nail down the new code we want to maintain.","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4124726d35662fbe69f7c4a09f874546b70862c9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"ddaab393_83cd6604","updated":"2025-06-13 18:50:51.000000000","message":"recheck\n\nLooks like some kind of neutron error?\n\n\u003e Details: {\u0027type\u0027: \u0027SecurityGroupInUse\u0027, \u0027message\u0027: \u0027Security Group None cannot perform precommit_delete due to Callback neutron.plugins.ml2.drivers.ovn.mech_driver.mech_driver.OVNMechanismDriver._delete_security_group_precommit-3957576 failed with \"Cannot find Port_Group with name\u003dpg_12d2bba2_25bd_4ecf_bdeb_8923d73833d9\".\u0027, \u0027detail\u0027: \u0027\u0027}","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"62d9a971faf3a689657ccff2d109b0aec8d7d260","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":38,"id":"bc8e44fa_6d9c12bf","updated":"2025-06-13 16:10:15.000000000","message":"recheck\n\ngrenade timeout","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":40,"id":"7c7e9377_6b1a08b8","updated":"2025-06-18 22:21:44.000000000","message":"thanks for squashing the .builder version traceback fix and looking at the ring.io refactor\n\nI\u0027m still working on .builder array resizing","commit_id":"f800acbc8ea614780401d62d3df5011eb04ebc14"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"ad00c215_e0d1a441","updated":"2025-08-05 07:23:12.000000000","message":"Ahahaha... the trouble with leaving a bazillion review comments is that it takes forever to even *try* to circle back on them all, even when it\u0027s **not** a patch that I\u0027ve gotten so emotionally invested in that I\u0027d rather go to the dentist than try to wrap my head around what this other person is saying about my code ;-)\n\n---\n\nOK, so Clay\u0027s identified some rough edges:\n\n- Network-order probably wasn\u0027t the right target to aim for. It *seemed* reasonably platform-agnostic, and is probably tolerable for single values, but when it comes to the assignment arrays it sucks that x86 and ARM both need to do a ton of byte-swapping for no real reason.\n\n- Section checksums include the leading BLOB-length bytes, while any [future `swift-ring-packer` tool](https://review.opendev.org/c/openstack/swift/+/866082) would just extract the raw value. Seems weird that extracting a ring would produce files whose checksums don\u0027t match what\u0027s in the index.\n\n  This points at a larger conceptual thing: if the format is a bunch of length-value encoded BLOBs with some special stuff at the end, which part is the section? The whole length-value, or just the value? You already pointed out the `with writer.section(name)` and `reader.read_section(name)` have some asymmetries on the topic.\n\nI\u0027d love to fix those. But for the longest time this patch has been in a weird spot:\n\n- We wanted to vet it in prod as part of the review process\n- That stalled out for a while (maybe my fault, but I was never really sold on the necessity), so we waited until were nearing the threshold for *requiring* wide device IDs and it becomes a blocking issue\n- Now we have it in prod, and we need to not break prod the next time we update it\n- So fixing those requires either (a) some delicate coaching with SRE to ensure we turn off ring v2 generation before upgrading, then turn it back on after, or (b) more code *that we don\u0027t want on master* to ensure we can read both v2rc4 or whatever and the v2 we actually want.\n\nBut now if we\u0027re merging it, I\u0027m kind of tempted to just accept them as warts...","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"593e8a80_76406f56","updated":"2025-07-30 07:49:51.000000000","message":"I do have a diff inline and I\u0027ll also have a follow up for silly nits, but this is looking good.\n\nI think the biggest question left is when should we resize the arrays if not when adding devs. Once we\u0027ve come to an agreement on that I think we should land it.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9a6c718935ab417766db2e47d6ac36e02dbb4e1c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"6e9bd789_6a837000","updated":"2025-08-04 07:11:15.000000000","message":"I\u0027ll wait for Tim et el to chime in. Otherwise, I might add a +A in my morn","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5547e2f2ecafa4944273104096e25fe295552b33","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"59aa2791_417eaf7a","updated":"2025-08-05 06:44:09.000000000","message":"KK, landing this. Everything else can be a follow up. Nice work team!!  ringv2 lives!","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"bdee9d29_9145ce64","updated":"2025-08-01 00:08:57.000000000","message":"Matt \u0026 Tim, I would really appreciate your help closing these comments.  Some are just pointers \u0026 kudos - if you or matt have seen them go ahead and mark them Ack.  If I\u0027m suggesting something should be different and you understand what I\u0027m saying but you\u0027re not going to change it just say that and mark it resolved.\n\nI\u0027m +2 because I\u0027d be genuinely happy seeing this merge.  I haven\u0027t yet found any other functional deficiencies that seem like blockers.  In many ways I think it will be easier and faster to prioritize and fix the remaining non-functional issues once the code is merged and I can just propose my own \"improvements\" to master; we\u0027ll see.  As long as everyone ELSE is happy to help me maintain ringv2 from here on out we should merge it.  Great work Tim.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"acd97a2ec90ff7e1c972b8a0f9def93a0d2b223c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":42,"id":"d6109223_a8d1b5cb","updated":"2025-07-28 09:02:29.000000000","message":"So how close are we getting here? Are the last of these discussions format specific or can they be handled in follow ups?","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"}],"doc/source/overview_ring_format.rst":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":false,"context_lines":[{"line_number":196,"context_line":"    gets large"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"* ``swift/ring/assignments`` - The ring replica2part2dev_id data structure"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"swift/ring/metadata"},{"line_number":201,"context_line":"~~~~~~~~~~~~~~~~~~~"},{"line_number":202,"context_line":"This BLOB is an ASCII-encoded JSON object full of metadata, similar"}],"source_content_type":"text/x-rst","patch_set":21,"id":"c6fa351f_768afda9","line":199,"updated":"2025-01-28 17:30:08.000000000","message":"Should probably add a note about how operators may add their own sections, but the `swift/` prefix is reserved for upstream use.","commit_id":"f7e05f4b57c3701f1232c361db111486a1b13907"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d88180c6f6bb817acb32adfb66b34ee4ead0101b","unresolved":true,"context_lines":[{"line_number":109,"context_line":"to reliably serialize additional large data structures after the"},{"line_number":110,"context_line":"replica-to-part-to-device table. So v2 has been designed to be extensable."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":".. note:: v2 is a new feature and maybe not be fully stable yet."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"The new format starts with magic similar to v1::"},{"line_number":115,"context_line":""}],"source_content_type":"text/x-rst","patch_set":37,"id":"88243ade_42932624","line":112,"updated":"2025-06-10 17:48:31.000000000","message":"I think we can probably drop this -- the more we\u0027ve reviewed this, the more stable it seems.","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9e5734a3b621024408970e6b4e87178824d45db6","unresolved":false,"context_lines":[{"line_number":109,"context_line":"to reliably serialize additional large data structures after the"},{"line_number":110,"context_line":"replica-to-part-to-device table. So v2 has been designed to be extensable."},{"line_number":111,"context_line":""},{"line_number":112,"context_line":".. note:: v2 is a new feature and maybe not be fully stable yet."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"The new format starts with magic similar to v1::"},{"line_number":115,"context_line":""}],"source_content_type":"text/x-rst","patch_set":37,"id":"7cf5e53d_13f1adfd","line":112,"in_reply_to":"88243ade_42932624","updated":"2025-06-11 21:00:49.000000000","message":"Done","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d88180c6f6bb817acb32adfb66b34ee4ead0101b","unresolved":true,"context_lines":[{"line_number":167,"context_line":"* the gzip stream is flushed with another ``Z_FULL_FLUSH``,"},{"line_number":168,"context_line":"* the stream is switched to uncompressed,"},{"line_number":169,"context_line":"* the eight-byte offset of the compressed start of the index is written, and"},{"line_number":170,"context_line":"* the stream is flushed with ``Z_FINISH``."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"By switching to uncompressed, we can know exactly how many bytes will be"},{"line_number":173,"context_line":"written in the tail, so that when reading we can quickly seek to read the"}],"source_content_type":"text/x-rst","patch_set":37,"id":"81f8838b_99748178","line":170,"updated":"2025-06-10 17:48:31.000000000","message":"What about the `CRC32` and `ISIZE`? Maybe better as\n\n\u003e - the gzip stream is closed; this involves:\n\u003e   - flushing the underlying deflate stream with `Z_FINISH`\n\u003e   - writing `CRC32` (of the full uncompressed data)\n\u003e   - writing `ISIZE` (the length of the full uncompressed data mod 2 ** 32)\n\nSomewhere in all of this we probably ought to suggest reading https://datatracker.ietf.org/doc/html/rfc1952 and possibly https://datatracker.ietf.org/doc/html/rfc1951 if you really want to get into the nitty-gritty.","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9e5734a3b621024408970e6b4e87178824d45db6","unresolved":false,"context_lines":[{"line_number":167,"context_line":"* the gzip stream is flushed with another ``Z_FULL_FLUSH``,"},{"line_number":168,"context_line":"* the stream is switched to uncompressed,"},{"line_number":169,"context_line":"* the eight-byte offset of the compressed start of the index is written, and"},{"line_number":170,"context_line":"* the stream is flushed with ``Z_FINISH``."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"By switching to uncompressed, we can know exactly how many bytes will be"},{"line_number":173,"context_line":"written in the tail, so that when reading we can quickly seek to read the"}],"source_content_type":"text/x-rst","patch_set":37,"id":"5a21ba84_17edf4e0","line":170,"in_reply_to":"81f8838b_99748178","updated":"2025-06-11 21:00:49.000000000","message":"Done","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d88180c6f6bb817acb32adfb66b34ee4ead0101b","unresolved":true,"context_lines":[{"line_number":170,"context_line":"* the stream is flushed with ``Z_FINISH``."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"By switching to uncompressed, we can know exactly how many bytes will be"},{"line_number":173,"context_line":"written in the tail, so that when reading we can quickly seek to read the"},{"line_number":174,"context_line":"index offset, seek to the index start, and read the index. From there we"},{"line_number":175,"context_line":"can do similar things for any other section."},{"line_number":176,"context_line":""}],"source_content_type":"text/x-rst","patch_set":37,"id":"ba7c8a33_7273bdd5","line":173,"range":{"start_line":173,"start_character":57,"end_line":173,"end_character":69},"updated":"2025-06-10 17:48:31.000000000","message":"Better as \"seek to and read\"","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9e5734a3b621024408970e6b4e87178824d45db6","unresolved":false,"context_lines":[{"line_number":170,"context_line":"* the stream is flushed with ``Z_FINISH``."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"By switching to uncompressed, we can know exactly how many bytes will be"},{"line_number":173,"context_line":"written in the tail, so that when reading we can quickly seek to read the"},{"line_number":174,"context_line":"index offset, seek to the index start, and read the index. From there we"},{"line_number":175,"context_line":"can do similar things for any other section."},{"line_number":176,"context_line":""}],"source_content_type":"text/x-rst","patch_set":37,"id":"c9a9635a_d8eec2ce","line":173,"range":{"start_line":173,"start_character":57,"end_line":173,"end_character":69},"in_reply_to":"ba7c8a33_7273bdd5","updated":"2025-06-11 21:00:49.000000000","message":"Done","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d88180c6f6bb817acb32adfb66b34ee4ead0101b","unresolved":true,"context_lines":[{"line_number":184,"context_line":"* Read/decompress the size of the index BLOB"},{"line_number":185,"context_line":"* Read/decompress the json serialized index."},{"line_number":186,"context_line":""},{"line_number":187,"context_line":".. note:: This 31 bytes is the 8 byte location and the tail of the gzip file."},{"line_number":188,"context_line":""},{"line_number":189,"context_line":"The currently defined section and section names upstream are as follows:"},{"line_number":190,"context_line":""}],"source_content_type":"text/x-rst","patch_set":37,"id":"7018ba7b_1fac7a34","line":187,"range":{"start_line":187,"start_character":10,"end_line":187,"end_character":77},"updated":"2025-06-10 17:48:31.000000000","message":"This could be improved. Maybe something like\n\n\u003e 31 bytes is the combined lengths of the deflate block containing the 8 byte offset, a `Z_FULL_FLUSH` block, the `Z_FINISH` block, and the `CRC32` and `ISIZE`.","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9e5734a3b621024408970e6b4e87178824d45db6","unresolved":false,"context_lines":[{"line_number":184,"context_line":"* Read/decompress the size of the index BLOB"},{"line_number":185,"context_line":"* Read/decompress the json serialized index."},{"line_number":186,"context_line":""},{"line_number":187,"context_line":".. note:: This 31 bytes is the 8 byte location and the tail of the gzip file."},{"line_number":188,"context_line":""},{"line_number":189,"context_line":"The currently defined section and section names upstream are as follows:"},{"line_number":190,"context_line":""}],"source_content_type":"text/x-rst","patch_set":37,"id":"3462fad5_9e26e702","line":187,"range":{"start_line":187,"start_character":10,"end_line":187,"end_character":77},"in_reply_to":"7018ba7b_1fac7a34","updated":"2025-06-11 21:00:49.000000000","message":"Done","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"acd97a2ec90ff7e1c972b8a0f9def93a0d2b223c","unresolved":true,"context_lines":[{"line_number":31,"context_line":"Initially, rings were simply pickle dumps of the RingData object. `With"},{"line_number":32,"context_line":"Swift 1.3.0 \u003chttps://opendev.org/openstack/swift/commit/fc6391ea\u003e`__, this"},{"line_number":33,"context_line":"changed to pickling a pure-stdlib data structure, but the core concept"},{"line_number":34,"context_line":"was the same."},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"Ring v1"},{"line_number":37,"context_line":"-------"}],"source_content_type":"text/x-rst","patch_set":41,"id":"7579d56c_40437363","line":34,"updated":"2025-07-28 09:02:29.000000000","message":"After this patch lands we should do a follow up to cover mention when these stopped being supported.. or should we just put in the next release now ;)","commit_id":"8a8dbe8d5b54cdacd35bc08c767dfbd20a8892a5"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c580074cf2b25292ccf2763967112c0f95042f9f","unresolved":false,"context_lines":[{"line_number":31,"context_line":"Initially, rings were simply pickle dumps of the RingData object. `With"},{"line_number":32,"context_line":"Swift 1.3.0 \u003chttps://opendev.org/openstack/swift/commit/fc6391ea\u003e`__, this"},{"line_number":33,"context_line":"changed to pickling a pure-stdlib data structure, but the core concept"},{"line_number":34,"context_line":"was the same."},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"Ring v1"},{"line_number":37,"context_line":"-------"}],"source_content_type":"text/x-rst","patch_set":41,"id":"d79a61f8_ab915f72","line":34,"in_reply_to":"7579d56c_40437363","updated":"2025-07-31 07:14:26.000000000","message":"Done","commit_id":"8a8dbe8d5b54cdacd35bc08c767dfbd20a8892a5"}],"swift/cli/ringbuilder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":1333,"context_line":"        ring_data.save("},{"line_number":1334,"context_line":"            pathjoin(backup_dir, \u0027%d.\u0027 % time() + basename(ring_file)),"},{"line_number":1335,"context_line":"            format_version\u003doptions.format_version)"},{"line_number":1336,"context_line":"        ring_data.save(ring_file, format_version\u003doptions.format_version)"},{"line_number":1337,"context_line":"        exit(EXIT_SUCCESS)"},{"line_number":1338,"context_line":""},{"line_number":1339,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":9,"id":"f43319ee_de265e51","line":1336,"updated":"2022-07-28 18:28:42.000000000","message":"Makes me wonder why we call save() again, instead of just copying...\n\nI guess it\u0027s for the atomicity? But we could still do a copy to a temp file, then link it in place.","commit_id":"4b2ba51a6d71ee7e3627b48dfe923d5c78e62133"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":1333,"context_line":"        ring_data.save("},{"line_number":1334,"context_line":"            pathjoin(backup_dir, \u0027%d.\u0027 % time() + basename(ring_file)),"},{"line_number":1335,"context_line":"            format_version\u003doptions.format_version)"},{"line_number":1336,"context_line":"        ring_data.save(ring_file, format_version\u003doptions.format_version)"},{"line_number":1337,"context_line":"        exit(EXIT_SUCCESS)"},{"line_number":1338,"context_line":""},{"line_number":1339,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":9,"id":"ed76a227_f8b0b14b","line":1336,"in_reply_to":"f43319ee_de265e51","updated":"2022-10-21 20:02:17.000000000","message":"Ack","commit_id":"4b2ba51a6d71ee7e3627b48dfe923d5c78e62133"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":621,"context_line":"                if builder_dict \u003d\u003d ring_dict:"},{"line_number":622,"context_line":"                    print(\u0027Ring file %s is up-to-date\u0027 % ring_file)"},{"line_number":623,"context_line":"                else:"},{"line_number":624,"context_line":"                    print(\u0027Ring file %s is obsolete\u0027 % ring_file)"},{"line_number":625,"context_line":""},{"line_number":626,"context_line":"        if ring_empty_error:"},{"line_number":627,"context_line":"            balance_per_dev \u003d defaultdict(int)"}],"source_content_type":"text/x-python","patch_set":17,"id":"acae3204_2b36d5b0","line":624,"updated":"2023-03-29 00:40:00.000000000","message":"so i guess this is the only place we try to compare builder.to_dict() and ring.to_dict(), and in this one case if the dev_id_bytes doesn\u0027t match it\u0027s ok","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c40c0cc14a628562037366a1b1e70adc2ec0d0da","unresolved":false,"context_lines":[{"line_number":621,"context_line":"                if builder_dict \u003d\u003d ring_dict:"},{"line_number":622,"context_line":"                    print(\u0027Ring file %s is up-to-date\u0027 % ring_file)"},{"line_number":623,"context_line":"                else:"},{"line_number":624,"context_line":"                    print(\u0027Ring file %s is obsolete\u0027 % ring_file)"},{"line_number":625,"context_line":""},{"line_number":626,"context_line":"        if ring_empty_error:"},{"line_number":627,"context_line":"            balance_per_dev \u003d defaultdict(int)"}],"source_content_type":"text/x-python","patch_set":17,"id":"04a03c4f_43ace9fa","line":624,"in_reply_to":"acae3204_2b36d5b0","updated":"2023-05-03 22:58:40.000000000","message":"I should maybe put these `pop()`s closer together, though.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":1064,"context_line":"            options.format_version \u003d DEFAULT_RING_FORMAT_VERSION"},{"line_number":1065,"context_line":"        else:"},{"line_number":1066,"context_line":"            # N.B. choices doesn\u0027t work with type\u003dint"},{"line_number":1067,"context_line":"            options.format_version \u003d int(options.format_version)"},{"line_number":1068,"context_line":""},{"line_number":1069,"context_line":"        def get_seed(index):"},{"line_number":1070,"context_line":"            if options.seed:"}],"source_content_type":"text/x-python","patch_set":17,"id":"5aa59642_2d1b2895","line":1067,"updated":"2023-03-29 00:40:00.000000000","message":"maybe an optparse thing?\n\n\t\u003e\u003e\u003e import argparse\n\t\u003e\u003e\u003e parser \u003d argparse.ArgumentParser()\n\t\u003e\u003e\u003e parser.add_argument(\u0027-c\u0027, choices\u003d[1, 2], type\u003dint)\n\t_StoreAction(option_strings\u003d[\u0027-c\u0027], dest\u003d\u0027c\u0027, nargs\u003dNone, const\u003dNone, default\u003dNone, type\u003d\u003cclass \u0027int\u0027\u003e, choices\u003d[1, 2], help\u003dNone, metavar\u003dNone)\n\t\u003e\u003e\u003e args \u003d parser.parse_args([\u0027-c\u0027, \u00271\u0027])\n\t\u003e\u003e\u003e args.c\n\t1","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c40c0cc14a628562037366a1b1e70adc2ec0d0da","unresolved":false,"context_lines":[{"line_number":1064,"context_line":"            options.format_version \u003d DEFAULT_RING_FORMAT_VERSION"},{"line_number":1065,"context_line":"        else:"},{"line_number":1066,"context_line":"            # N.B. choices doesn\u0027t work with type\u003dint"},{"line_number":1067,"context_line":"            options.format_version \u003d int(options.format_version)"},{"line_number":1068,"context_line":""},{"line_number":1069,"context_line":"        def get_seed(index):"},{"line_number":1070,"context_line":"            if options.seed:"}],"source_content_type":"text/x-python","patch_set":17,"id":"0c28a47f_2af97358","line":1067,"in_reply_to":"5aa59642_2d1b2895","updated":"2023-05-03 22:58:40.000000000","message":"Must be:\n```\n\u003e\u003e\u003e import optparse\n\u003e\u003e\u003e parser \u003d optparse.OptionParser()\n\u003e\u003e\u003e parser.add_option(\u0027-c\u0027, choices\u003d[1, 2], type\u003dint)\nTraceback (most recent call last):\n  File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\n  File \"/usr/lib/python3.10/optparse.py\", line 1000, in add_option\n    option \u003d self.option_class(*args, **kwargs)\n  File \"/usr/lib/python3.10/optparse.py\", line 581, in __init__\n    checker(self)\n  File \"/usr/lib/python3.10/optparse.py\", line 672, in _check_choice\n    raise OptionError(\noptparse.OptionError: option -c: must not supply choices for type \u0027int\u0027\n```\nI don\u0027t really want to add \"port to argparse\" as a blocker, though.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":1309,"context_line":"    new device information."},{"line_number":1310,"context_line":"        \"\"\""},{"line_number":1311,"context_line":"        usage \u003d Commands.write_ring.__doc__.strip()"},{"line_number":1312,"context_line":"        parser \u003d optparse.OptionParser(usage)"},{"line_number":1313,"context_line":"        parser.add_option(\u0027--format-version\u0027,"},{"line_number":1314,"context_line":"                          choices\u003dFORMAT_CHOICES, default\u003dNone,"},{"line_number":1315,"context_line":"                          help\u003d\"specify ring format version\")"}],"source_content_type":"text/x-python","patch_set":17,"id":"8551c9da_9ca70c09","line":1312,"updated":"2023-03-29 00:40:00.000000000","message":"interesting, so here we\u0027re adding a parser to a command that didn\u0027t previously have one, but sticking with optparse I assume to keep in the style of the surrounding code","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":1309,"context_line":"    new device information."},{"line_number":1310,"context_line":"        \"\"\""},{"line_number":1311,"context_line":"        usage \u003d Commands.write_ring.__doc__.strip()"},{"line_number":1312,"context_line":"        parser \u003d optparse.OptionParser(usage)"},{"line_number":1313,"context_line":"        parser.add_option(\u0027--format-version\u0027,"},{"line_number":1314,"context_line":"                          choices\u003dFORMAT_CHOICES, default\u003dNone,"},{"line_number":1315,"context_line":"                          help\u003d\"specify ring format version\")"}],"source_content_type":"text/x-python","patch_set":17,"id":"b60aaa59_5c6de83e","line":1312,"in_reply_to":"0e48ea62_6727f9ab","updated":"2025-05-20 19:58:47.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":1309,"context_line":"    new device information."},{"line_number":1310,"context_line":"        \"\"\""},{"line_number":1311,"context_line":"        usage \u003d Commands.write_ring.__doc__.strip()"},{"line_number":1312,"context_line":"        parser \u003d optparse.OptionParser(usage)"},{"line_number":1313,"context_line":"        parser.add_option(\u0027--format-version\u0027,"},{"line_number":1314,"context_line":"                          choices\u003dFORMAT_CHOICES, default\u003dNone,"},{"line_number":1315,"context_line":"                          help\u003d\"specify ring format version\")"}],"source_content_type":"text/x-python","patch_set":17,"id":"0e48ea62_6727f9ab","line":1312,"in_reply_to":"8551c9da_9ca70c09","updated":"2023-04-03 23:05:30.000000000","message":"I\u0027m happy to see a follow-up patch (or more likely, a series of patches) to have us get rid of `optparse` -- but yeah, having `swift-ring-builder` be schizophrenic sounds pretty terrible.\n\nFWIW:\n\n $ grep -Rl optparse swift bin | egrep -v .pyc$\n swift/cli/recon.py\n swift/cli/dispersion_report.py\n swift/cli/ringbuilder.py\n swift/common/utils.py\n swift/common/ring/utils.py\n bin/swift-container-sharder\n bin/swift-object-reconstructor\n bin/swift-account-info\n bin/swift-oldies\n bin/swift-object-info\n bin/swift-init\n bin/swift-reconciler-enqueue\n bin/swift-object-replicator\n bin/swift-dispersion-populate\n bin/swift-orphans\n bin/swift-object-auditor\n bin/swift-container-replicator\n bin/swift-get-nodes\n bin/swift-object-expirer\n bin/swift-config\n bin/swift-account-replicator\n bin/swift-container-info","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6814294fdd41644fa2eb6bb6faf72e42e9561b7b","unresolved":true,"context_lines":[{"line_number":1750,"context_line":"        mkdir(backup_dir)"},{"line_number":1751,"context_line":"    except OSError as err:"},{"line_number":1752,"context_line":"        if err.errno !\u003d EEXIST:"},{"line_number":1753,"context_line":"            raise"},{"line_number":1754,"context_line":""},{"line_number":1755,"context_line":"    if len(argv) \u003d\u003d 2:"},{"line_number":1756,"context_line":"        command \u003d \"default\""}],"source_content_type":"text/x-python","patch_set":24,"id":"46e8a406_64daa16d","line":1753,"updated":"2025-03-20 20:39:35.000000000","message":"since we have the (write_builder, version) white list anyway I wonder if we could make this error/exit conditional - or explicitly mark the read-only operations with a @read-only that causes getattr(command).needs_backup to return False?","commit_id":"1e0fb4dde53d336040f0bd07c683a521b2056197"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"de7534f73d62d896821b2cfb5d129bc1248ee677","unresolved":true,"context_lines":[{"line_number":1750,"context_line":"        mkdir(backup_dir)"},{"line_number":1751,"context_line":"    except OSError as err:"},{"line_number":1752,"context_line":"        if err.errno !\u003d EEXIST:"},{"line_number":1753,"context_line":"            raise"},{"line_number":1754,"context_line":""},{"line_number":1755,"context_line":"    if len(argv) \u003d\u003d 2:"},{"line_number":1756,"context_line":"        command \u003d \"default\""}],"source_content_type":"text/x-python","patch_set":24,"id":"b6c43ff7_cb07d99d","line":1753,"in_reply_to":"16a2b530_951d6d65","updated":"2025-07-17 05:44:09.000000000","message":"I took this into account in https://review.opendev.org/c/openstack/swift/+/955263 as well","commit_id":"1e0fb4dde53d336040f0bd07c683a521b2056197"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"63f4373e2662b8c672efa57ce2d7ee15233f8cbc","unresolved":true,"context_lines":[{"line_number":1750,"context_line":"        mkdir(backup_dir)"},{"line_number":1751,"context_line":"    except OSError as err:"},{"line_number":1752,"context_line":"        if err.errno !\u003d EEXIST:"},{"line_number":1753,"context_line":"            raise"},{"line_number":1754,"context_line":""},{"line_number":1755,"context_line":"    if len(argv) \u003d\u003d 2:"},{"line_number":1756,"context_line":"        command \u003d \"default\""}],"source_content_type":"text/x-python","patch_set":24,"id":"16a2b530_951d6d65","line":1753,"in_reply_to":"46e8a406_64daa16d","updated":"2025-03-21 04:08:00.000000000","message":"I guess default is another that is readonly and this is not required either, but then we need. Though I guess we special case the \u0027default\u0027 option, but it is possible to pass in I suppose.\n\nBut this seems more like a pre-existing issue that v2 has added to, rather than one created by it. So maybe a clean up in a follow up patch.","commit_id":"1e0fb4dde53d336040f0bd07c683a521b2056197"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":false,"context_lines":[{"line_number":1750,"context_line":"        mkdir(backup_dir)"},{"line_number":1751,"context_line":"    except OSError as err:"},{"line_number":1752,"context_line":"        if err.errno !\u003d EEXIST:"},{"line_number":1753,"context_line":"            raise"},{"line_number":1754,"context_line":""},{"line_number":1755,"context_line":"    if len(argv) \u003d\u003d 2:"},{"line_number":1756,"context_line":"        command \u003d \"default\""}],"source_content_type":"text/x-python","patch_set":24,"id":"7280835d_c80d3ee7","line":1753,"in_reply_to":"b6c43ff7_cb07d99d","updated":"2025-07-30 07:49:51.000000000","message":"ditto","commit_id":"1e0fb4dde53d336040f0bd07c683a521b2056197"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6814294fdd41644fa2eb6bb6faf72e42e9561b7b","unresolved":true,"context_lines":[{"line_number":600,"context_line":"        print(\u0027%d partitions, %.6f replicas, %d regions, %d zones, \u0027"},{"line_number":601,"context_line":"              \u0027%d devices, %.02f balance%s\u0027 % ("},{"line_number":602,"context_line":"                  builder.parts, builder.replicas, regions, zones, dev_count,"},{"line_number":603,"context_line":"                  balance, dispersion_trailer))"},{"line_number":604,"context_line":"        print(\u0027The minimum number of hours before a partition can be \u0027"},{"line_number":605,"context_line":"              \u0027reassigned is %s (%s remaining)\u0027 % ("},{"line_number":606,"context_line":"                  builder.min_part_hours,"}],"source_content_type":"text/x-python","patch_set":27,"id":"16281e91_ee84b75b","line":603,"updated":"2025-03-20 20:39:35.000000000","message":"do we want to expose dev_id_bytes next to dev_count?","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":600,"context_line":"        print(\u0027%d partitions, %.6f replicas, %d regions, %d zones, \u0027"},{"line_number":601,"context_line":"              \u0027%d devices, %.02f balance%s\u0027 % ("},{"line_number":602,"context_line":"                  builder.parts, builder.replicas, regions, zones, dev_count,"},{"line_number":603,"context_line":"                  balance, dispersion_trailer))"},{"line_number":604,"context_line":"        print(\u0027The minimum number of hours before a partition can be \u0027"},{"line_number":605,"context_line":"              \u0027reassigned is %s (%s remaining)\u0027 % ("},{"line_number":606,"context_line":"                  builder.min_part_hours,"}],"source_content_type":"text/x-python","patch_set":27,"id":"6aa62bc1_32505315","line":603,"in_reply_to":"16281e91_ee84b75b","updated":"2025-05-20 19:58:47.000000000","message":"Done","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6814294fdd41644fa2eb6bb6faf72e42e9561b7b","unresolved":true,"context_lines":[{"line_number":686,"context_line":"            if reader:"},{"line_number":687,"context_line":"                reader.close()"},{"line_number":688,"context_line":"        print(\u0027%s: Serialization version: %d, build version: %d\u0027 %"},{"line_number":689,"context_line":"              (argv[1], serialization_version, version))"},{"line_number":690,"context_line":"        exit(EXIT_SUCCESS)"},{"line_number":691,"context_line":""},{"line_number":692,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":27,"id":"ed84f950_35a3cf21","line":689,"updated":"2025-03-20 20:39:35.000000000","message":"i would really like it if I could ask a v2 ring.gz what it\u0027s dev_id_bytes is","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":686,"context_line":"            if reader:"},{"line_number":687,"context_line":"                reader.close()"},{"line_number":688,"context_line":"        print(\u0027%s: Serialization version: %d, build version: %d\u0027 %"},{"line_number":689,"context_line":"              (argv[1], serialization_version, version))"},{"line_number":690,"context_line":"        exit(EXIT_SUCCESS)"},{"line_number":691,"context_line":""},{"line_number":692,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":27,"id":"c5a1a984_87f3159f","line":689,"in_reply_to":"44f4b5fc_3806acdb","updated":"2025-05-20 19:58:47.000000000","message":"Done","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"887f705893fe9e54225bba243b6c23f8c81ec776","unresolved":true,"context_lines":[{"line_number":686,"context_line":"            if reader:"},{"line_number":687,"context_line":"                reader.close()"},{"line_number":688,"context_line":"        print(\u0027%s: Serialization version: %d, build version: %d\u0027 %"},{"line_number":689,"context_line":"              (argv[1], serialization_version, version))"},{"line_number":690,"context_line":"        exit(EXIT_SUCCESS)"},{"line_number":691,"context_line":""},{"line_number":692,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":27,"id":"44f4b5fc_3806acdb","line":689,"in_reply_to":"ed84f950_35a3cf21","updated":"2025-03-21 08:29:19.000000000","message":"Shall we just throw it into here, or a whole new ringbuilder ring command.\n\nMaybe something like:\n\n    \u003cring\u003e: Serialization version: 2 (\u003cdev_id_bytes\u003e), build version: 50\n\nor:\n\n    \u003cring\u003e: Serialization version: 2, Device id byte length: 4, build version: 50","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6814294fdd41644fa2eb6bb6faf72e42e9561b7b","unresolved":true,"context_lines":[{"line_number":1717,"context_line":""},{"line_number":1718,"context_line":"    builder_file, ring_file \u003d parse_builder_ring_filename_args(argv)"},{"line_number":1719,"context_line":"    if builder_file !\u003d argv[1]:"},{"line_number":1720,"context_line":"        if len(argv) \u003e 2 and argv[2] in (\u0027write_buider\u0027, \u0027version\u0027):"},{"line_number":1721,"context_line":"            pass"},{"line_number":1722,"context_line":"        else:"},{"line_number":1723,"context_line":"            print(\u0027Note: using %s instead of %s as builder file\u0027 % ("}],"source_content_type":"text/x-python","patch_set":27,"id":"8e524ebf_a997dce7","line":1720,"updated":"2025-03-20 20:39:35.000000000","message":"there appears to be a typo in `write_builder` - is this tested?","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":1717,"context_line":""},{"line_number":1718,"context_line":"    builder_file, ring_file \u003d parse_builder_ring_filename_args(argv)"},{"line_number":1719,"context_line":"    if builder_file !\u003d argv[1]:"},{"line_number":1720,"context_line":"        if len(argv) \u003e 2 and argv[2] in (\u0027write_buider\u0027, \u0027version\u0027):"},{"line_number":1721,"context_line":"            pass"},{"line_number":1722,"context_line":"        else:"},{"line_number":1723,"context_line":"            print(\u0027Note: using %s instead of %s as builder file\u0027 % ("}],"source_content_type":"text/x-python","patch_set":27,"id":"e00faa85_284aa624","line":1720,"in_reply_to":"1e3be271_f617a4a7","updated":"2025-05-06 20:28:41.000000000","message":"In all fairness, [tests are not immune from typos](https://github.com/openstack/swift/blob/2.35.0/test/unit/common/ring/test_builder.py#L868).","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":false,"context_lines":[{"line_number":1717,"context_line":""},{"line_number":1718,"context_line":"    builder_file, ring_file \u003d parse_builder_ring_filename_args(argv)"},{"line_number":1719,"context_line":"    if builder_file !\u003d argv[1]:"},{"line_number":1720,"context_line":"        if len(argv) \u003e 2 and argv[2] in (\u0027write_buider\u0027, \u0027version\u0027):"},{"line_number":1721,"context_line":"            pass"},{"line_number":1722,"context_line":"        else:"},{"line_number":1723,"context_line":"            print(\u0027Note: using %s instead of %s as builder file\u0027 % ("}],"source_content_type":"text/x-python","patch_set":27,"id":"22954b09_0be450fc","line":1720,"in_reply_to":"7ad489d8_488950e0","updated":"2025-07-30 07:49:51.000000000","message":"I\u0027ve reworked the arg handling a little in a follow up, this discussion can move there if we want to talk more about it. https://review.opendev.org/c/openstack/swift/+/955263","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"63f4373e2662b8c672efa57ce2d7ee15233f8cbc","unresolved":true,"context_lines":[{"line_number":1717,"context_line":""},{"line_number":1718,"context_line":"    builder_file, ring_file \u003d parse_builder_ring_filename_args(argv)"},{"line_number":1719,"context_line":"    if builder_file !\u003d argv[1]:"},{"line_number":1720,"context_line":"        if len(argv) \u003e 2 and argv[2] in (\u0027write_buider\u0027, \u0027version\u0027):"},{"line_number":1721,"context_line":"            pass"},{"line_number":1722,"context_line":"        else:"},{"line_number":1723,"context_line":"            print(\u0027Note: using %s instead of %s as builder file\u0027 % ("}],"source_content_type":"text/x-python","patch_set":27,"id":"1e3be271_f617a4a7","line":1720,"in_reply_to":"8e524ebf_a997dce7","updated":"2025-03-21 04:08:00.000000000","message":"I guess not. That\u0027s my bad, but all it does is stop printing the note, so didn\u0027t seem like an interesting enough test.","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"de7534f73d62d896821b2cfb5d129bc1248ee677","unresolved":true,"context_lines":[{"line_number":1717,"context_line":""},{"line_number":1718,"context_line":"    builder_file, ring_file \u003d parse_builder_ring_filename_args(argv)"},{"line_number":1719,"context_line":"    if builder_file !\u003d argv[1]:"},{"line_number":1720,"context_line":"        if len(argv) \u003e 2 and argv[2] in (\u0027write_buider\u0027, \u0027version\u0027):"},{"line_number":1721,"context_line":"            pass"},{"line_number":1722,"context_line":"        else:"},{"line_number":1723,"context_line":"            print(\u0027Note: using %s instead of %s as builder file\u0027 % ("}],"source_content_type":"text/x-python","patch_set":27,"id":"7ad489d8_488950e0","line":1720,"in_reply_to":"d9450a23_608b6e43","updated":"2025-07-17 05:44:09.000000000","message":"yeah, those 2 subcommands require the ring to be passed in, not the builder.. so printing a note saying it\u0027ll use some builder file instead, when it doesn\u0027t is clearly wrong.\n\nMaybe this whole thing could be refactored so the subcommands throw the note as they\u0027ll know ranther then putting the logic in main. But that can easily be a follow up, this just extends what main always did.","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1717,"context_line":""},{"line_number":1718,"context_line":"    builder_file, ring_file \u003d parse_builder_ring_filename_args(argv)"},{"line_number":1719,"context_line":"    if builder_file !\u003d argv[1]:"},{"line_number":1720,"context_line":"        if len(argv) \u003e 2 and argv[2] in (\u0027write_buider\u0027, \u0027version\u0027):"},{"line_number":1721,"context_line":"            pass"},{"line_number":1722,"context_line":"        else:"},{"line_number":1723,"context_line":"            print(\u0027Note: using %s instead of %s as builder file\u0027 % ("}],"source_content_type":"text/x-python","patch_set":27,"id":"d9450a23_608b6e43","line":1720,"in_reply_to":"e00faa85_284aa624","updated":"2025-06-13 21:04:18.000000000","message":"\u003e didn\u0027t seem like an interesting enough test.\n\nyeah I have a lot of trouble understanding the ringbuilder arg handling at a glance; I wasn\u0027t quite sure what this part was doing.\n\nSo the idea with the bugfix here is that the `write_builder` subcommand will no longer print a `Note` if you tell it `write_builder` and pass it a path to a builder!?","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6814294fdd41644fa2eb6bb6faf72e42e9561b7b","unresolved":true,"context_lines":[{"line_number":1736,"context_line":"        exit(EXIT_ERROR)"},{"line_number":1737,"context_line":"    except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:"},{"line_number":1738,"context_line":"        if len(argv) \u003c 3 or argv[2] not in (\u0027create\u0027, \u0027write_builder\u0027,"},{"line_number":1739,"context_line":"                                            \u0027version\u0027):"},{"line_number":1740,"context_line":"            print(e)"},{"line_number":1741,"context_line":"            exit(EXIT_ERROR)"},{"line_number":1742,"context_line":"    except Exception as e:"}],"source_content_type":"text/x-python","patch_set":27,"id":"a6939cde_dfbeeda7","line":1739,"updated":"2025-03-20 20:39:35.000000000","message":"interesting that this is duplicated with part of the list on L1720","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1736,"context_line":"        exit(EXIT_ERROR)"},{"line_number":1737,"context_line":"    except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:"},{"line_number":1738,"context_line":"        if len(argv) \u003c 3 or argv[2] not in (\u0027create\u0027, \u0027write_builder\u0027,"},{"line_number":1739,"context_line":"                                            \u0027version\u0027):"},{"line_number":1740,"context_line":"            print(e)"},{"line_number":1741,"context_line":"            exit(EXIT_ERROR)"},{"line_number":1742,"context_line":"    except Exception as e:"}],"source_content_type":"text/x-python","patch_set":27,"id":"cb9be017_07d08bc5","line":1739,"in_reply_to":"36f38ac1_0d8f8084","updated":"2025-06-13 21:04:18.000000000","message":"\u003e except this is for not erroring out \n\nso there\u0027s two catagories of subcommands:\n\n```\nexempt_from_note_builder_not_exist \u003d {\u0027write_builder\u0027, \u0027version\u0027}\nexempt_from_error_builder_not_exist \u003d exempt_from_note_builder_not_exist + {\u0027create\u0027}\n```","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":false,"context_lines":[{"line_number":1736,"context_line":"        exit(EXIT_ERROR)"},{"line_number":1737,"context_line":"    except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:"},{"line_number":1738,"context_line":"        if len(argv) \u003c 3 or argv[2] not in (\u0027create\u0027, \u0027write_builder\u0027,"},{"line_number":1739,"context_line":"                                            \u0027version\u0027):"},{"line_number":1740,"context_line":"            print(e)"},{"line_number":1741,"context_line":"            exit(EXIT_ERROR)"},{"line_number":1742,"context_line":"    except Exception as e:"}],"source_content_type":"text/x-python","patch_set":27,"id":"94bc8ea4_e4872f77","line":1739,"in_reply_to":"3d72d2e8_a8436009","updated":"2025-07-30 07:49:51.000000000","message":"Have a follow up that reworks this.. so let\u0027s move the discussion there.","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"63f4373e2662b8c672efa57ce2d7ee15233f8cbc","unresolved":true,"context_lines":[{"line_number":1736,"context_line":"        exit(EXIT_ERROR)"},{"line_number":1737,"context_line":"    except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:"},{"line_number":1738,"context_line":"        if len(argv) \u003c 3 or argv[2] not in (\u0027create\u0027, \u0027write_builder\u0027,"},{"line_number":1739,"context_line":"                                            \u0027version\u0027):"},{"line_number":1740,"context_line":"            print(e)"},{"line_number":1741,"context_line":"            exit(EXIT_ERROR)"},{"line_number":1742,"context_line":"    except Exception as e:"}],"source_content_type":"text/x-python","patch_set":27,"id":"36f38ac1_0d8f8084","line":1739,"in_reply_to":"a6939cde_dfbeeda7","updated":"2025-03-21 04:08:00.000000000","message":"yeah except this is for not erroring out if the builder isn\u0027t there.","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"de7534f73d62d896821b2cfb5d129bc1248ee677","unresolved":true,"context_lines":[{"line_number":1736,"context_line":"        exit(EXIT_ERROR)"},{"line_number":1737,"context_line":"    except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:"},{"line_number":1738,"context_line":"        if len(argv) \u003c 3 or argv[2] not in (\u0027create\u0027, \u0027write_builder\u0027,"},{"line_number":1739,"context_line":"                                            \u0027version\u0027):"},{"line_number":1740,"context_line":"            print(e)"},{"line_number":1741,"context_line":"            exit(EXIT_ERROR)"},{"line_number":1742,"context_line":"    except Exception as e:"}],"source_content_type":"text/x-python","patch_set":27,"id":"3d72d2e8_a8436009","line":1739,"in_reply_to":"cb9be017_07d08bc5","updated":"2025-07-17 05:44:09.000000000","message":"Maybe something like: https://review.opendev.org/c/openstack/swift/+/955263","commit_id":"0345b2b810c2ed35b4c17cbd9e4abb9102814b48"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5331c8cea7736697abee8a831ff495ad4fa5773d","unresolved":true,"context_lines":[{"line_number":598,"context_line":"        dispersion_trailer \u003d \u0027\u0027 if builder.dispersion is None else ("},{"line_number":599,"context_line":"            \u0027, %.02f dispersion\u0027 % (builder.dispersion))"},{"line_number":600,"context_line":"        print(\u0027%d partitions, %.6f replicas, %d regions, %d zones, \u0027"},{"line_number":601,"context_line":"              \u0027%d devices, %d-byte IDs, %.02f balance%s\u0027 % ("},{"line_number":602,"context_line":"                  builder.parts, builder.replicas, regions, zones, dev_count,"},{"line_number":603,"context_line":"                  builder.dev_id_bytes, balance, dispersion_trailer))"},{"line_number":604,"context_line":"        print(\u0027The minimum number of hours before a partition can be \u0027"}],"source_content_type":"text/x-python","patch_set":28,"id":"17d91662_125d8bdf","line":601,"updated":"2025-03-24 03:55:49.000000000","message":"Yeah, much better output thanks Tim","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":598,"context_line":"        dispersion_trailer \u003d \u0027\u0027 if builder.dispersion is None else ("},{"line_number":599,"context_line":"            \u0027, %.02f dispersion\u0027 % (builder.dispersion))"},{"line_number":600,"context_line":"        print(\u0027%d partitions, %.6f replicas, %d regions, %d zones, \u0027"},{"line_number":601,"context_line":"              \u0027%d devices, %d-byte IDs, %.02f balance%s\u0027 % ("},{"line_number":602,"context_line":"                  builder.parts, builder.replicas, regions, zones, dev_count,"},{"line_number":603,"context_line":"                  builder.dev_id_bytes, balance, dispersion_trailer))"},{"line_number":604,"context_line":"        print(\u0027The minimum number of hours before a partition can be \u0027"}],"source_content_type":"text/x-python","patch_set":28,"id":"3709486a_2d511099","line":601,"in_reply_to":"17d91662_125d8bdf","updated":"2025-05-20 19:58:47.000000000","message":"Acknowledged","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":685,"context_line":"            exit(EXIT_ERROR)"},{"line_number":686,"context_line":"        finally:"},{"line_number":687,"context_line":"            if reader:"},{"line_number":688,"context_line":"                reader.close()"},{"line_number":689,"context_line":"        print(\u0027%s: Serialization version: %d (%d-byte IDs), \u0027"},{"line_number":690,"context_line":"              \u0027build version: %d\u0027 %"},{"line_number":691,"context_line":"              (argv[1], serialization_version, dev_id_bytes, version))"}],"source_content_type":"text/x-python","patch_set":34,"id":"38d92f9a_07dd2646","line":688,"updated":"2025-06-13 21:04:18.000000000","message":"this isn\u0027t the ideal failure mode if you accidently pass version a builder instead of ring.gz\n```\nvagrant@saio:~$ swift-ring-builder /etc/swift/object.ring.gz version\n/etc/swift/object.ring.gz: Serialization version: 1 (2-byte IDs), build version: 5\nvagrant@saio:~$ swift-ring-builder /etc/swift/object.builder version\nTraceback (most recent call last):\n  File \"/vagrant/swift/swift/cli/ringbuilder.py\", line 675, in version\n    reader \u003d RingReader(open(argv[1], \u0027br\u0027))\n  File \"/vagrant/swift/swift/common/ring/io.py\", line 262, in __init__\n    magic \u003d self.read(4)\n  File \"/vagrant/swift/swift/common/ring/io.py\", line 148, in read\n    if self._buffer_chunk():\n  File \"/vagrant/swift/swift/common/ring/io.py\", line 133, in _buffer_chunk\n    self._decompress_from_buffer(len(self.compressed_buffer) - keep)\n  File \"/vagrant/swift/swift/common/ring/io.py\", line 95, in _decompress_from_buffer\n    self.buffer +\u003d self.decompressor.decompress(chunk)\nzlib.error: Error -3 while decompressing data: incorrect header check\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/usr/local/bin/swift-ring-builder\", line 8, in \u003cmodule\u003e\n    sys.exit(error_handling_main())\n  File \"/vagrant/swift/swift/cli/ringbuilder.py\", line 1786, in error_handling_main\n    main()\n  File \"/vagrant/swift/swift/cli/ringbuilder.py\", line 1768, in main\n    getattr(Commands, command, Commands.unknown)()\n  File \"/vagrant/swift/swift/cli/ringbuilder.py\", line 687, in version\n    if reader:\nUnboundLocalError: local variable \u0027reader\u0027 referenced before assignment\n```","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ef4242d4b8cf95ca87cb59725093fd81a19dbfc5","unresolved":true,"context_lines":[{"line_number":685,"context_line":"            exit(EXIT_ERROR)"},{"line_number":686,"context_line":"        finally:"},{"line_number":687,"context_line":"            if reader:"},{"line_number":688,"context_line":"                reader.close()"},{"line_number":689,"context_line":"        print(\u0027%s: Serialization version: %d (%d-byte IDs), \u0027"},{"line_number":690,"context_line":"              \u0027build version: %d\u0027 %"},{"line_number":691,"context_line":"              (argv[1], serialization_version, dev_id_bytes, version))"}],"source_content_type":"text/x-python","patch_set":34,"id":"5c7be4dd_d05ed17c","line":688,"in_reply_to":"38d92f9a_07dd2646","updated":"2025-06-17 02:56:47.000000000","message":"How much of that do you want cleaned up? The `UnboundLocalError` is fixed, but we still expect you to actually pass in a ring, not a builder. *Maybe* the subcommand could make some sense of a builder? But it still can\u0027t do anything for `Serialization version` -- which was the impetus for adding the subcommand...","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":false,"context_lines":[{"line_number":685,"context_line":"            exit(EXIT_ERROR)"},{"line_number":686,"context_line":"        finally:"},{"line_number":687,"context_line":"            if reader:"},{"line_number":688,"context_line":"                reader.close()"},{"line_number":689,"context_line":"        print(\u0027%s: Serialization version: %d (%d-byte IDs), \u0027"},{"line_number":690,"context_line":"              \u0027build version: %d\u0027 %"},{"line_number":691,"context_line":"              (argv[1], serialization_version, dev_id_bytes, version))"}],"source_content_type":"text/x-python","patch_set":34,"id":"d7312ea7_3d2fb04d","line":688,"in_reply_to":"5c7be4dd_d05ed17c","updated":"2025-06-18 22:21:44.000000000","message":"Done","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":48,"context_line":"EXIT_WARNING \u003d 1"},{"line_number":49,"context_line":"EXIT_ERROR \u003d 2"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"FORMAT_CHOICES \u003d [str(v) for v in RING_CODECS]"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"global argv, backup_dir, builder, builder_file, ring_file"},{"line_number":54,"context_line":"argv \u003d backup_dir \u003d builder \u003d builder_file \u003d ring_file \u003d None"}],"source_content_type":"text/x-python","patch_set":36,"id":"e86ab818_8f98bc58","line":51,"updated":"2025-06-13 21:04:18.000000000","message":"I\u0027m not sure `--format\u003d0` should be valid/allowed.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":false,"context_lines":[{"line_number":48,"context_line":"EXIT_WARNING \u003d 1"},{"line_number":49,"context_line":"EXIT_ERROR \u003d 2"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"FORMAT_CHOICES \u003d [str(v) for v in RING_CODECS]"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"global argv, backup_dir, builder, builder_file, ring_file"},{"line_number":54,"context_line":"argv \u003d backup_dir \u003d builder \u003d builder_file \u003d ring_file \u003d None"}],"source_content_type":"text/x-python","patch_set":36,"id":"90514b87_93cd152b","line":51,"in_reply_to":"2c9ae383_b9d52e32","updated":"2025-07-30 07:49:51.000000000","message":"ringv0 has been removed","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"de7534f73d62d896821b2cfb5d129bc1248ee677","unresolved":true,"context_lines":[{"line_number":48,"context_line":"EXIT_WARNING \u003d 1"},{"line_number":49,"context_line":"EXIT_ERROR \u003d 2"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"FORMAT_CHOICES \u003d [str(v) for v in RING_CODECS]"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"global argv, backup_dir, builder, builder_file, ring_file"},{"line_number":54,"context_line":"argv \u003d backup_dir \u003d builder \u003d builder_file \u003d ring_file \u003d None"}],"source_content_type":"text/x-python","patch_set":36,"id":"2c9ae383_b9d52e32","line":51,"in_reply_to":"6f201d34_db114f47","updated":"2025-07-17 05:44:09.000000000","message":"Tim\u0027s got a patch to remove it.. should we just remove ringv0 then. And make sure we mention it in the change log.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":48,"context_line":"EXIT_WARNING \u003d 1"},{"line_number":49,"context_line":"EXIT_ERROR \u003d 2"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"FORMAT_CHOICES \u003d [str(v) for v in RING_CODECS]"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"global argv, backup_dir, builder, builder_file, ring_file"},{"line_number":54,"context_line":"argv \u003d backup_dir \u003d builder \u003d builder_file \u003d ring_file \u003d None"}],"source_content_type":"text/x-python","patch_set":36,"id":"6f201d34_db114f47","line":51,"in_reply_to":"e68627f9_1803daf3","updated":"2025-06-18 22:21:44.000000000","message":"\u003e we should just get rid of it\nI think that\u0027s scope creep - although we probably *could* get rid of the legacy format it\u0027s probably not optimal to try to do that as part of \"introduce ringv2\"\n\n\u003e make it reasonably not-painful to actually test\n\nI went with a class boolean `unsafe_allow_write_pickle_for_testing`\n\nhttps://review.opendev.org/c/openstack/swift/+/952613/1/swift/common/ring/io.py#232\n\nI\u0027d be ok with `--format\u003d0 --unsafe-allow-write-pickle-for-testing` but it wasn\u0027t obvious to me it\u0027d be worth it.  swift-ring-builder rebalance/write_ring can\u0027t *currently* write a pickle ring, so I assumed we don\u0027t need to ADD that functionality as part of introducing a v2 ring format.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":48,"context_line":"EXIT_WARNING \u003d 1"},{"line_number":49,"context_line":"EXIT_ERROR \u003d 2"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"FORMAT_CHOICES \u003d [str(v) for v in RING_CODECS]"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"global argv, backup_dir, builder, builder_file, ring_file"},{"line_number":54,"context_line":"argv \u003d backup_dir \u003d builder \u003d builder_file \u003d ring_file \u003d None"}],"source_content_type":"text/x-python","patch_set":36,"id":"e68627f9_1803daf3","line":51,"in_reply_to":"e86ab818_8f98bc58","updated":"2025-06-16 23:28:49.000000000","message":"If the format is dead enough that we don\u0027t want to support it, we should just get rid of it. If it\u0027s something we still need to support, we should make it reasonably not-painful to actually test that we **do** still support it.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"}],"swift/common/ring/builder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":101,"context_line":"        self.parts \u003d 2 ** self.part_power"},{"line_number":102,"context_line":"        self.devs \u003d []"},{"line_number":103,"context_line":"        self.devs_changed \u003d False"},{"line_number":104,"context_line":"        self._dev_id_bytes \u003d 1"},{"line_number":105,"context_line":"        self.version \u003d 0"},{"line_number":106,"context_line":"        self.overload \u003d 0.0"},{"line_number":107,"context_line":"        self._id \u003d None"}],"source_content_type":"text/x-python","patch_set":3,"id":"8e008c9a_ec2941ee","line":104,"updated":"2022-04-23 05:36:19.000000000","message":"Clay\u0027s comment at https://review.opendev.org/c/openstack/swift/+/808533/9/swift/common/ring/builder.py#101 still applies:\n\n\u003e it\u0027s akward maybe to keep this piece of state consistent across the lifetime of the object","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":101,"context_line":"        self.parts \u003d 2 ** self.part_power"},{"line_number":102,"context_line":"        self.devs \u003d []"},{"line_number":103,"context_line":"        self.devs_changed \u003d False"},{"line_number":104,"context_line":"        self._dev_id_bytes \u003d 1"},{"line_number":105,"context_line":"        self.version \u003d 0"},{"line_number":106,"context_line":"        self.overload \u003d 0.0"},{"line_number":107,"context_line":"        self._id \u003d None"}],"source_content_type":"text/x-python","patch_set":3,"id":"a68c08e0_8eed0341","line":104,"in_reply_to":"77895e0a_2a0df650","updated":"2022-06-08 23:25:18.000000000","message":"Done","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":true,"context_lines":[{"line_number":101,"context_line":"        self.parts \u003d 2 ** self.part_power"},{"line_number":102,"context_line":"        self.devs \u003d []"},{"line_number":103,"context_line":"        self.devs_changed \u003d False"},{"line_number":104,"context_line":"        self._dev_id_bytes \u003d 1"},{"line_number":105,"context_line":"        self.version \u003d 0"},{"line_number":106,"context_line":"        self.overload \u003d 0.0"},{"line_number":107,"context_line":"        self._id \u003d None"}],"source_content_type":"text/x-python","patch_set":3,"id":"77895e0a_2a0df650","line":104,"in_reply_to":"8e008c9a_ec2941ee","updated":"2022-05-13 06:40:27.000000000","message":"Hmm, yeah ok. If we\u0027re going to calulate this on every write then I guess it could only be a serialization value and not actually live in the object.\nI want to work on getting this landed so we need to make a decsion, Clay does make a point.\n\nThere are a bunch of blobs that need this value now, replica2part2dev and past2part2dev, depending on how we serialize it we could just make it something we store in a ringlike blob, and keep it out of metadata altogether? Or keep it in the metadata but don\u0027t pull it into the Ring and Ringbuilder objects.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":164,"context_line":"    def dev_id_bytes(self, new_dev_id_bytes):"},{"line_number":165,"context_line":"        # TODO: should this really be a property? Needing to resize arrays"},{"line_number":166,"context_line":"        # seems like it might warrant a proper function (and it would mirror"},{"line_number":167,"context_line":"        # RingData...)"},{"line_number":168,"context_line":"        if self._replica2part2dev:"},{"line_number":169,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":170,"context_line":"                resize_array(p2d, new_dev_id_bytes)"}],"source_content_type":"text/x-python","patch_set":3,"id":"2dcd84ab_df47eace","line":167,"updated":"2022-04-23 05:36:19.000000000","message":"^^^","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":true,"context_lines":[{"line_number":164,"context_line":"    def dev_id_bytes(self, new_dev_id_bytes):"},{"line_number":165,"context_line":"        # TODO: should this really be a property? Needing to resize arrays"},{"line_number":166,"context_line":"        # seems like it might warrant a proper function (and it would mirror"},{"line_number":167,"context_line":"        # RingData...)"},{"line_number":168,"context_line":"        if self._replica2part2dev:"},{"line_number":169,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":170,"context_line":"                resize_array(p2d, new_dev_id_bytes)"}],"source_content_type":"text/x-python","patch_set":3,"id":"a6caadef_05f4a837","line":167,"in_reply_to":"2dcd84ab_df47eace","updated":"2022-05-13 06:40:27.000000000","message":"After attempting the push a bunch of the builder strcutures in the ring later in this chain all I can say is.. YES, let\u0027s keep the builder and ring apis are close as we can!\n\nTurn this into a function or turn the ring\u0027s into a setter... but maybe force someone to use a function because it does resize the array, so we want to make sure someone needs to know what they\u0027re doing by doing it explicitly.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":164,"context_line":"    def dev_id_bytes(self, new_dev_id_bytes):"},{"line_number":165,"context_line":"        # TODO: should this really be a property? Needing to resize arrays"},{"line_number":166,"context_line":"        # seems like it might warrant a proper function (and it would mirror"},{"line_number":167,"context_line":"        # RingData...)"},{"line_number":168,"context_line":"        if self._replica2part2dev:"},{"line_number":169,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":170,"context_line":"                resize_array(p2d, new_dev_id_bytes)"}],"source_content_type":"text/x-python","patch_set":3,"id":"f8a53299_f2987032","line":167,"in_reply_to":"a6caadef_05f4a837","updated":"2022-06-08 23:25:18.000000000","message":"Done","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":1046,"context_line":"            self.devs.pop()"},{"line_number":1047,"context_line":""},{"line_number":1048,"context_line":"        # Consider shrinking the device IDs themselves"},{"line_number":1049,"context_line":"        new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1050,"context_line":"        new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"},{"line_number":1051,"context_line":"        # Only shrink if the IDs all fit in the lower half of the next size"},{"line_number":1052,"context_line":"        # down; this avoids excess churn when adding/removing devices near"}],"source_content_type":"text/x-python","patch_set":3,"id":"6bb11181_31939412","line":1049,"updated":"2022-04-23 05:36:19.000000000","message":"Can go to zero...","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"86f15c7640ac7f8bebe0a3a47d284392cddffb6e","unresolved":false,"context_lines":[{"line_number":1046,"context_line":"            self.devs.pop()"},{"line_number":1047,"context_line":""},{"line_number":1048,"context_line":"        # Consider shrinking the device IDs themselves"},{"line_number":1049,"context_line":"        new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1050,"context_line":"        new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"},{"line_number":1051,"context_line":"        # Only shrink if the IDs all fit in the lower half of the next size"},{"line_number":1052,"context_line":"        # down; this avoids excess churn when adding/removing devices near"}],"source_content_type":"text/x-python","patch_set":3,"id":"03f2e355_d284e829","line":1049,"in_reply_to":"6bb11181_31939412","updated":"2022-04-29 00:14:56.000000000","message":"Ack","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":1047,"context_line":""},{"line_number":1048,"context_line":"        # Consider shrinking the device IDs themselves"},{"line_number":1049,"context_line":"        new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1050,"context_line":"        new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"},{"line_number":1051,"context_line":"        # Only shrink if the IDs all fit in the lower half of the next size"},{"line_number":1052,"context_line":"        # down; this avoids excess churn when adding/removing devices near"},{"line_number":1053,"context_line":"        # the limit of a particular dev_id_bytes"}],"source_content_type":"text/x-python","patch_set":3,"id":"88f50980_d49366f4","line":1050,"updated":"2022-04-23 05:36:19.000000000","message":"... in which case this is *also* zero...","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"86f15c7640ac7f8bebe0a3a47d284392cddffb6e","unresolved":false,"context_lines":[{"line_number":1047,"context_line":""},{"line_number":1048,"context_line":"        # Consider shrinking the device IDs themselves"},{"line_number":1049,"context_line":"        new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1050,"context_line":"        new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"},{"line_number":1051,"context_line":"        # Only shrink if the IDs all fit in the lower half of the next size"},{"line_number":1052,"context_line":"        # down; this avoids excess churn when adding/removing devices near"},{"line_number":1053,"context_line":"        # the limit of a particular dev_id_bytes"}],"source_content_type":"text/x-python","patch_set":3,"id":"4a6e4e3f_75e88443","line":1050,"in_reply_to":"88f50980_d49366f4","updated":"2022-04-29 00:14:56.000000000","message":"Ack","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":1051,"context_line":"        # Only shrink if the IDs all fit in the lower half of the next size"},{"line_number":1052,"context_line":"        # down; this avoids excess churn when adding/removing devices near"},{"line_number":1053,"context_line":"        # the limit of a particular dev_id_bytes"},{"line_number":1054,"context_line":"        if len(self.devs) \u003c new_none_dev_id // 2:"},{"line_number":1055,"context_line":"            self.dev_id_bytes \u003d new_dev_id_bytes"},{"line_number":1056,"context_line":""},{"line_number":1057,"context_line":"        return removed_devs"}],"source_content_type":"text/x-python","patch_set":3,"id":"d5ca1489_929f36bb","line":1054,"updated":"2022-04-23 05:36:19.000000000","message":"... so this won\u0027t be true.\n\nStill, might be better to do something like\n\n if self.dev_id_bytes \u003e 1:\n     # Consider shrinking the device IDs themselves\n     ...\n\nso we don\u0027t have to spot-check the logic every time we\u0027re reading through.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"86f15c7640ac7f8bebe0a3a47d284392cddffb6e","unresolved":false,"context_lines":[{"line_number":1051,"context_line":"        # Only shrink if the IDs all fit in the lower half of the next size"},{"line_number":1052,"context_line":"        # down; this avoids excess churn when adding/removing devices near"},{"line_number":1053,"context_line":"        # the limit of a particular dev_id_bytes"},{"line_number":1054,"context_line":"        if len(self.devs) \u003c new_none_dev_id // 2:"},{"line_number":1055,"context_line":"            self.dev_id_bytes \u003d new_dev_id_bytes"},{"line_number":1056,"context_line":""},{"line_number":1057,"context_line":"        return removed_devs"}],"source_content_type":"text/x-python","patch_set":3,"id":"4c2e9d4e_2999154c","line":1054,"in_reply_to":"d5ca1489_929f36bb","updated":"2022-04-29 00:14:56.000000000","message":"Done","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if self._replica2part2dev:"},{"line_number":166,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":167,"context_line":"                resize_array(p2d, new_dev_id_bytes)"},{"line_number":168,"context_line":"                for p2d in self._replica2part2dev]"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    @property"},{"line_number":171,"context_line":"    def dev_id_type_code(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"ad153b6a_7e4afc0c","line":168,"updated":"2022-07-27 22:25:32.000000000","message":"this isn\u0027t your typical getter/setter...\n\n\t(Pdb) builder.dev_id_bytes\n\t2\n\t(Pdb) builder.set_dev_id_bytes(4)\n\t(Pdb) builder.dev_id_bytes\n\t4\n\t(Pdb) builder._replica2part2dev \u003d None\n\t(Pdb) builder.dev_id_bytes\n\t1","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if self._replica2part2dev:"},{"line_number":166,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":167,"context_line":"                resize_array(p2d, new_dev_id_bytes)"},{"line_number":168,"context_line":"                for p2d in self._replica2part2dev]"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    @property"},{"line_number":171,"context_line":"    def dev_id_type_code(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"7aabc9ae_dbc3c015","line":168,"in_reply_to":"3261acfe_81b384bb","updated":"2023-03-29 00:40:00.000000000","message":"I think the source of truth for this property is *either* _replica2part2dev[0].itemsize OR cacl_dev_id_bytes(len(devs))\n\n... i was trying to demonstrate it\u0027s obviously a leaky abstraction and maybe it\u0027s not good to mix the two concepts.  When code wants builder.dev_id_bytes (or to change it!?) maybe it should have an interface that makes it more obvious if we\u0027re sampling the data structure or counting devices (or both?  maybe they\u0027re different in releavant ways)","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":165,"context_line":"        if self._replica2part2dev:"},{"line_number":166,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":167,"context_line":"                resize_array(p2d, new_dev_id_bytes)"},{"line_number":168,"context_line":"                for p2d in self._replica2part2dev]"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    @property"},{"line_number":171,"context_line":"    def dev_id_type_code(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"0c591ff0_037b6948","line":168,"in_reply_to":"38052539_3a7b5c98","updated":"2025-06-13 21:04:18.000000000","message":"I think I understand the motivations here - hopefully we can agree on the most obvious way to express/handle the concept of dev_id_bytes when lacking a replica2part2dev_id table.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if self._replica2part2dev:"},{"line_number":166,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":167,"context_line":"                resize_array(p2d, new_dev_id_bytes)"},{"line_number":168,"context_line":"                for p2d in self._replica2part2dev]"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    @property"},{"line_number":171,"context_line":"    def dev_id_type_code(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"38052539_3a7b5c98","line":168,"in_reply_to":"7aabc9ae_dbc3c015","updated":"2023-04-03 23:05:30.000000000","message":"The ground-truth is that `dev_id_bytes` *as a concept* is the `itemsize`. If you don\u0027t have a `_replica2part2dev_id` table, a whole bunch of things basically stop making sense, which is why I\u0027m kind of OK with the no-op in `set_dev_id_bytes`:\n\n (Pdb) builder._replica2part2dev \u003d None\n (Pdb) builder.dev_id_bytes\n 1\n (Pdb) builder.set_dev_id_bytes(4)\n (Pdb) builder.dev_id_bytes\n 1\n\nThe fallback when we *don\u0027t* have a `_replica2part2dev_id` table was just a nicety -- a \"reasonable\" answer when you\u0027ve never done a rebalance. Maybe it\u0027d be better to just raise an error instead?\n\nIt\u0027d add some (unnecessarily?) sharp edges to `none_dev_id` and `max_dev_id` though.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        if self._replica2part2dev:"},{"line_number":166,"context_line":"            self._replica2part2dev \u003d ["},{"line_number":167,"context_line":"                resize_array(p2d, new_dev_id_bytes)"},{"line_number":168,"context_line":"                for p2d in self._replica2part2dev]"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    @property"},{"line_number":171,"context_line":"    def dev_id_type_code(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"3261acfe_81b384bb","line":168,"in_reply_to":"ad153b6a_7e4afc0c","updated":"2022-07-28 18:28:42.000000000","message":"No, it\u0027s not -- which is part of why it\u0027s not implemented as\n\n builder.dev_id_bytes \u003d 4\n\nThe data structure is the source of truth for the property -- makes perfect sense to me that if you obliterate the data structure, you\u0027re going to start getting weird answers when you query the property.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":211,"context_line":"        try:"},{"line_number":212,"context_line":"            yield"},{"line_number":213,"context_line":"        finally:"},{"line_number":214,"context_line":"            self.logger.disabled \u003d True"},{"line_number":215,"context_line":""},{"line_number":216,"context_line":"    @property"},{"line_number":217,"context_line":"    def min_part_seconds_left(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"67b0e1c1_fab27217","line":214,"updated":"2022-07-27 22:25:32.000000000","message":"logging from w/i the ring module is sort of weird...","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":211,"context_line":"        try:"},{"line_number":212,"context_line":"            yield"},{"line_number":213,"context_line":"        finally:"},{"line_number":214,"context_line":"            self.logger.disabled \u003d True"},{"line_number":215,"context_line":""},{"line_number":216,"context_line":"    @property"},{"line_number":217,"context_line":"    def min_part_seconds_left(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"79156893_0aa7c7d9","line":214,"in_reply_to":"67b0e1c1_fab27217","updated":"2022-07-28 18:28:42.000000000","message":"Not sure what that has to do with this patch... it\u0027s been around a while; see https://github.com/openstack/swift/commit/5070869a and https://github.com/openstack/swift/commit/8d3b3b2e","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":211,"context_line":"        try:"},{"line_number":212,"context_line":"            yield"},{"line_number":213,"context_line":"        finally:"},{"line_number":214,"context_line":"            self.logger.disabled \u003d True"},{"line_number":215,"context_line":""},{"line_number":216,"context_line":"    @property"},{"line_number":217,"context_line":"    def min_part_seconds_left(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"cd77d05d_adf7f8f7","line":214,"in_reply_to":"79156893_0aa7c7d9","updated":"2023-03-29 00:40:00.000000000","message":"maybe i just tried to do some logging during testing/debugging and forgot i had to enable my logger.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c40c0cc14a628562037366a1b1e70adc2ec0d0da","unresolved":false,"context_lines":[{"line_number":211,"context_line":"        try:"},{"line_number":212,"context_line":"            yield"},{"line_number":213,"context_line":"        finally:"},{"line_number":214,"context_line":"            self.logger.disabled \u003d True"},{"line_number":215,"context_line":""},{"line_number":216,"context_line":"    @property"},{"line_number":217,"context_line":"    def min_part_seconds_left(self):"}],"source_content_type":"text/x-python","patch_set":8,"id":"537c5eb8_8565da2b","line":214,"in_reply_to":"cd77d05d_adf7f8f7","updated":"2023-05-03 22:58:40.000000000","message":"Ack","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0e5c3a59ce83ff7b26cc5c736bf8bc6a61619772","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":12,"id":"beb480a0_65af082c","line":445,"updated":"2022-08-25 19:38:45.000000000","message":"This has implications for rollbacks -- if you try the new code in a small cluster, the builder may shrink dev_id_bytes. If you later try to rebalance with older code, you get\n\nTraceback (most recent call last):\n  File \".../bin/swift-ring-builder\", line 7, in \u003cmodule\u003e\n    exec(compile(f.read(), __file__, \u0027exec\u0027))\n  File \".../bin/swift-ring-builder\", line 37, in \u003cmodule\u003e\n    sys.exit(main())\n  File \".../swift/cli/ringbuilder.py\", line 1688, in main\n    getattr(Commands, command, Commands.unknown)()\n  File \".../swift/cli/ringbuilder.py\", line 1073, in rebalance\n    parts, balance, removed_devs \u003d builder.rebalance(seed\u003dget_seed(3))\n  File \".../swift/common/ring/builder.py\", line 564, in rebalance\n    self._gather_parts_for_dispersion(assign_parts, replica_plan)\n  File \".../swift/common/ring/builder.py\", line 1126, in _gather_parts_for_dispersion\n    self._replica2part2dev[replica][part] \u003d NONE_DEV\nOverflowError: unsigned byte integer is greater than maximum\n\nWould it be better for us to\n\n* require that operators opt in to changing the width, likely with a new set_dev_id_bytes subcommand,\n* only allow builders to use 2- or 4-byte widths (while still allowing rings to go down to 1-byte), or\n* just not worry about it?","commit_id":"00949fa087706962da9315a1ba7b7a313c46ecf0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":12,"id":"a9e3fa7d_74f77c80","line":445,"in_reply_to":"beb480a0_65af082c","updated":"2022-10-21 20:02:17.000000000","message":"Done -- went with the second option; builders can now only shrink 4-\u003e2 byte widths.","commit_id":"00949fa087706962da9315a1ba7b7a313c46ecf0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":173,"context_line":""},{"line_number":174,"context_line":"    @property"},{"line_number":175,"context_line":"    def max_dev_id(self):"},{"line_number":176,"context_line":"        return none_dev_id(self.dev_id_bytes) - 1"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"    @property"},{"line_number":179,"context_line":"    def none_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"4591b1e0_c974ce44","line":176,"updated":"2023-03-29 00:40:00.000000000","message":"how is this value different when we look at replica2part2dev_id vs calc_dev_id_bytes(len(devs)) - when is that difference significant?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":173,"context_line":""},{"line_number":174,"context_line":"    @property"},{"line_number":175,"context_line":"    def max_dev_id(self):"},{"line_number":176,"context_line":"        return none_dev_id(self.dev_id_bytes) - 1"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"    @property"},{"line_number":179,"context_line":"    def none_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"853efb28_169731b0","line":176,"in_reply_to":"0975c5fd_e158582d","updated":"2025-08-04 07:10:27.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":true,"context_lines":[{"line_number":173,"context_line":""},{"line_number":174,"context_line":"    @property"},{"line_number":175,"context_line":"    def max_dev_id(self):"},{"line_number":176,"context_line":"        return none_dev_id(self.dev_id_bytes) - 1"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"    @property"},{"line_number":179,"context_line":"    def none_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"0975c5fd_e158582d","line":176,"in_reply_to":"0d226dff_9145af22","updated":"2025-08-01 00:08:57.000000000","message":"\u003e not actually be the max_dev_id we support\n\na doc string would be great - I don\u0027t *think* anyone is going to get confused about self.max_dev_id vs whatever wide-device-ids actual limit is... I do think we have a TooManyDevices error somewhere.  We do have:\n\n```\ndiff --git a/swift/common/ring/utils.py b/swift/common/ring/utils.py\nindex bdb0e9692..758112f2a 100644\n--- a/swift/common/ring/utils.py\n+++ b/swift/common/ring/utils.py\n@@ -54,6 +54,7 @@ def calc_dev_id_bytes(max_dev_id):\n         if max_dev_id \u003c none_dev_id(x):\n             return x\n     else:\n+        asdf\n         # \u003e 4B devices??\n         raise exceptions.DevIdBytesTooSmall(\u0027Way too many devices!\u0027)\n \n```\n\nbut, I\u0027m not sure it\u0027s tested...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":173,"context_line":""},{"line_number":174,"context_line":"    @property"},{"line_number":175,"context_line":"    def max_dev_id(self):"},{"line_number":176,"context_line":"        return none_dev_id(self.dev_id_bytes) - 1"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"    @property"},{"line_number":179,"context_line":"    def none_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"754379c2_c0657f63","line":176,"in_reply_to":"4591b1e0_c974ce44","updated":"2025-06-13 21:04:18.000000000","message":"before initial re-balance is when a builder won\u0027t have a replica2part2dev_id array\n\n... so if you add 64K + 1 devices before you rebalance the add_device code will want to know max_dev_id\n\nAlthough I\u0027m not sure *why* - it \"could* \"just\" add the device to the devices list and wait until you create the arrays to calculate `len(devs)`","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":true,"context_lines":[{"line_number":173,"context_line":""},{"line_number":174,"context_line":"    @property"},{"line_number":175,"context_line":"    def max_dev_id(self):"},{"line_number":176,"context_line":"        return none_dev_id(self.dev_id_bytes) - 1"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"    @property"},{"line_number":179,"context_line":"    def none_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"0d226dff_9145af22","line":176,"in_reply_to":"754379c2_c0657f63","updated":"2025-07-30 07:49:51.000000000","message":"Also really getting the max_id for the current dev_id_bytes.. which maybe not actually be the max_dev_id we support. Maybe a doc string to mention this at least?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"e06613c4_d1eeef3a","line":445,"updated":"2023-03-29 00:40:00.000000000","message":"maybe instead of a setter we just need a self.recalc_dev_id_bytes\n\nI don\u0027t know if we do that everytime we add a device, or just right before serialization","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"53f99b55_73560aa9","line":445,"in_reply_to":"21a04175_847e1222","updated":"2025-07-30 07:49:51.000000000","message":"Let\u0027s write up the side-note bug in launchpad, so it isn\u0027t forgotten about.\n\nI can see Clay\u0027s point here. I like belt and braces of making sure the none_dev is confusing etc.. but if someone want to add devices to the builder but not actually rebalance would they expect their ring structure that isn\u0027t using any of the larger devids yet to suddently double in size..\n\nMaybe it doesn\u0027t matter so much as it\u0027s a builder file.. but one day I\u0027d like to intergrate the builder into the ring and in which case maybe resizing the ring arrays only when it needs to makes more sense. I.e. move this  `self.set_dev_id_bytes` to reblance.\n\nBut honestly I don\u0027t mind if thats now or a follow up.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c580074cf2b25292ccf2763967112c0f95042f9f","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"9e3347ab_cff3f328","line":445,"in_reply_to":"31b3cda5_a3858a60","updated":"2025-07-31 07:14:26.000000000","message":"Well that\u0027s true. I do need to put a Null like value in those tables.. and none_dev is what we have... so that\u0027s a good point.\n\nI don\u0027t really like that the history tables are filled with None_ids or that the index table is stuck at 2 bytes so it\u0027ll have to use an node_dev for 2 bytes, so will defintely be wrong. I\u0027d rather a more sparse array. Scipy I think has a sparse array implementation, bet we can\u0027t use that.. or I could look at just unsing a map (I guess just a dict {\u003cpart\u003e: (\u003cnode_id\u003e, \u003cnode_index\u003e)}). But that\u0027ll only help in smaller rebalances (like a bad disk) rather then an expansion where I assume more parts will move.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4b26e6feb579985c28c6f80724cbceab1814d0a4","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"31b3cda5_a3858a60","line":445,"in_reply_to":"53f99b55_73560aa9","updated":"2025-07-30 18:38:26.000000000","message":"So I agree that currently, we *never* write down arrays with `NONE_DEV` in them. We only ever set entries to `NONE_DEV` during rebalance, and they should all be changed to *valid* dev ids by the time we\u0027re writing anything down, whether in rings or builders, so the only opportunity for ambiguity if we *don\u0027t* immediately resize the arrays is in that rebalance code.\n\nBut peeking around the corner: When we add a history table in https://review.opendev.org/c/openstack/swift/+/834621 we have a new array where every part gets a slot: either a valid dev id, or `NONE_DEV`. That *does* get written down, so there\u0027s immediate ambiguity if we don\u0027t resize promptly: if you add a 65,536th device (i.e., dev id 65535) and *don\u0027t* widen the tables, suddenly\n```\ndev_id \u003d self._past2part2dev_id[0][50]\ndev \u003d self.devs[dev_id] if dev_id !\u003d self.none_dev else None\n```\nand\n```\ntry:\n    dev \u003d self.devs[self._past2part2dev_id[0][50]]\nexcept IndexError:\n    dev \u003d None\n```\ndo very different things.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f5d01d7f3b9c50795c5be7073519f94feb9cf493","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"21a04175_847e1222","line":445,"in_reply_to":"73ebdb05_5de4bdc7","updated":"2025-06-18 23:39:11.000000000","message":"\u003e but the arrays don\u0027t need to support that dev_id until they want to assign it partitions\n\nIs this mainly about how our controller goes and adds a bunch of object disks to account/container rings with zero weight or something? I feel like that\u0027s not the norm for other operators; if they\u0027re adding devices, they probably want to use them, so they have *some* weight. Even for us, we\u0027ll still need to widen the A/C rings with the next A/C node ingest -- which typically would happen around the same time anyway.\n\n\u003e we should probably change that validate method to ensure that even if you have 65K devices, there\u0027s no `NONE_DEV` in the replica2part2dev_id table\n\nDone. Also caused me to spot a bug where we could kick over to 4-byte dev ids earlier than needed on initial rebalance.\n\nSide note: It smells like a bug that we don\u0027t call `validate` as part of `rebalance`... the CLI does it post-rebalance, sure, but anyone using this as an SDK (like we are) has to remember to do it, too...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"da980950_fcd0d757","line":445,"in_reply_to":"9e3347ab_cff3f328","updated":"2025-08-01 00:08:57.000000000","message":"\u003e but anyone using this as an SDK (like we are) has to remember to do it, too\n\ngood point, but if we change it I\u0027d recommend a new `safe_rebalance` entry point; I see code like:\n\n```\n        yield builder\n\n        if any(builder.devs) and validate:\n            builder.validate()\n\n```\n\nin our BuilderWrapper.\n\n\u003e we should probably change that validate method to ensure that even if you have 65K devices, there\u0027s no NONE_DEV in the replica2part2dev_id table\n\u003e Done. \n\nNice!\n\n\u003e Also caused me to spot a bug where we could kick over to 4-byte dev ids earlier than needed on initial rebalance\n\nHey look!  Following a smell can have benificial side effects!?\n\n\u003e if you add a 65,536th device (i.e., dev id 65535) and don\u0027t widen the tablestables, suddenly ... do very different things.\n\nThis is the curx of my understanding that we DO want to force the devs list and the r2p2d arrays to act together as some kind atomic unit.  Tim provided the anology of a foreign key constraint.\n\nThis change calls `self.set_dev_id_bytes` (which resize\u0027s the array as a side-effect) in `add_dev` and `_remove_failed_devices` (which happens during rebalance).   This approach is a kind of \"if you always do it right there won\u0027t ever be a problem\" sort of a approach to an application level constraint.\n\nA *very minor* improvement is to encapsulate the invariant - then you train developers if you see devs.append or devs.pop and not `self._make_the_thing_always_true()` then you know it\u0027s a smell.\n\n956312: add dev_id_bytes invariant helper | https://review.opendev.org/c/openstack/swift/+/956312\n\nThe \"right\" way to do it is make an object to encapsulate the entire invariant so that self.devs.append or self.devs.pop just can\u0027t NOT keep array.itemsize in sync.  However since the RingBuilder is already a kind of encapsulation it might not be super high value to go much further than a clear idempotent pattern for enforcing the invariant.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"44a8cbef_8c4de8f5","line":445,"in_reply_to":"da980950_fcd0d757","updated":"2025-08-04 07:10:27.000000000","message":"Yeah, all great points.. and there is a follow up, so we can move disuccsions there. But I\u0027m not against making sure devs.{append,pop} can\u0027t NOT keep itemsize in sync.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"fdca2daa_c382fedc","line":445,"in_reply_to":"e06613c4_d1eeef3a","updated":"2023-04-03 23:05:30.000000000","message":"In practice, a lot of people are using swift-ring-builder, where adding a device implies a reserialization. I suppose we could delay it until there\u0027s a rebalance that actually requires the higher ids, though.\n\nFWIW, I think the main reason I wanted to be a little eager in expanding was that I didn\u0027t want any room to wonder whether that 65,535 entry in `_replica2part2dev_id` should be interpreted as `NONE_DEV` or the just-added device that was going to force you over to 4-byte dev ids.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":442,"context_line":"            raise exceptions.DuplicateDeviceError("},{"line_number":443,"context_line":"                \u0027Duplicate device id: %d\u0027 % dev[\u0027id\u0027])"},{"line_number":444,"context_line":"        if dev[\u0027id\u0027] \u003e self.max_dev_id:"},{"line_number":445,"context_line":"            self.set_dev_id_bytes(calc_dev_id_bytes(dev[\u0027id\u0027]))"},{"line_number":446,"context_line":""},{"line_number":447,"context_line":"        # Add holes to self.devs to ensure self.devs[dev[\u0027id\u0027]] will be the dev"},{"line_number":448,"context_line":"        while dev[\u0027id\u0027] \u003e\u003d len(self.devs):"}],"source_content_type":"text/x-python","patch_set":17,"id":"73ebdb05_5de4bdc7","line":445,"in_reply_to":"fdca2daa_c382fedc","updated":"2025-06-13 21:04:18.000000000","message":"\u003e adding a device implies a reserialization\n\nthat\u0027s *true* - but the arrays don\u0027t need to support that dev_id until they want to assign it partitions.\n\n\u003e I didn\u0027t want any room to wonder whether that 65,535 entry in _replica2part2dev_id should be interpreted as NONE_DEV\n\nthat can ONLY happen during `_gather_parts_for_xxx` (or I guess replica count adjustment) i.e. only during rebalance.\n\nIIRC there\u0027s some special handling to make sure that we never write a `NONE_DEV` to the replica2part2dev_id except while we\u0027re holding parts that we want to reassign, except... I think it wasn\u0027t really spelling it correctly to account for a variable `NONE_DEV`\n\nhttps://github.com/NVIDIA/swift/blob/master/swift/common/ring/builder.py#L750C30-L750C37\n\nso now that `dev_len` can exceed 64K we should probably change that validate method to ensure that even if you have 65K devices, there\u0027s no `NONE_DEV` in the replica2part2dev_id table.\n\nWhere as if you want to rebalance and *assign* \u003e64K devices parts you\u0027ll need to re-size your arrays and bump dev_id_bytes first.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":1047,"context_line":"            # down; this avoids excess churn when adding/removing devices near"},{"line_number":1048,"context_line":"            # the limit of a particular dev_id_bytes"},{"line_number":1049,"context_line":"            if len(self.devs) \u003c new_none_dev_id // 2:"},{"line_number":1050,"context_line":"                self.set_dev_id_bytes(new_dev_id_bytes)"},{"line_number":1051,"context_line":""},{"line_number":1052,"context_line":"        return removed_devs"},{"line_number":1053,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"4a9dffcc_7c301b6e","line":1050,"updated":"2023-03-29 00:40:00.000000000","message":"_gather_parts_from_failed_devices is like *during* rebalance.  I\u0027m surprised/confused why we want to trim the devices list and recalculate dev_id_bytes while we\u0027re shuffling parts around?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1047,"context_line":"            # down; this avoids excess churn when adding/removing devices near"},{"line_number":1048,"context_line":"            # the limit of a particular dev_id_bytes"},{"line_number":1049,"context_line":"            if len(self.devs) \u003c new_none_dev_id // 2:"},{"line_number":1050,"context_line":"                self.set_dev_id_bytes(new_dev_id_bytes)"},{"line_number":1051,"context_line":""},{"line_number":1052,"context_line":"        return removed_devs"},{"line_number":1053,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"64ebd819_3216f2e6","line":1050,"in_reply_to":"095993d0_9c4769d5","updated":"2025-06-13 21:04:18.000000000","message":"\u003e when would be better?\n\npresumably at the end - like after we\u0027ve removed all the devices from the dev list\n\nOr maybe better, never?  Once you\u0027ve gone over 64K devices you get `dev_id_bytes\u003d4` and that won\u0027t ever go down - as always you just start adding Nones to your devs list.  Save \"re-use device id slots and trim the device list\" for another day!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":1047,"context_line":"            # down; this avoids excess churn when adding/removing devices near"},{"line_number":1048,"context_line":"            # the limit of a particular dev_id_bytes"},{"line_number":1049,"context_line":"            if len(self.devs) \u003c new_none_dev_id // 2:"},{"line_number":1050,"context_line":"                self.set_dev_id_bytes(new_dev_id_bytes)"},{"line_number":1051,"context_line":""},{"line_number":1052,"context_line":"        return removed_devs"},{"line_number":1053,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"095993d0_9c4769d5","line":1050,"in_reply_to":"4a9dffcc_7c301b6e","updated":"2023-04-03 23:05:30.000000000","message":"...when would be better?\n\nThis is the spot where we drop a device -- we remove all of its assignments, we set its former entry in `devs` to `None` -- why wouldn\u0027t we trim/resize here, too?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f5d01d7f3b9c50795c5be7073519f94feb9cf493","unresolved":true,"context_lines":[{"line_number":1047,"context_line":"            # down; this avoids excess churn when adding/removing devices near"},{"line_number":1048,"context_line":"            # the limit of a particular dev_id_bytes"},{"line_number":1049,"context_line":"            if len(self.devs) \u003c new_none_dev_id // 2:"},{"line_number":1050,"context_line":"                self.set_dev_id_bytes(new_dev_id_bytes)"},{"line_number":1051,"context_line":""},{"line_number":1052,"context_line":"        return removed_devs"},{"line_number":1053,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"83458262_8ca87771","line":1050,"in_reply_to":"64ebd819_3216f2e6","updated":"2025-06-18 23:39:11.000000000","message":"\u003e like after we\u0027ve removed all the devices from the dev list\n\nThat\u0027s literally where we are!\n\n\u003e Or maybe better, never?\n\nI thought we *liked* rollback strategies!\n\n\u003e Once you\u0027ve gone over 64K devices you get dev_id_bytes\u003d4 and that won\u0027t ever go down\n\nSo that\u0027s almost certainly the behavior you\u0027ll get once you kick over to wide device ids in prod -- once you go over, you\u0027d need to drop to sub-32k devices to actually size down again. This is about making it easier to test:\n- upgrade your testing environment\n- add a device with a stupid-high dev id\n- see that things work (or don\u0027t!)\n- remove the device to roll back to narrow device ids again, so you\u0027re ready to test again\n\n\u003e  Save \"re-use device id slots and trim the device list\"\n\nSo we already have \"re-use device id slots\" -- it\u0027s just that our controller doesn\u0027t do that.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":true,"context_lines":[{"line_number":1047,"context_line":"            # down; this avoids excess churn when adding/removing devices near"},{"line_number":1048,"context_line":"            # the limit of a particular dev_id_bytes"},{"line_number":1049,"context_line":"            if len(self.devs) \u003c new_none_dev_id // 2:"},{"line_number":1050,"context_line":"                self.set_dev_id_bytes(new_dev_id_bytes)"},{"line_number":1051,"context_line":""},{"line_number":1052,"context_line":"        return removed_devs"},{"line_number":1053,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"fb416b60_029a59bc","line":1050,"in_reply_to":"83458262_8ca87771","updated":"2025-07-30 07:49:51.000000000","message":"We only go gone when we\u0027re well into the previous dev_id_bytes. And I think it\u0027s cool it can roll down. What if your decommissing or scaling down and old cluster, seems like a nice feature to me!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":1113,"context_line":"                    new_parts +\u003d 1"},{"line_number":1114,"context_line":"                self._replica2part2dev.append("},{"line_number":1115,"context_line":"                    array(self.dev_id_type_code,"},{"line_number":1116,"context_line":"                          itertools.repeat(self.none_dev_id, desired_length)))"},{"line_number":1117,"context_line":""},{"line_number":1118,"context_line":"        self.logger.debug("},{"line_number":1119,"context_line":"            \"%d new parts and %d removed parts from replica-count change\","}],"source_content_type":"text/x-python","patch_set":17,"id":"53ca6674_5bb4c896","line":1116,"updated":"2023-03-29 00:40:00.000000000","message":"_adjust_replica2part2dev_size seems like would be a reasonable place to trip the devices list and recalculate our dev_id_size/code","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":1113,"context_line":"                    new_parts +\u003d 1"},{"line_number":1114,"context_line":"                self._replica2part2dev.append("},{"line_number":1115,"context_line":"                    array(self.dev_id_type_code,"},{"line_number":1116,"context_line":"                          itertools.repeat(self.none_dev_id, desired_length)))"},{"line_number":1117,"context_line":""},{"line_number":1118,"context_line":"        self.logger.debug("},{"line_number":1119,"context_line":"            \"%d new parts and %d removed parts from replica-count change\","}],"source_content_type":"text/x-python","patch_set":17,"id":"b9ca0687_4c0f1f12","line":1116,"in_reply_to":"53ca6674_5bb4c896","updated":"2023-04-03 23:05:30.000000000","message":"Bu we do this *before* calling `_gather_parts_from_failed_devices`! We haven\u0027t actually removed any devices yet.\n\nI suppose we could swap the order, but `_adjust_replica2part2dev_size` will need to become more robust to encountering unassigned partitions.\n\nMaybe it\u0027d be better to add a new `_adjust_replica2part2dev_width` helper function, and have\n\n # gather parts from replica count adjustment\n self._adjust_replica2part2dev_size(assign_parts)\n # gather parts from failed devices\n removed_devs \u003d self._gather_parts_from_failed_devices(assign_parts)\n # now that we\u0027ve removed devices, see if we can scale down\n self._adjust_replica2part2dev_width()\n\nup in `rebalance`...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f5d01d7f3b9c50795c5be7073519f94feb9cf493","unresolved":true,"context_lines":[{"line_number":1113,"context_line":"                    new_parts +\u003d 1"},{"line_number":1114,"context_line":"                self._replica2part2dev.append("},{"line_number":1115,"context_line":"                    array(self.dev_id_type_code,"},{"line_number":1116,"context_line":"                          itertools.repeat(self.none_dev_id, desired_length)))"},{"line_number":1117,"context_line":""},{"line_number":1118,"context_line":"        self.logger.debug("},{"line_number":1119,"context_line":"            \"%d new parts and %d removed parts from replica-count change\","}],"source_content_type":"text/x-python","patch_set":17,"id":"e85f193f_1d0e1732","line":1116,"in_reply_to":"789e4aa3_4e203421","updated":"2025-06-18 23:39:11.000000000","message":"Huh. So this was unexpected: I decided to break out a `_remove_failed_devices`, because it\u0027s not obvious (to me, at least) that `_gather_parts_from_failed_devices` should go clearing out anything in `self.devs`, while it *does* make sense that\n\n- the place where we `None`-out `self.devs` entries should also be the place where we trim trailing `None`s and\n- the place where we trim trailing `None`s (and thereby get an accurate notion of `max_dev_id`) should be the same place that we (might) make dev-id-width adjustments.\n\nTaking your\n\n\u003e \u003e when would be better?\n\u003e\n\u003e presumably at the end\n\nto heart, I tried sliding that to the end of `rebalance` -- but it broke `test_get_more_nodes`! Had to keep it above the `for gather_count in range(MAX_BALANCE_GATHER_COUNT)`...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":1113,"context_line":"                    new_parts +\u003d 1"},{"line_number":1114,"context_line":"                self._replica2part2dev.append("},{"line_number":1115,"context_line":"                    array(self.dev_id_type_code,"},{"line_number":1116,"context_line":"                          itertools.repeat(self.none_dev_id, desired_length)))"},{"line_number":1117,"context_line":""},{"line_number":1118,"context_line":"        self.logger.debug("},{"line_number":1119,"context_line":"            \"%d new parts and %d removed parts from replica-count change\","}],"source_content_type":"text/x-python","patch_set":17,"id":"39759feb_e84aff08","line":1116,"in_reply_to":"943ee549_c02362d0","updated":"2025-08-04 07:10:27.000000000","message":"Let\u0027s do anything else as a follow up, as discussed ;)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1113,"context_line":"                    new_parts +\u003d 1"},{"line_number":1114,"context_line":"                self._replica2part2dev.append("},{"line_number":1115,"context_line":"                    array(self.dev_id_type_code,"},{"line_number":1116,"context_line":"                          itertools.repeat(self.none_dev_id, desired_length)))"},{"line_number":1117,"context_line":""},{"line_number":1118,"context_line":"        self.logger.debug("},{"line_number":1119,"context_line":"            \"%d new parts and %d removed parts from replica-count change\","}],"source_content_type":"text/x-python","patch_set":17,"id":"789e4aa3_4e203421","line":1116,"in_reply_to":"b9ca0687_4c0f1f12","updated":"2025-06-13 21:04:18.000000000","message":"yeah that would be a lot better.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":true,"context_lines":[{"line_number":1113,"context_line":"                    new_parts +\u003d 1"},{"line_number":1114,"context_line":"                self._replica2part2dev.append("},{"line_number":1115,"context_line":"                    array(self.dev_id_type_code,"},{"line_number":1116,"context_line":"                          itertools.repeat(self.none_dev_id, desired_length)))"},{"line_number":1117,"context_line":""},{"line_number":1118,"context_line":"        self.logger.debug("},{"line_number":1119,"context_line":"            \"%d new parts and %d removed parts from replica-count change\","}],"source_content_type":"text/x-python","patch_set":17,"id":"943ee549_c02362d0","line":1116,"in_reply_to":"e85f193f_1d0e1732","updated":"2025-08-01 00:08:57.000000000","message":"I think we might have been running into the same thing:\n\n956312: add dev_id_bytes invariant helper | https://review.opendev.org/c/openstack/swift/+/956312\n\nI would be happy to do a prefactor to clean up some of this up if you\u0027re down for another rebase.\n\nI\u0027m still squarely of the opinion that rebalance should assert/enforce the dev-length-array-itemsize invariant.  The lazy removal of failed devices from the devs list during rebalance is unavoidably one place that must be considered - KUDOS. What I don\u0027t like is that currently that happens in middle of gathering parts: after we\u0027ve already started putting self.none_dev_id into the arrays!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dad76bead53d34c339b6cf5f904698f6681b5a66","unresolved":true,"context_lines":[{"line_number":1042,"context_line":"        while self.devs and self.devs[-1] is None:"},{"line_number":1043,"context_line":"            self.devs.pop()"},{"line_number":1044,"context_line":""},{"line_number":1045,"context_line":"        if self.dev_id_bytes \u003e 2:"},{"line_number":1046,"context_line":"            # Consider shrinking the device IDs themselves"},{"line_number":1047,"context_line":"            new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1048,"context_line":"            new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"}],"source_content_type":"text/x-python","patch_set":20,"id":"a80518ac_aff29082","line":1045,"range":{"start_line":1045,"start_character":31,"end_line":1045,"end_character":32},"updated":"2023-07-07 19:48:16.000000000","message":"I could see an argument that we should be checking for `\u003e 4` here, so builders can\u0027t go down to `dev_id_bytes \u003d 1` -- just to de-risk rollbacks. Pretty sure that\u0027s the only potential for a permanent change to builders with this patch, assuming you haven\u0027t gone past the 64k limit.","commit_id":"1ff766bcff07c6a038a19518cb569418abe32cbd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"58f759cd839f5087aa1db3fea5109ea81b434b5d","unresolved":true,"context_lines":[{"line_number":1042,"context_line":"        while self.devs and self.devs[-1] is None:"},{"line_number":1043,"context_line":"            self.devs.pop()"},{"line_number":1044,"context_line":""},{"line_number":1045,"context_line":"        if self.dev_id_bytes \u003e 2:"},{"line_number":1046,"context_line":"            # Consider shrinking the device IDs themselves"},{"line_number":1047,"context_line":"            new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1048,"context_line":"            new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"}],"source_content_type":"text/x-python","patch_set":20,"id":"75c72780_00314ed0","line":1045,"range":{"start_line":1045,"start_character":31,"end_line":1045,"end_character":32},"in_reply_to":"0031aac5_afe9e433","updated":"2025-01-15 09:35:33.000000000","message":"We\u0027re used to a minimum of 4 bytes. so happy to keep it that way if it makes it less confusing, esp in the mixed world.","commit_id":"1ff766bcff07c6a038a19518cb569418abe32cbd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":1042,"context_line":"        while self.devs and self.devs[-1] is None:"},{"line_number":1043,"context_line":"            self.devs.pop()"},{"line_number":1044,"context_line":""},{"line_number":1045,"context_line":"        if self.dev_id_bytes \u003e 2:"},{"line_number":1046,"context_line":"            # Consider shrinking the device IDs themselves"},{"line_number":1047,"context_line":"            new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1048,"context_line":"            new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"}],"source_content_type":"text/x-python","patch_set":20,"id":"95acb926_d0b0ef2e","line":1045,"range":{"start_line":1045,"start_character":31,"end_line":1045,"end_character":32},"in_reply_to":"75c72780_00314ed0","updated":"2025-06-13 21:04:18.000000000","message":"this has been changed/fixed - v2 never goes smaller than dev_id_bytes 2 AFAIK","commit_id":"1ff766bcff07c6a038a19518cb569418abe32cbd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3f069ba1d9a2ad8e270b40503f37ad0ae9a5806f","unresolved":true,"context_lines":[{"line_number":1042,"context_line":"        while self.devs and self.devs[-1] is None:"},{"line_number":1043,"context_line":"            self.devs.pop()"},{"line_number":1044,"context_line":""},{"line_number":1045,"context_line":"        if self.dev_id_bytes \u003e 2:"},{"line_number":1046,"context_line":"            # Consider shrinking the device IDs themselves"},{"line_number":1047,"context_line":"            new_dev_id_bytes \u003d self.dev_id_bytes // 2"},{"line_number":1048,"context_line":"            new_none_dev_id \u003d none_dev_id(new_dev_id_bytes)"}],"source_content_type":"text/x-python","patch_set":20,"id":"0031aac5_afe9e433","line":1045,"range":{"start_line":1045,"start_character":31,"end_line":1045,"end_character":32},"in_reply_to":"a80518ac_aff29082","updated":"2023-07-26 00:03:30.000000000","message":"idk, on my vsaio I\u0027m seeing my ring\u0027s dev_id_bytes go from 2 to 1, but my builder already holds at 2\n\nN.B. my ring can go *back* to bytes 2 if I forget to specify format_verison\u003d2 when I call rebalance/write_ring.","commit_id":"1ff766bcff07c6a038a19518cb569418abe32cbd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"        # Trim the dev list"},{"line_number":1039,"context_line":"        while self.devs and self.devs[-1] is None:"},{"line_number":1040,"context_line":"            self.devs.pop()"},{"line_number":1041,"context_line":""},{"line_number":1042,"context_line":"        if self.dev_id_bytes \u003e 2:"},{"line_number":1043,"context_line":"            # Consider shrinking the device IDs themselves"}],"source_content_type":"text/x-python","patch_set":34,"id":"6b0a5a7c_03ac2367","line":1040,"updated":"2025-06-13 21:04:18.000000000","message":"this could definitely happen after we\u0027ve placed all the parts.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":true,"context_lines":[{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"        # Trim the dev list"},{"line_number":1039,"context_line":"        while self.devs and self.devs[-1] is None:"},{"line_number":1040,"context_line":"            self.devs.pop()"},{"line_number":1041,"context_line":""},{"line_number":1042,"context_line":"        if self.dev_id_bytes \u003e 2:"},{"line_number":1043,"context_line":"            # Consider shrinking the device IDs themselves"}],"source_content_type":"text/x-python","patch_set":34,"id":"7e9d3086_9285d3ee","line":1040,"in_reply_to":"6b0a5a7c_03ac2367","updated":"2025-08-01 00:08:57.000000000","message":"I hope I meant after we\u0027ve *gathered* all the parts.  But in reality I think it\u0027s just always safe to trim the device list of trailing None\u0027s - we check to make sure none of the assigned device_ids in r2p2d point a dev entry that is None (and now also that they\u0027re not `self.none_device_id`)\n\nI actually think it would be most helpful to encapsulate this as part of enforcing the invariant on the length of the devs list and the itemsize of the r2p2d arrays:\n\n956312: add dev_id_bytes invariant helper | https://review.opendev.org/c/openstack/swift/+/956312","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"        # Trim the dev list"},{"line_number":1039,"context_line":"        while self.devs and self.devs[-1] is None:"},{"line_number":1040,"context_line":"            self.devs.pop()"},{"line_number":1041,"context_line":""},{"line_number":1042,"context_line":"        if self.dev_id_bytes \u003e 2:"},{"line_number":1043,"context_line":"            # Consider shrinking the device IDs themselves"}],"source_content_type":"text/x-python","patch_set":34,"id":"fa27034e_7a5ee255","line":1040,"in_reply_to":"7e9d3086_9285d3ee","updated":"2025-08-04 07:10:27.000000000","message":"Acknowledged","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":true,"context_lines":[{"line_number":1033,"context_line":"                        self.logger.debug("},{"line_number":1034,"context_line":"                            \"Gathered %d/%d from dev %d [dev removed]\","},{"line_number":1035,"context_line":"                            part, replica, dev_id)"},{"line_number":1036,"context_line":"        return gathered_parts"},{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"    def _remove_failed_devices(self):"},{"line_number":1039,"context_line":"        removed_devs \u003d 0"}],"source_content_type":"text/x-python","patch_set":42,"id":"02f5e549_7cfe3192","line":1036,"updated":"2025-07-30 07:49:51.000000000","message":"Did we also want to debug log the number of gathered parts anywhere. We don\u0027t seem to use this, though I like that is does return it.\n\nBut this is defintley a follow up rather then something that needs to be fixed here unless there is another patchset.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4b26e6feb579985c28c6f80724cbceab1814d0a4","unresolved":true,"context_lines":[{"line_number":1033,"context_line":"                        self.logger.debug("},{"line_number":1034,"context_line":"                            \"Gathered %d/%d from dev %d [dev removed]\","},{"line_number":1035,"context_line":"                            part, replica, dev_id)"},{"line_number":1036,"context_line":"        return gathered_parts"},{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"    def _remove_failed_devices(self):"},{"line_number":1039,"context_line":"        removed_devs \u003d 0"}],"source_content_type":"text/x-python","patch_set":42,"id":"8b9eec3f_07d0208a","line":1036,"in_reply_to":"02f5e549_7cfe3192","updated":"2025-07-30 18:38:26.000000000","message":"So we\u0027ve got `Gathered %d parts` up at L614 -- pretty sure that should include these. Might still be interesting to get some breakdown like\n```\nGathered W parts from failed devices\nGathered X parts for dispersion\nGathered Y parts for balance (attempt 1)\nGathered Z parts for balance (attempt 2)\n...\n```","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c580074cf2b25292ccf2763967112c0f95042f9f","unresolved":false,"context_lines":[{"line_number":1033,"context_line":"                        self.logger.debug("},{"line_number":1034,"context_line":"                            \"Gathered %d/%d from dev %d [dev removed]\","},{"line_number":1035,"context_line":"                            part, replica, dev_id)"},{"line_number":1036,"context_line":"        return gathered_parts"},{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"    def _remove_failed_devices(self):"},{"line_number":1039,"context_line":"        removed_devs \u003d 0"}],"source_content_type":"text/x-python","patch_set":42,"id":"6e6dcf77_6534d045","line":1036,"in_reply_to":"8b9eec3f_07d0208a","updated":"2025-07-31 07:14:26.000000000","message":"Acknowledged","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"}],"swift/common/ring/io.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c9b68e79237f7f7698652e1af6d54c43da7fdde1","unresolved":false,"context_lines":[{"line_number":166,"context_line":""},{"line_number":167,"context_line":"        self.load_index()"},{"line_number":168,"context_line":""},{"line_number":169,"context_line":"        self.zlib_fp.seek(0)"},{"line_number":170,"context_line":""},{"line_number":171,"context_line":"    def close(self):"},{"line_number":172,"context_line":"        self.fp.close()"}],"source_content_type":"text/x-python","patch_set":1,"id":"e8bd0cf0_d5bff0ef","line":169,"updated":"2022-03-20 23:33:01.000000000","message":"Love this! So nicely contained. Great work!","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"aa480289bd97277813643a0af771cdf41deea8a2","unresolved":true,"context_lines":[{"line_number":268,"context_line":"        #     compressed end,"},{"line_number":269,"context_line":"        #     uncompressed end,"},{"line_number":270,"context_line":"        #     sha256"},{"line_number":271,"context_line":"        #   ]"},{"line_number":272,"context_line":"        self.index \u003d {}"},{"line_number":273,"context_line":"        self.flushed \u003d True"},{"line_number":274,"context_line":"        self.hasher \u003d None"}],"source_content_type":"text/x-python","patch_set":1,"id":"891fd521_44e347b7","line":271,"updated":"2022-03-22 05:06:20.000000000","message":"I wonder if we should have:\n  ...\n  hash,\n  digest]\n\nSo we\u0027re more future proof?","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b938e76532bc222772ace8612d57f2317d4df0cd","unresolved":false,"context_lines":[{"line_number":268,"context_line":"        #     compressed end,"},{"line_number":269,"context_line":"        #     uncompressed end,"},{"line_number":270,"context_line":"        #     sha256"},{"line_number":271,"context_line":"        #   ]"},{"line_number":272,"context_line":"        self.index \u003d {}"},{"line_number":273,"context_line":"        self.flushed \u003d True"},{"line_number":274,"context_line":"        self.hasher \u003d None"}],"source_content_type":"text/x-python","patch_set":1,"id":"be4446a4_155f021a","line":271,"in_reply_to":"7e50ff11_93dbcfc7","updated":"2022-04-23 01:06:14.000000000","message":"Done","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"51e6e7c9da4b3859d48cd5d75611a10dffe57cf5","unresolved":true,"context_lines":[{"line_number":268,"context_line":"        #     compressed end,"},{"line_number":269,"context_line":"        #     uncompressed end,"},{"line_number":270,"context_line":"        #     sha256"},{"line_number":271,"context_line":"        #   ]"},{"line_number":272,"context_line":"        self.index \u003d {}"},{"line_number":273,"context_line":"        self.flushed \u003d True"},{"line_number":274,"context_line":"        self.hasher \u003d None"}],"source_content_type":"text/x-python","patch_set":1,"id":"7e50ff11_93dbcfc7","line":271,"in_reply_to":"891fd521_44e347b7","updated":"2022-03-23 18:39:54.000000000","message":"Good idea!","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c9b68e79237f7f7698652e1af6d54c43da7fdde1","unresolved":true,"context_lines":[{"line_number":345,"context_line":"            self.hasher \u003d None"},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"    def write_blob(self, data, fmt\u003d\"!Q\"):"},{"line_number":348,"context_line":"        self.write(struct.pack(fmt, len(data)) + data)"},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"    def write_json(self, data, fmt\u003d\"!Q\"):"},{"line_number":351,"context_line":"        json_data \u003d json.dumps(data, sort_keys\u003dTrue, ensure_ascii\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":1,"id":"b84f1259_8b953813","line":348,"updated":"2022-03-20 23:33:01.000000000","message":"do we really need to encode the length of each blob into the file. Doesn\u0027t we know this from the index. So in theory we just need to write blogs right? The length should be inferred from the start - end. Or we could even store the length in the index.\n\nOr do we want to have a backup of being able to load data without an index, and then guess the type/section?","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"51e6e7c9da4b3859d48cd5d75611a10dffe57cf5","unresolved":true,"context_lines":[{"line_number":345,"context_line":"            self.hasher \u003d None"},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"    def write_blob(self, data, fmt\u003d\"!Q\"):"},{"line_number":348,"context_line":"        self.write(struct.pack(fmt, len(data)) + data)"},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"    def write_json(self, data, fmt\u003d\"!Q\"):"},{"line_number":351,"context_line":"        json_data \u003d json.dumps(data, sort_keys\u003dTrue, ensure_ascii\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":1,"id":"c2673477_4050de9a","line":348,"in_reply_to":"b84f1259_8b953813","updated":"2022-03-23 18:39:54.000000000","message":"I was thinking about ways in which we could rebuild the ring if it got recompressed without all my fancy zlib stream manipulation -- which is also why I write down both the compressed and uncompressed offsets in the index. I agree, though, that it\u0027s a fine line to walk between having enough redundancy to recover, and not having *so much* redundancy that there\u0027s no clear source of truth.","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":false,"context_lines":[{"line_number":345,"context_line":"            self.hasher \u003d None"},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"    def write_blob(self, data, fmt\u003d\"!Q\"):"},{"line_number":348,"context_line":"        self.write(struct.pack(fmt, len(data)) + data)"},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"    def write_json(self, data, fmt\u003d\"!Q\"):"},{"line_number":351,"context_line":"        json_data \u003d json.dumps(data, sort_keys\u003dTrue, ensure_ascii\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":1,"id":"fe242b41_6dd4a189","line":348,"in_reply_to":"c2673477_4050de9a","updated":"2022-05-13 06:40:27.000000000","message":"Having extra redundency doesn\u0027t hurt and if we remove it, it might bite us later, so let\u0027s just keep it!","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b938e76532bc222772ace8612d57f2317d4df0cd","unresolved":true,"context_lines":[{"line_number":256,"context_line":"    def __init__(self, filename, mtime\u003d1300507380.0):"},{"line_number":257,"context_line":"        self.filename \u003d filename"},{"line_number":258,"context_line":"        self.raw_fp \u003d tempfile.NamedTemporaryFile("},{"line_number":259,"context_line":"            dir\u003dos.path.dirname(os.path.realpath(filename)),"},{"line_number":260,"context_line":"            prefix\u003dfilename,"},{"line_number":261,"context_line":"            delete\u003dFalse)"},{"line_number":262,"context_line":"        self.gzip_fp \u003d gzip.GzipFile("}],"source_content_type":"text/x-python","patch_set":2,"id":"55a4a5c8_4016d88a","line":259,"range":{"start_line":259,"start_character":40,"end_line":259,"end_character":48},"updated":"2022-04-23 01:06:14.000000000","message":"I\u0027m getting skeptical of whether I really want realpath here -- I think maybe I was thinking of normpath? Significantly, if filename is itself a symlink, I think it\u0027s perfectly reasonable to replace the *link*, and leave the target.\n\nI think I was mostly worried about relative paths being supplied? Maybe that\u0027s misplaced; maybe I don\u0027t need anything more than\n\n os.path.dirname(filename)","commit_id":"4d54bb9ec7b2d419da0dff36143a9266fc0819ee"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":256,"context_line":"    def __init__(self, filename, mtime\u003d1300507380.0):"},{"line_number":257,"context_line":"        self.filename \u003d filename"},{"line_number":258,"context_line":"        self.raw_fp \u003d tempfile.NamedTemporaryFile("},{"line_number":259,"context_line":"            dir\u003dos.path.dirname(os.path.realpath(filename)),"},{"line_number":260,"context_line":"            prefix\u003dfilename,"},{"line_number":261,"context_line":"            delete\u003dFalse)"},{"line_number":262,"context_line":"        self.gzip_fp \u003d gzip.GzipFile("}],"source_content_type":"text/x-python","patch_set":2,"id":"f8a0a222_142ae34e","line":259,"range":{"start_line":259,"start_character":40,"end_line":259,"end_character":48},"in_reply_to":"0fc1ed8e_afee6bf4","updated":"2022-06-08 23:25:18.000000000","message":"Done","commit_id":"4d54bb9ec7b2d419da0dff36143a9266fc0819ee"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":true,"context_lines":[{"line_number":256,"context_line":"    def __init__(self, filename, mtime\u003d1300507380.0):"},{"line_number":257,"context_line":"        self.filename \u003d filename"},{"line_number":258,"context_line":"        self.raw_fp \u003d tempfile.NamedTemporaryFile("},{"line_number":259,"context_line":"            dir\u003dos.path.dirname(os.path.realpath(filename)),"},{"line_number":260,"context_line":"            prefix\u003dfilename,"},{"line_number":261,"context_line":"            delete\u003dFalse)"},{"line_number":262,"context_line":"        self.gzip_fp \u003d gzip.GzipFile("}],"source_content_type":"text/x-python","patch_set":2,"id":"0fc1ed8e_afee6bf4","line":259,"range":{"start_line":259,"start_character":40,"end_line":259,"end_character":48},"in_reply_to":"55a4a5c8_4016d88a","updated":"2022-05-13 06:40:27.000000000","message":"Hmm, yeah. The relative paths case is really what I throught you put it in for. I agree with your sysmlink statement though. But what would someone expect, if I give you a path with a symlink, update the target. \n\nDoes norm fix reletive paths? I don\u0027t think so, so maybe we leave it? (I guess I could test this).","commit_id":"4d54bb9ec7b2d419da0dff36143a9266fc0819ee"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"8a0244d2562e348f0213e13e80105e3cbf6e96c7","unresolved":true,"context_lines":[{"line_number":257,"context_line":"        self.filename \u003d filename"},{"line_number":258,"context_line":"        self.raw_fp \u003d tempfile.NamedTemporaryFile("},{"line_number":259,"context_line":"            dir\u003dos.path.dirname(os.path.realpath(filename)),"},{"line_number":260,"context_line":"            prefix\u003dfilename,"},{"line_number":261,"context_line":"            delete\u003dFalse)"},{"line_number":262,"context_line":"        self.gzip_fp \u003d gzip.GzipFile("},{"line_number":263,"context_line":"            filename, mode\u003d\u0027wb\u0027, fileobj\u003dself.raw_fp, mtime\u003dmtime)"}],"source_content_type":"text/x-python","patch_set":2,"id":"0e95eb58_bd86662a","line":260,"range":{"start_line":260,"start_character":19,"end_line":260,"end_character":27},"updated":"2022-04-21 05:41:01.000000000","message":"This needs to be:\n\n prefix\u003dos.path.basename(filename),\n\nOtherwise if can mess up the location. I\u0027ve run into this when trying to write a ring as a builder file and we try and back it up to a backups dir. The filename will come in as backups/blah.ring.gz the realpath above would correct it. But means we also get path thrown in the prefix.","commit_id":"4d54bb9ec7b2d419da0dff36143a9266fc0819ee"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b938e76532bc222772ace8612d57f2317d4df0cd","unresolved":false,"context_lines":[{"line_number":257,"context_line":"        self.filename \u003d filename"},{"line_number":258,"context_line":"        self.raw_fp \u003d tempfile.NamedTemporaryFile("},{"line_number":259,"context_line":"            dir\u003dos.path.dirname(os.path.realpath(filename)),"},{"line_number":260,"context_line":"            prefix\u003dfilename,"},{"line_number":261,"context_line":"            delete\u003dFalse)"},{"line_number":262,"context_line":"        self.gzip_fp \u003d gzip.GzipFile("},{"line_number":263,"context_line":"            filename, mode\u003d\u0027wb\u0027, fileobj\u003dself.raw_fp, mtime\u003dmtime)"}],"source_content_type":"text/x-python","patch_set":2,"id":"21eb487b_c9f5f661","line":260,"range":{"start_line":260,"start_character":19,"end_line":260,"end_character":27},"in_reply_to":"0e95eb58_bd86662a","updated":"2022-04-23 01:06:14.000000000","message":"Done","commit_id":"4d54bb9ec7b2d419da0dff36143a9266fc0819ee"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":true,"context_lines":[{"line_number":69,"context_line":"        always advance (unless we\u0027re already at EOF). The amount buffered can"},{"line_number":70,"context_line":"        vary wildly, from 0 to many multiples of ``DEFAULT_BUFFER_SIZE``."},{"line_number":71,"context_line":"        Callers should call this in a loop and monitor the size of ``buffer``"},{"line_number":72,"context_line":"        and whether we\u0027ve hit EOF; see ``read``."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"        :returns: False if we hit the end of the file, True otherwise"},{"line_number":75,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":4,"id":"f303092f_37441a78","line":72,"range":{"start_line":72,"start_character":35,"end_line":72,"end_character":47},"updated":"2022-05-13 06:40:27.000000000","message":"Is there suppose to be something to read in ``read``? or is this expecting someone to look at the code implementation of read?","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":69,"context_line":"        always advance (unless we\u0027re already at EOF). The amount buffered can"},{"line_number":70,"context_line":"        vary wildly, from 0 to many multiples of ``DEFAULT_BUFFER_SIZE``."},{"line_number":71,"context_line":"        Callers should call this in a loop and monitor the size of ``buffer``"},{"line_number":72,"context_line":"        and whether we\u0027ve hit EOF; see ``read``."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"        :returns: False if we hit the end of the file, True otherwise"},{"line_number":75,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":4,"id":"2be8be27_7e998a11","line":72,"range":{"start_line":72,"start_character":35,"end_line":72,"end_character":47},"in_reply_to":"f303092f_37441a78","updated":"2022-06-08 23:25:18.000000000","message":"I was thinking of the implementation; reworded a bit.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":true,"context_lines":[{"line_number":103,"context_line":"        # we may have *almost* found the flush marker;"},{"line_number":104,"context_line":"        # gotta keep some of the tail"},{"line_number":105,"context_line":"        keep \u003d len(ZLIB_FLUSH_MARKER) - 1"},{"line_number":106,"context_line":"        chunk \u003d self.compressed_buffer[:-keep]"},{"line_number":107,"context_line":"        self.compressed_buffer \u003d self.compressed_buffer[-keep:]"},{"line_number":108,"context_line":"        self.pos +\u003d len(chunk)"},{"line_number":109,"context_line":"        # note that there\u0027s no guarantee that buffer will actually grow --"}],"source_content_type":"text/x-python","patch_set":4,"id":"df21de52_9602e04c","line":106,"updated":"2022-05-13 06:40:27.000000000","message":"So I guess if there is no flush buffer found we\u0027ll always hold back some, but seems it\u0027s only 3 bytes. And to get the benefit of seeking to flush markers 3-bytes are totally worth it.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":103,"context_line":"        # we may have *almost* found the flush marker;"},{"line_number":104,"context_line":"        # gotta keep some of the tail"},{"line_number":105,"context_line":"        keep \u003d len(ZLIB_FLUSH_MARKER) - 1"},{"line_number":106,"context_line":"        chunk \u003d self.compressed_buffer[:-keep]"},{"line_number":107,"context_line":"        self.compressed_buffer \u003d self.compressed_buffer[-keep:]"},{"line_number":108,"context_line":"        self.pos +\u003d len(chunk)"},{"line_number":109,"context_line":"        # note that there\u0027s no guarantee that buffer will actually grow --"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa374cd4_71934e8a","line":106,"in_reply_to":"35f3857d_46cb362f","updated":"2022-10-21 20:02:17.000000000","message":"Ack","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":true,"context_lines":[{"line_number":103,"context_line":"        # we may have *almost* found the flush marker;"},{"line_number":104,"context_line":"        # gotta keep some of the tail"},{"line_number":105,"context_line":"        keep \u003d len(ZLIB_FLUSH_MARKER) - 1"},{"line_number":106,"context_line":"        chunk \u003d self.compressed_buffer[:-keep]"},{"line_number":107,"context_line":"        self.compressed_buffer \u003d self.compressed_buffer[-keep:]"},{"line_number":108,"context_line":"        self.pos +\u003d len(chunk)"},{"line_number":109,"context_line":"        # note that there\u0027s no guarantee that buffer will actually grow --"}],"source_content_type":"text/x-python","patch_set":4,"id":"35f3857d_46cb362f","line":106,"in_reply_to":"df21de52_9602e04c","updated":"2022-06-08 23:25:18.000000000","message":"Yeah, that was my thinking, too.\n\nFWIW, this is to save us some IO/CPU during a linear read, where we\u0027re going to attempt a seek to where we already are. This could all be simplified down to something like\n\n    def _buffer_chunk(self):\n        chunk \u003d self.fp.read(self.chunk_size)\n        if not chunk:\n            return False\n        self.pos +\u003d len(chunk)\n        self.buffer +\u003d self.decompressor.decompress(chunk)\n        return True\n\nbut without the flush-marker detection, we\u0027d never (or at any rate, rarely?) trip the early return in seek.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"13268cfd836813866fcc7b6d3d27e142fee8e0cb","unresolved":true,"context_lines":[{"line_number":320,"context_line":"        if checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":321,"context_line":"            checksum \u003d getattr(hashlib, checksum_method)(prefix)"},{"line_number":322,"context_line":"        else:"},{"line_number":323,"context_line":"            # TODO: log warning about unsupported checksum"},{"line_number":324,"context_line":"            checksum \u003d checksum_value \u003d None"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"        with FixedLengthReader("}],"source_content_type":"text/x-python","patch_set":4,"id":"df9ebf30_beb22db9","line":323,"updated":"2022-05-13 06:40:27.000000000","message":"I guess we should do this todo for completeness sake and so we can land this soon","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":320,"context_line":"        if checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":321,"context_line":"            checksum \u003d getattr(hashlib, checksum_method)(prefix)"},{"line_number":322,"context_line":"        else:"},{"line_number":323,"context_line":"            # TODO: log warning about unsupported checksum"},{"line_number":324,"context_line":"            checksum \u003d checksum_value \u003d None"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"        with FixedLengthReader("}],"source_content_type":"text/x-python","patch_set":4,"id":"ecb10242_77e5fc7b","line":323,"in_reply_to":"df9ebf30_beb22db9","updated":"2022-06-08 23:25:18.000000000","message":"Done","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9104a2463c032f42a0c3b7a044fe180661f3df57","unresolved":true,"context_lines":[{"line_number":465,"context_line":"        if name in self.index:"},{"line_number":466,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":467,"context_line":"        self.flush()"},{"line_number":468,"context_line":"        self.index[name] \u003d [self.tell(), self.pos, None, None, None, None]"},{"line_number":469,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":470,"context_line":"        try:"},{"line_number":471,"context_line":"            yield self"}],"source_content_type":"text/x-python","patch_set":4,"id":"a91e767c_f8e13603","line":468,"range":{"start_line":468,"start_character":33,"end_line":468,"end_character":39},"updated":"2022-05-17 01:39:46.000000000","message":"tell() does a flush so do we need a flush on 467?","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":465,"context_line":"        if name in self.index:"},{"line_number":466,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":467,"context_line":"        self.flush()"},{"line_number":468,"context_line":"        self.index[name] \u003d [self.tell(), self.pos, None, None, None, None]"},{"line_number":469,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":470,"context_line":"        try:"},{"line_number":471,"context_line":"            yield self"}],"source_content_type":"text/x-python","patch_set":4,"id":"c1af7885_401f3961","line":468,"range":{"start_line":468,"start_character":33,"end_line":468,"end_character":39},"in_reply_to":"a91e767c_f8e13603","updated":"2022-06-08 23:25:18.000000000","message":"I figured, better safe than sorry. And with the\n\n if not self.flushed:\n\nguard, there\u0027s no harm to it beyond the extra function call.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":55,"context_line":"        ``os.SEEK_SET`` and the reader is already at the desired position)."},{"line_number":56,"context_line":"        As a result, callers should be careful to ``seek()`` to flush"},{"line_number":57,"context_line":"        boundaries, to ensure that subsequent ``read()`` calls work properly."},{"line_number":58,"context_line":"        \"\"\""},{"line_number":59,"context_line":"        if (pos, whence) \u003d\u003d (self.pos, os.SEEK_SET):"},{"line_number":60,"context_line":"            # small optimization for linear reads"},{"line_number":61,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":8,"id":"3d4a2c44_6adb4414","line":58,"updated":"2022-07-27 22:25:32.000000000","message":"this doesn\u0027t really define the failure mode or how to determine if a pos is a safe flush boundary... since this is a new class I can\u0027t help but wonder if we\u0027ve spent enough time to thinking about other interfaces that might be easier to use?","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":55,"context_line":"        ``os.SEEK_SET`` and the reader is already at the desired position)."},{"line_number":56,"context_line":"        As a result, callers should be careful to ``seek()`` to flush"},{"line_number":57,"context_line":"        boundaries, to ensure that subsequent ``read()`` calls work properly."},{"line_number":58,"context_line":"        \"\"\""},{"line_number":59,"context_line":"        if (pos, whence) \u003d\u003d (self.pos, os.SEEK_SET):"},{"line_number":60,"context_line":"            # small optimization for linear reads"},{"line_number":61,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":8,"id":"85ea73b4_2cbdb44b","line":58,"in_reply_to":"3d4a2c44_6adb4414","updated":"2022-07-28 18:28:42.000000000","message":"FWIW, I consider this mostly an internal interface -- my expectation is that any external projects that want to manipulate rings would use RingWriter/RingReader (and would want some familiarity with FixedLengthReader, since they\u0027ll be returned when opening a section).\n\n\u003e this doesn\u0027t really define the failure mode\n\nI can add some to the docstring, either here, or for read(), or both. Basically, it\u0027ll raise something like\n\n zlib.error: Error -3 while decompressing data: incorrect header check\n\n\u003e how to determine if a pos is a safe flush boundary\n\nIt amounts to making sure you\u0027ve flushed your compressor with zlib.Z_FULL_FLUSH, written the result, and calling tell(). Part of why the RingWriter does a flush() as part of tell() -- so any position you try to track while writing will be a valid seek position when reading.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":55,"context_line":"        ``os.SEEK_SET`` and the reader is already at the desired position)."},{"line_number":56,"context_line":"        As a result, callers should be careful to ``seek()`` to flush"},{"line_number":57,"context_line":"        boundaries, to ensure that subsequent ``read()`` calls work properly."},{"line_number":58,"context_line":"        \"\"\""},{"line_number":59,"context_line":"        if (pos, whence) \u003d\u003d (self.pos, os.SEEK_SET):"},{"line_number":60,"context_line":"            # small optimization for linear reads"},{"line_number":61,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":8,"id":"9ce6de4a_aef25105","line":58,"in_reply_to":"85ea73b4_2cbdb44b","updated":"2025-06-13 21:04:18.000000000","message":"\u003e I consider this mostly an internal interface\n\nHonestly `_GzipReader` may be appropriate, or maybe something like `_RingGzReader` to indicate it\u0027s not for reading gzip files \"generally\"\n\n\u003e Part of why the RingWriter does a flush() as part of tell() \n\nok, maybe that\u0027s hinting at what feels disconnected - the generic sounding `GzipReader` is doing some contortions to \"support\" a style of access used by a `RingReader` but it only \"works\" because the `RingWriter` was careful.\n\nI think I\u0027m heading in the direction of a generic `RingGzReader` \u0026 `RingGzWriter` that are a very thin wrapper around `GzipFile`, `RingData`\u0027s load/save pass one of these to the three codecs:\n\n```\nRING_CODECS \u003d {\n    0: LegacyPickleCodec,\n    1: JsonAndArrayCodec,\n    2: IndexedSectionCodec,\n}\n```\n\n```\n    def load(...):\n        with RingGzReader(filename) as fp:\n            reader \u003d RING_CODECS[version](fp)\n            ring_dict \u003d reader.deserialize()\n            return RingData.from_dict(ring_dict)\n\n...\n\n    def save(...):\n        with RingGzWriter(filename) as fp:\n            writer \u003d RING_CODECS[version](fp)\n            ring_dict \u003d self.to_dict()\n            writer.serialize(ring_dict)\n```\n\n... one difference is that while the `RingGzWriter` does know how to flush, it doesn\u0027t know how to `write_section` - that\u0027s purely an implementation detail of the `IndexedSectionCodec`","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":264,"context_line":""},{"line_number":265,"context_line":"        # See notes in RingWriter.write_index and RingWriter.__exit__ for"},{"line_number":266,"context_line":"        # where this 31 (\u003d 18 + 13) came from."},{"line_number":267,"context_line":"        self.zlib_fp.seek(-31, os.SEEK_END)"},{"line_number":268,"context_line":"        try:"},{"line_number":269,"context_line":"            index_start, \u003d struct.unpack(\"!Q\", self.zlib_fp.read(8))"},{"line_number":270,"context_line":"        except zlib.error:"}],"source_content_type":"text/x-python","patch_set":8,"id":"708dc1ce_1e728f74","line":267,"updated":"2022-07-27 22:25:32.000000000","message":"__exit__ talks about the last 13 bytes, but where does 18 come from?\n\nstill might look better as a constant","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":264,"context_line":""},{"line_number":265,"context_line":"        # See notes in RingWriter.write_index and RingWriter.__exit__ for"},{"line_number":266,"context_line":"        # where this 31 (\u003d 18 + 13) came from."},{"line_number":267,"context_line":"        self.zlib_fp.seek(-31, os.SEEK_END)"},{"line_number":268,"context_line":"        try:"},{"line_number":269,"context_line":"            index_start, \u003d struct.unpack(\"!Q\", self.zlib_fp.read(8))"},{"line_number":270,"context_line":"        except zlib.error:"}],"source_content_type":"text/x-python","patch_set":8,"id":"7c6b3560_55770be2","line":267,"in_reply_to":"708dc1ce_1e728f74","updated":"2022-07-28 18:28:42.000000000","message":"From the comments in write_index:\n\n # switch to uncompressed\n # ... which allows us to know that this will be exactly 18 bytes","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dad76bead53d34c339b6cf5f904698f6681b5a66","unresolved":false,"context_lines":[{"line_number":264,"context_line":""},{"line_number":265,"context_line":"        # See notes in RingWriter.write_index and RingWriter.__exit__ for"},{"line_number":266,"context_line":"        # where this 31 (\u003d 18 + 13) came from."},{"line_number":267,"context_line":"        self.zlib_fp.seek(-31, os.SEEK_END)"},{"line_number":268,"context_line":"        try:"},{"line_number":269,"context_line":"            index_start, \u003d struct.unpack(\"!Q\", self.zlib_fp.read(8))"},{"line_number":270,"context_line":"        except zlib.error:"}],"source_content_type":"text/x-python","patch_set":8,"id":"53b5142a_40788a1e","line":267,"in_reply_to":"7c6b3560_55770be2","updated":"2023-07-07 19:48:16.000000000","message":"lol upgrading to real markdown made it look like I was shouting when I really just wanted to quote the comments","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":324,"context_line":"        if checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":325,"context_line":"            checksum \u003d getattr(hashlib, checksum_method)(prefix)"},{"line_number":326,"context_line":"        else:"},{"line_number":327,"context_line":"            logging.getLogger(\u0027swift.common.ring\u0027).warning("},{"line_number":328,"context_line":"                \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":329,"context_line":"                checksum_method, checksum_value, section)"},{"line_number":330,"context_line":"            checksum \u003d checksum_value \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"fc5af36d_c475a76e","line":327,"updated":"2022-07-27 22:25:32.000000000","message":"this isn\u0027t exactly the same as what we do in builder; I wonder how much consistency we could get in here... \n\nhttps://github.com/openstack/swift/blob/master/swift/common/ring/builder.py#L137-L141","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":324,"context_line":"        if checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":325,"context_line":"            checksum \u003d getattr(hashlib, checksum_method)(prefix)"},{"line_number":326,"context_line":"        else:"},{"line_number":327,"context_line":"            logging.getLogger(\u0027swift.common.ring\u0027).warning("},{"line_number":328,"context_line":"                \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":329,"context_line":"                checksum_method, checksum_value, section)"},{"line_number":330,"context_line":"            checksum \u003d checksum_value \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"7031ab1c_57b24249","line":327,"in_reply_to":"2c95b097_2b1e2a7a","updated":"2025-06-13 21:04:18.000000000","message":"Acknowledged","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":324,"context_line":"        if checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":325,"context_line":"            checksum \u003d getattr(hashlib, checksum_method)(prefix)"},{"line_number":326,"context_line":"        else:"},{"line_number":327,"context_line":"            logging.getLogger(\u0027swift.common.ring\u0027).warning("},{"line_number":328,"context_line":"                \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":329,"context_line":"                checksum_method, checksum_value, section)"},{"line_number":330,"context_line":"            checksum \u003d checksum_value \u003d None"}],"source_content_type":"text/x-python","patch_set":8,"id":"2c95b097_2b1e2a7a","line":327,"in_reply_to":"fc5af36d_c475a76e","updated":"2022-07-28 18:28:42.000000000","message":"Everything in builder is logged at debug, though; I\u0027m not sure it\u0027s an apples-to-apples comparison. I stand by this being logged at warning -- and at least on py3 there\u0027s a logger-of-last-resort that writes to stderr instead of writing that near-useless message. Given the plan to drop py27 support this cycle, IDK that it\u0027s worth adding too much cruft here.\n\nI suppose if nothing else we should at least use \u0027swift.ring\u0027 here, though :-/\n\nExpectation is that this should only trip if/when we want to change checksum method, and even then, only if you start writing rings on new code and pushing them out to servers still running old. There\u0027s no plan to do so, just want to make sure it\u0027s not too painful if we do.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":452,"context_line":"        self.flush()"},{"line_number":453,"context_line":"        # ... so we can start up another with whatever level we want"},{"line_number":454,"context_line":"        self.gzip_fp.compress \u003d zlib.compressobj("},{"line_number":455,"context_line":"            lvl, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)"},{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    @contextlib.contextmanager"},{"line_number":458,"context_line":"    def section(self, name):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5a954876_7355e368","line":455,"updated":"2022-07-27 22:25:32.000000000","message":"it assumed we\u0027d want group the compressed and uncompressed data together; but this looks like we could interleave?  I only see this called with lvl 0 in write_index?","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":452,"context_line":"        self.flush()"},{"line_number":453,"context_line":"        # ... so we can start up another with whatever level we want"},{"line_number":454,"context_line":"        self.gzip_fp.compress \u003d zlib.compressobj("},{"line_number":455,"context_line":"            lvl, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)"},{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    @contextlib.contextmanager"},{"line_number":458,"context_line":"    def section(self, name):"}],"source_content_type":"text/x-python","patch_set":8,"id":"e9990f81_1ea85916","line":455,"in_reply_to":"5a954876_7355e368","updated":"2022-07-28 18:28:42.000000000","message":"Yeah, with this we can switch the compression level around basically whenever we want -- but it only matters for our purposes when writing that tail on the index, and even then, only for the sake of knowing the offset to use in load_index.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":false,"context_lines":[{"line_number":452,"context_line":"        self.flush()"},{"line_number":453,"context_line":"        # ... so we can start up another with whatever level we want"},{"line_number":454,"context_line":"        self.gzip_fp.compress \u003d zlib.compressobj("},{"line_number":455,"context_line":"            lvl, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)"},{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    @contextlib.contextmanager"},{"line_number":458,"context_line":"    def section(self, name):"}],"source_content_type":"text/x-python","patch_set":8,"id":"15c19433_1538c3c0","line":455,"in_reply_to":"78f1d240_540ce5bb","updated":"2025-01-28 17:30:08.000000000","message":"Done","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":452,"context_line":"        self.flush()"},{"line_number":453,"context_line":"        # ... so we can start up another with whatever level we want"},{"line_number":454,"context_line":"        self.gzip_fp.compress \u003d zlib.compressobj("},{"line_number":455,"context_line":"            lvl, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)"},{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    @contextlib.contextmanager"},{"line_number":458,"context_line":"    def section(self, name):"}],"source_content_type":"text/x-python","patch_set":8,"id":"78f1d240_540ce5bb","line":455,"in_reply_to":"e9990f81_1ea85916","updated":"2023-03-29 00:40:00.000000000","message":"ok, so RingWriter doesn\u0027t really need to \"arbitrarily\" set the compression level. \n Although our gzip_fp might support it this interface is meant to only expose ringwriter.prepare_to_write_index()\n \nI guess we wouldn\u0027t want to move these two lines *into* write_index, but maybe it should be a \"private\" function?","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":476,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":477,"context_line":"        self.flush()"},{"line_number":478,"context_line":"        self.current_section \u003d name"},{"line_number":479,"context_line":"        self.index[name] \u003d [self.tell(), self.pos, None, None, None, None]"},{"line_number":480,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":481,"context_line":"        try:"},{"line_number":482,"context_line":"            yield self"}],"source_content_type":"text/x-python","patch_set":8,"id":"1dc87a59_a5be6e52","line":479,"updated":"2022-07-27 22:25:32.000000000","message":"umm... do you want a dictionary or dataclass or something?  What are these fields?","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":476,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":477,"context_line":"        self.flush()"},{"line_number":478,"context_line":"        self.current_section \u003d name"},{"line_number":479,"context_line":"        self.index[name] \u003d [self.tell(), self.pos, None, None, None, None]"},{"line_number":480,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":481,"context_line":"        try:"},{"line_number":482,"context_line":"            yield self"}],"source_content_type":"text/x-python","patch_set":8,"id":"99dfb4ad_bff58d50","line":479,"in_reply_to":"1dc87a59_a5be6e52","updated":"2022-07-28 18:28:42.000000000","message":"dataclass could be nice -- but at the moment, we still support py27 so that seems to be out. I could try it as a namedtuple -- maybe as a follow-up?","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0f3c8fb93ded0a5b996c3e9fea2129b167689a16","unresolved":true,"context_lines":[{"line_number":476,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":477,"context_line":"        self.flush()"},{"line_number":478,"context_line":"        self.current_section \u003d name"},{"line_number":479,"context_line":"        self.index[name] \u003d [self.tell(), self.pos, None, None, None, None]"},{"line_number":480,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":481,"context_line":"        try:"},{"line_number":482,"context_line":"            yield self"}],"source_content_type":"text/x-python","patch_set":8,"id":"e1b7056f_aec7fc93","line":479,"in_reply_to":"99dfb4ad_bff58d50","updated":"2022-07-28 23:02:45.000000000","message":"This wasn\u0027t near as bad as I\u0027d feared: https://review.opendev.org/c/openstack/swift/+/851451\n\nClay, WDYT? Should I squash it in?","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6758d76e50815b54673c2e466a593c06bb78874a","unresolved":false,"context_lines":[{"line_number":476,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":477,"context_line":"        self.flush()"},{"line_number":478,"context_line":"        self.current_section \u003d name"},{"line_number":479,"context_line":"        self.index[name] \u003d [self.tell(), self.pos, None, None, None, None]"},{"line_number":480,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":481,"context_line":"        try:"},{"line_number":482,"context_line":"            yield self"}],"source_content_type":"text/x-python","patch_set":8,"id":"b9492c1a_1d089326","line":479,"in_reply_to":"e1b7056f_aec7fc93","updated":"2022-08-03 00:55:03.000000000","message":"Done","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":94,"context_line":"        chunk \u003d self.fp.read(self.chunk_size)"},{"line_number":95,"context_line":"        if not chunk:"},{"line_number":96,"context_line":"            self.buffer +\u003d self.decompressor.decompress(self.compressed_buffer)"},{"line_number":97,"context_line":"            self.pos +\u003d len(self.compressed_buffer)"},{"line_number":98,"context_line":"            self.compressed_buffer \u003d b\"\""},{"line_number":99,"context_line":"            return False"},{"line_number":100,"context_line":"        self.compressed_buffer +\u003d chunk"}],"source_content_type":"text/x-python","patch_set":17,"id":"bd1a107e_619b4822","line":97,"updated":"2023-03-29 00:40:00.000000000","message":"is this line only needed if the chunk is empty?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[{"line_number":94,"context_line":"        chunk \u003d self.fp.read(self.chunk_size)"},{"line_number":95,"context_line":"        if not chunk:"},{"line_number":96,"context_line":"            self.buffer +\u003d self.decompressor.decompress(self.compressed_buffer)"},{"line_number":97,"context_line":"            self.pos +\u003d len(self.compressed_buffer)"},{"line_number":98,"context_line":"            self.compressed_buffer \u003d b\"\""},{"line_number":99,"context_line":"            return False"},{"line_number":100,"context_line":"        self.compressed_buffer +\u003d chunk"}],"source_content_type":"text/x-python","patch_set":17,"id":"03374a4c_203b8051","line":97,"in_reply_to":"bd1a107e_619b4822","updated":"2025-03-26 19:12:40.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":104,"context_line":"            end \u003d x + len(ZLIB_FLUSH_MARKER)"},{"line_number":105,"context_line":"            chunk \u003d self.compressed_buffer[:end]"},{"line_number":106,"context_line":"            self.compressed_buffer \u003d self.compressed_buffer[end:]"},{"line_number":107,"context_line":"            self.pos +\u003d len(chunk)"},{"line_number":108,"context_line":"            self.buffer +\u003d self.decompressor.decompress(chunk)"},{"line_number":109,"context_line":"            return True"},{"line_number":110,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"bc44a464_e1d4dcf0","line":107,"updated":"2023-03-29 00:40:00.000000000","message":"oic, here we take care of it.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[{"line_number":104,"context_line":"            end \u003d x + len(ZLIB_FLUSH_MARKER)"},{"line_number":105,"context_line":"            chunk \u003d self.compressed_buffer[:end]"},{"line_number":106,"context_line":"            self.compressed_buffer \u003d self.compressed_buffer[end:]"},{"line_number":107,"context_line":"            self.pos +\u003d len(chunk)"},{"line_number":108,"context_line":"            self.buffer +\u003d self.decompressor.decompress(chunk)"},{"line_number":109,"context_line":"            return True"},{"line_number":110,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"9373e252_8486684c","line":107,"in_reply_to":"bc44a464_e1d4dcf0","updated":"2025-03-26 19:12:40.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":106,"context_line":"            self.compressed_buffer \u003d self.compressed_buffer[end:]"},{"line_number":107,"context_line":"            self.pos +\u003d len(chunk)"},{"line_number":108,"context_line":"            self.buffer +\u003d self.decompressor.decompress(chunk)"},{"line_number":109,"context_line":"            return True"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"        # we may have *almost* found the flush marker;"},{"line_number":112,"context_line":"        # gotta keep some of the tail"}],"source_content_type":"text/x-python","patch_set":17,"id":"eabb701d_89cca869","line":109,"updated":"2023-03-29 00:40:00.000000000","message":"wait, this is the same block as L85-L92 again","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[{"line_number":106,"context_line":"            self.compressed_buffer \u003d self.compressed_buffer[end:]"},{"line_number":107,"context_line":"            self.pos +\u003d len(chunk)"},{"line_number":108,"context_line":"            self.buffer +\u003d self.decompressor.decompress(chunk)"},{"line_number":109,"context_line":"            return True"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"        # we may have *almost* found the flush marker;"},{"line_number":112,"context_line":"        # gotta keep some of the tail"}],"source_content_type":"text/x-python","patch_set":17,"id":"86636ac8_874e1f11","line":109,"in_reply_to":"eabb701d_89cca869","updated":"2025-03-26 19:12:40.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":113,"context_line":"        keep \u003d len(ZLIB_FLUSH_MARKER) - 1"},{"line_number":114,"context_line":"        chunk \u003d self.compressed_buffer[:-keep]"},{"line_number":115,"context_line":"        self.compressed_buffer \u003d self.compressed_buffer[-keep:]"},{"line_number":116,"context_line":"        self.pos +\u003d len(chunk)"},{"line_number":117,"context_line":"        # note that there\u0027s no guarantee that buffer will actually grow --"},{"line_number":118,"context_line":"        # but we don\u0027t want to have more in compressed_buffer than strictly"},{"line_number":119,"context_line":"        # necessary"}],"source_content_type":"text/x-python","patch_set":17,"id":"94235ce8_5cfc7ce1","line":116,"updated":"2023-03-29 00:40:00.000000000","message":"and here again.\n\nI actually hate writing/reading/maintaing this kind of code - I feel like i\u0027ve seen a few nice interfaces for buffered-io, i wonder if we could leverage them.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[{"line_number":113,"context_line":"        keep \u003d len(ZLIB_FLUSH_MARKER) - 1"},{"line_number":114,"context_line":"        chunk \u003d self.compressed_buffer[:-keep]"},{"line_number":115,"context_line":"        self.compressed_buffer \u003d self.compressed_buffer[-keep:]"},{"line_number":116,"context_line":"        self.pos +\u003d len(chunk)"},{"line_number":117,"context_line":"        # note that there\u0027s no guarantee that buffer will actually grow --"},{"line_number":118,"context_line":"        # but we don\u0027t want to have more in compressed_buffer than strictly"},{"line_number":119,"context_line":"        # necessary"}],"source_content_type":"text/x-python","patch_set":17,"id":"18103ab6_c14d924b","line":116,"in_reply_to":"94235ce8_5cfc7ce1","updated":"2025-03-26 19:12:40.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":185,"context_line":"        return ["},{"line_number":186,"context_line":"            read_network_order_array(type_code, row)"},{"line_number":187,"context_line":"            for row in iter(lambda: self.read(max_row_len), b\u0027\u0027)"},{"line_number":188,"context_line":"        ]"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"    def close(self):"},{"line_number":191,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"da7b40b5_a6be7679","line":188,"updated":"2023-03-29 00:40:00.000000000","message":"I think these SectionReaders are somehow both too abstract and too concrete for me.\n\nThis is a very specific helper for a very specific type of section, why does it make sense to hang this method here?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"56f6ff8c_a10281d4","line":225,"updated":"2023-03-29 00:40:00.000000000","message":"i did not know you could subclass named tuples and add properties\n\nwhy is this a namedtuple subclass and not just a regular object class?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"a786f66d_70a5ced8","line":225,"in_reply_to":"56f6ff8c_a10281d4","updated":"2023-04-03 23:05:30.000000000","message":"...because we can\u0027t use https://docs.python.org/3/library/dataclasses.html ?\n\nRegular class would be fine, too, I suppose -- I\u0027m not so concerned about the memory footprint of per-instance dicts. Mainly I wanted to avoid a bunch of boilerplate `__init__` code.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":false,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"20edfe88_ffb606e2","line":225,"in_reply_to":"7504c64a_d866a782","updated":"2025-06-18 22:21:44.000000000","message":"FWIW I\u0027ve warmed to using a dataclass here - if anything I wish modern python made it even easier to declare these sort of structs.  I\u0027m sure I fully understand all the implications of mutable but uncomparible objects being hashible - or how necessary it is that IndexEntry\u0027s be comparable for testing.  But this seems like one reasonable way to do what we need to do.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"7504c64a_d866a782","line":225,"in_reply_to":"7cada673_3b3bc6f6","updated":"2025-06-16 23:28:49.000000000","message":"\u003e I\u0027m sure we\u0027ll all benefit from understanding/using dataclasses more\n\nWhat better way to explore that than using it in a small, self-contained bit of greenfield code?\n\n\u003e some explicit code you only have to write once might still be the most efficient way to get the idea across to other maintainers\n\nI feel like it\u0027s still plenty explicit; it\u0027s just declarative rather than procedural, which means you don\u0027t need to go spot-checking for copy/paste bugs like\n```\nself.compressed_start \u003d compressed_start\nself.uncompressed_start \u003d uncompressed_start\nself.compressed_end \u003d compressed_start\nself.uncompressed_end \u003d uncompressed_end\n```\n\n\u003e Poor ShardRanges\n\nI\u0027m not sure a 550 line class (which subclasses a 200 line class) should be used as a model of readability. And while I understand the need to have `upper`/`lower`, some of the other type-checking setters (without which there\u0027d be no need for a corresponding getter) are unnecessary and should have had that logic pushed back to the caller. I like writing Python, not Java.\n\n\u003e have an explicit `__eq__` and `__repr__` that have to get updated with every new attribute!\n\nThat doesn\u0027t strike me as code to aspire to! It might well be *necessary*, but I don\u0027t see why it would be for `IndexEntry`.\n\nSide note: the [comment around `__hash__`](https://github.com/openstack/swift/blob/2.35.0/swift/common/utils/__init__.py#L4346-L4350) strikes me as misleading at best and part of why I **don\u0027t** want to go implementing a bunch of dunder methods if I don\u0027t have to. We would be much, **much** better served by a comment explaining *why* we decided to make this object hashable despite being mutable. https://review.opendev.org/c/openstack/swift/+/952725","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":true,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"b73c21de_34e75e52","line":225,"in_reply_to":"94d3752f_d7cd6c71","updated":"2025-05-20 19:58:47.000000000","message":"🎉 https://review.opendev.org/c/openstack/swift/+/949684 🎉\n\nMaybe I\u0027ll take a look at doing this with dataclasses now...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"08d68a7ae55f7a38ad0f401a01242ded5d81471b","unresolved":false,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"27a17763_1cf2437c","line":225,"in_reply_to":"9f3063b0_44ac8c73","updated":"2025-06-09 01:31:48.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":true,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"94d3752f_d7cd6c71","line":225,"in_reply_to":"a786f66d_70a5ced8","updated":"2025-01-28 17:30:08.000000000","message":"Before anyone gets excited about us having dropped py2 -- dataclasses were introduced in py37 and we still support py36.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"7cada673_3b3bc6f6","line":225,"in_reply_to":"b73c21de_34e75e52","updated":"2025-06-13 21:04:18.000000000","message":"\u003e Mainly I wanted to avoid a bunch of boilerplate `__init__` code\n\nI\u0027m sure we\u0027ll all benifit from understanding/using dataclasses more; but avoiding some explicit code you only have to write once might still be the most efficient way to get the idea across to other maintainers.  Poor ShardRanges have an explicit `__eq__` and `__repr__` that have to get updated with every new attribute!\n\nhttps://va.lent.in/optimize-for-readability-first/","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"946c1fb770d4bd769708dfd24306ba0f043f753b","unresolved":true,"context_lines":[{"line_number":222,"context_line":"    def uncompressed_length(self):"},{"line_number":223,"context_line":"        if self.uncompressed_end is None:"},{"line_number":224,"context_line":"            return None"},{"line_number":225,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":""},{"line_number":228,"context_line":"class RingReader(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"9f3063b0_44ac8c73","line":225,"in_reply_to":"b73c21de_34e75e52","updated":"2025-05-28 01:04:57.000000000","message":"I don\u0027t _hate_ it... https://review.opendev.org/c/openstack/swift/+/951049","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":234,"context_line":"    chunk_size \u003d DEFAULT_BUFFER_SIZE"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"    def __init__(self, filename):"},{"line_number":237,"context_line":"        self.fp \u003d open(filename, \u0027rb\u0027)"},{"line_number":238,"context_line":"        self.zlib_fp \u003d ZlibReader(self.fp)"},{"line_number":239,"context_line":"        self.index \u003d {}"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"b96fe962_80fbb839","line":237,"updated":"2023-04-03 23:05:30.000000000","message":"Part of me wonders if it\u0027d be better to have this accept the already-opened fileobj -- then tests could exercise it more simply with `BytesIO`...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"11b7880a1275f04ac271690978ed36008dde2021","unresolved":false,"context_lines":[{"line_number":234,"context_line":"    chunk_size \u003d DEFAULT_BUFFER_SIZE"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"    def __init__(self, filename):"},{"line_number":237,"context_line":"        self.fp \u003d open(filename, \u0027rb\u0027)"},{"line_number":238,"context_line":"        self.zlib_fp \u003d ZlibReader(self.fp)"},{"line_number":239,"context_line":"        self.index \u003d {}"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"082c645f_44c12725","line":237,"in_reply_to":"b96fe962_80fbb839","updated":"2025-01-10 04:45:55.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":235,"context_line":""},{"line_number":236,"context_line":"    def __init__(self, filename):"},{"line_number":237,"context_line":"        self.fp \u003d open(filename, \u0027rb\u0027)"},{"line_number":238,"context_line":"        self.zlib_fp \u003d ZlibReader(self.fp)"},{"line_number":239,"context_line":"        self.index \u003d {}"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"        magic \u003d self.zlib_fp.read(4)"}],"source_content_type":"text/x-python","patch_set":17,"id":"28af2a70_aa50c916","line":238,"updated":"2023-03-29 00:40:00.000000000","message":"ok, so RingReader *has a* ZlibReader, but instead of self.zlib_reader it calls it a \"zlib_fp\" which actually seems like what you might have named self.fp","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c40c0cc14a628562037366a1b1e70adc2ec0d0da","unresolved":true,"context_lines":[{"line_number":235,"context_line":""},{"line_number":236,"context_line":"    def __init__(self, filename):"},{"line_number":237,"context_line":"        self.fp \u003d open(filename, \u0027rb\u0027)"},{"line_number":238,"context_line":"        self.zlib_fp \u003d ZlibReader(self.fp)"},{"line_number":239,"context_line":"        self.index \u003d {}"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"        magic \u003d self.zlib_fp.read(4)"}],"source_content_type":"text/x-python","patch_set":17,"id":"f30f7a5d_ea301293","line":238,"in_reply_to":"25a3af88_6c9b9403","updated":"2023-05-03 22:58:40.000000000","message":"Or was the point that `RingReader` should instead *be a* `ZlibReader`? Might could simplify some things...\n\nStarted to experiment with it in https://review.opendev.org/c/openstack/swift/+/882184 but I\u0027m not sure about it yet.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":235,"context_line":""},{"line_number":236,"context_line":"    def __init__(self, filename):"},{"line_number":237,"context_line":"        self.fp \u003d open(filename, \u0027rb\u0027)"},{"line_number":238,"context_line":"        self.zlib_fp \u003d ZlibReader(self.fp)"},{"line_number":239,"context_line":"        self.index \u003d {}"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"        magic \u003d self.zlib_fp.read(4)"}],"source_content_type":"text/x-python","patch_set":17,"id":"25a3af88_6c9b9403","line":238,"in_reply_to":"28af2a70_aa50c916","updated":"2023-04-03 23:05:30.000000000","message":"I\u0027m happy to rename things -- sounds like you maybe like `self.raw_fp`/`self.fp` in place of `self.fp`/`self.zlib_fp`? Or maybe we want `self.compressed_fp`/`self.uncompressed_fp`?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"11b7880a1275f04ac271690978ed36008dde2021","unresolved":false,"context_lines":[{"line_number":235,"context_line":""},{"line_number":236,"context_line":"    def __init__(self, filename):"},{"line_number":237,"context_line":"        self.fp \u003d open(filename, \u0027rb\u0027)"},{"line_number":238,"context_line":"        self.zlib_fp \u003d ZlibReader(self.fp)"},{"line_number":239,"context_line":"        self.index \u003d {}"},{"line_number":240,"context_line":""},{"line_number":241,"context_line":"        magic \u003d self.zlib_fp.read(4)"}],"source_content_type":"text/x-python","patch_set":17,"id":"185bbe4e_3edcee8b","line":238,"in_reply_to":"f30f7a5d_ea301293","updated":"2025-01-10 04:45:55.000000000","message":"Done -- experiment got merged in.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":262,"context_line":"        return self"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"    def __exit__(self, *args):"},{"line_number":265,"context_line":"        self.close()"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"    def read(self, amt):"},{"line_number":268,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"ba0bc7ac_f09d07bb","line":265,"updated":"2023-03-29 00:40:00.000000000","message":"this is nice, i\u0027m thankful for the context manager interface.  I dunno if it\u0027s useful to have an explict \"open\" method like diskfile, or have that implicit in __init__ (it seems like you have in the context manager case you have an open fd before you enter __enter__)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[{"line_number":262,"context_line":"        return self"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"    def __exit__(self, *args):"},{"line_number":265,"context_line":"        self.close()"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"    def read(self, amt):"},{"line_number":268,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"cf2aaa71_dfa07ff5","line":265,"in_reply_to":"ba0bc7ac_f09d07bb","updated":"2025-03-26 19:12:40.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[{"line_number":262,"context_line":"        return self"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"    def __exit__(self, *args):"},{"line_number":265,"context_line":"        self.close()"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"    def read(self, amt):"},{"line_number":268,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"f43448d4_cff73fe9","line":265,"in_reply_to":"ba0bc7ac_f09d07bb","updated":"2025-03-26 19:12:40.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":330,"context_line":"        \"\"\""},{"line_number":331,"context_line":"        prefix \u003d self.zlib_fp.read(struct.calcsize(fmt))"},{"line_number":332,"context_line":"        blob_length, \u003d struct.unpack(fmt, prefix)"},{"line_number":333,"context_line":"        return self.zlib_fp.read(blob_length)"},{"line_number":334,"context_line":""},{"line_number":335,"context_line":"    def read_section(self, section):"},{"line_number":336,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"c82f53cc_ca493d10","line":333,"updated":"2023-03-29 00:40:00.000000000","message":"is this *only* used in v1?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":330,"context_line":"        \"\"\""},{"line_number":331,"context_line":"        prefix \u003d self.zlib_fp.read(struct.calcsize(fmt))"},{"line_number":332,"context_line":"        blob_length, \u003d struct.unpack(fmt, prefix)"},{"line_number":333,"context_line":"        return self.zlib_fp.read(blob_length)"},{"line_number":334,"context_line":""},{"line_number":335,"context_line":"    def read_section(self, section):"},{"line_number":336,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"4d1291c4_88ad429e","line":333,"in_reply_to":"20756166_f7307621","updated":"2025-06-13 21:04:18.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":330,"context_line":"        \"\"\""},{"line_number":331,"context_line":"        prefix \u003d self.zlib_fp.read(struct.calcsize(fmt))"},{"line_number":332,"context_line":"        blob_length, \u003d struct.unpack(fmt, prefix)"},{"line_number":333,"context_line":"        return self.zlib_fp.read(blob_length)"},{"line_number":334,"context_line":""},{"line_number":335,"context_line":"    def read_section(self, section):"},{"line_number":336,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"20756166_f7307621","line":333,"in_reply_to":"c82f53cc_ca493d10","updated":"2023-04-03 23:05:30.000000000","message":"For the most part, now. At some point before v2 included an index and named sections, `deserialize_v2` looked like\n\n ring_dict \u003d json.loads(reader.read_blob())\n ring_dict[\u0027devs\u0027] \u003d json.loads(reader.read_blob())\n ...\n\nor something, though I can\u0027t find the patchset(/patch?) now.\n\nWe also still use it in `load_index`, though.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":337,"context_line":"        Read an entire section\u0027s data"},{"line_number":338,"context_line":"        \"\"\""},{"line_number":339,"context_line":"        with self.open_section(section) as reader:"},{"line_number":340,"context_line":"            return reader.read()"},{"line_number":341,"context_line":""},{"line_number":342,"context_line":"    @contextlib.contextmanager"},{"line_number":343,"context_line":"    def open_section(self, section):"}],"source_content_type":"text/x-python","patch_set":17,"id":"cdf27749_297c34e5","line":340,"updated":"2023-03-29 00:40:00.000000000","message":"so this read method is normally be good enough if we don\u0027t mind using some memory, and we use it when the data is a \"small\" amount of json.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":337,"context_line":"        Read an entire section\u0027s data"},{"line_number":338,"context_line":"        \"\"\""},{"line_number":339,"context_line":"        with self.open_section(section) as reader:"},{"line_number":340,"context_line":"            return reader.read()"},{"line_number":341,"context_line":""},{"line_number":342,"context_line":"    @contextlib.contextmanager"},{"line_number":343,"context_line":"    def open_section(self, section):"}],"source_content_type":"text/x-python","patch_set":17,"id":"8676260e_c0dce1e9","line":340,"in_reply_to":"2823d499_df5c5341","updated":"2025-06-13 21:04:18.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"58f759cd839f5087aa1db3fea5109ea81b434b5d","unresolved":true,"context_lines":[{"line_number":337,"context_line":"        Read an entire section\u0027s data"},{"line_number":338,"context_line":"        \"\"\""},{"line_number":339,"context_line":"        with self.open_section(section) as reader:"},{"line_number":340,"context_line":"            return reader.read()"},{"line_number":341,"context_line":""},{"line_number":342,"context_line":"    @contextlib.contextmanager"},{"line_number":343,"context_line":"    def open_section(self, section):"}],"source_content_type":"text/x-python","patch_set":17,"id":"2823d499_df5c5341","line":340,"in_reply_to":"cdf27749_297c34e5","updated":"2025-01-15 09:35:33.000000000","message":"yeah, and if you want more say use the open_section context manager below if you want more memory effiency.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":365,"context_line":"        if entry.checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":366,"context_line":"            checksum \u003d getattr(hashlib, entry.checksum_method)(prefix)"},{"line_number":367,"context_line":"            checksum_value \u003d entry.checksum_value"},{"line_number":368,"context_line":"        else:"},{"line_number":369,"context_line":"            logging.getLogger(\u0027swift.ring\u0027).warning("},{"line_number":370,"context_line":"                \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":371,"context_line":"                entry.checksum_method, entry.checksum_value, section)"}],"source_content_type":"text/x-python","patch_set":17,"id":"59989c5f_b4a1e920","line":368,"updated":"2023-04-03 23:05:30.000000000","message":"This should be\n\n elif entry.checksum_method is not None:","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":false,"context_lines":[{"line_number":365,"context_line":"        if entry.checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":366,"context_line":"            checksum \u003d getattr(hashlib, entry.checksum_method)(prefix)"},{"line_number":367,"context_line":"            checksum_value \u003d entry.checksum_value"},{"line_number":368,"context_line":"        else:"},{"line_number":369,"context_line":"            logging.getLogger(\u0027swift.ring\u0027).warning("},{"line_number":370,"context_line":"                \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":371,"context_line":"                entry.checksum_method, entry.checksum_value, section)"}],"source_content_type":"text/x-python","patch_set":17,"id":"d107fe36_a9181b56","line":368,"in_reply_to":"59989c5f_b4a1e920","updated":"2025-01-28 17:30:08.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":377,"context_line":"            digest\u003dchecksum_value,"},{"line_number":378,"context_line":"            checksum\u003dchecksum,"},{"line_number":379,"context_line":"        ) as reader:"},{"line_number":380,"context_line":"            yield reader"},{"line_number":381,"context_line":""},{"line_number":382,"context_line":""},{"line_number":383,"context_line":"class RingWriter(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"cccfb159_06183db8","line":380,"updated":"2023-03-29 00:40:00.000000000","message":"and then here RingReader uses it\u0027s ZlibReader to build a SectionReader, ok","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":377,"context_line":"            digest\u003dchecksum_value,"},{"line_number":378,"context_line":"            checksum\u003dchecksum,"},{"line_number":379,"context_line":"        ) as reader:"},{"line_number":380,"context_line":"            yield reader"},{"line_number":381,"context_line":""},{"line_number":382,"context_line":""},{"line_number":383,"context_line":"class RingWriter(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"f05a9c0c_3ceaad89","line":380,"in_reply_to":"cccfb159_06183db8","updated":"2025-08-04 07:10:27.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":394,"context_line":""},{"line_number":395,"context_line":"    Note that the index will only be written if named sections were defined."},{"line_number":396,"context_line":"    \"\"\""},{"line_number":397,"context_line":"    checksum_method \u003d \u0027sha256\u0027"},{"line_number":398,"context_line":""},{"line_number":399,"context_line":"    def __init__(self, filename, mtime\u003d1300507380.0):"},{"line_number":400,"context_line":"        self.filename \u003d filename"}],"source_content_type":"text/x-python","patch_set":17,"id":"0d57e661_45df6902","line":397,"updated":"2023-03-29 00:40:00.000000000","message":"at this point, does this make more sense as a class attribute or constant?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":513,"context_line":"            raise ValueError(\u0027Cannot create new section; currently writing %r\u0027"},{"line_number":514,"context_line":"                             % self.current_section)"},{"line_number":515,"context_line":"        if \u0027*\u0027 in name:"},{"line_number":516,"context_line":"            raise ValueError(\u0027Cannot write section with wildcard: %s\u0027 % name)"},{"line_number":517,"context_line":"        if name in self.index:"},{"line_number":518,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":519,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"9e35d17a_b0dccb0f","line":516,"updated":"2023-03-29 00:40:00.000000000","message":"i\u0027m looking forward to understanding why this limitation is enforced.\n\nit\u0027s just reserved, we could reserve other characters too - maybe a lot more, but idk about \"forward compatibility\"","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":513,"context_line":"            raise ValueError(\u0027Cannot create new section; currently writing %r\u0027"},{"line_number":514,"context_line":"                             % self.current_section)"},{"line_number":515,"context_line":"        if \u0027*\u0027 in name:"},{"line_number":516,"context_line":"            raise ValueError(\u0027Cannot write section with wildcard: %s\u0027 % name)"},{"line_number":517,"context_line":"        if name in self.index:"},{"line_number":518,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":519,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"fe8787fd_c92fba92","line":516,"in_reply_to":"9e35d17a_b0dccb0f","updated":"2023-04-03 23:05:30.000000000","message":"IIRC I had some thought a while ago about being able to make a calls like\n\n ring \u003d Ring(fname, only_load_sections\u003d(\n     \"swift/ring/metadata\", \"swift/ring/devices\"))\n\nor\n\n ring \u003d Ring(fname, only_load_sections\u003d\"swift/ring/*\")\n\nas a finer-grained replacement for `metadata_only`. Maybe even something like\n\n Ring \u003d functools.partial(RingOrBuilder, only_load_sections\u003d\"swift/ring/*\")\n\nonce we\u0027ve got unified rings/builders. Also means that https://review.opendev.org/c/openstack/swift/+/866082 could eventually be updated to accept something like\n\n swift-ring-packer unpack object.ring.gz \u0027swift/**\u0027\n\nwithout any ambiguity.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":513,"context_line":"            raise ValueError(\u0027Cannot create new section; currently writing %r\u0027"},{"line_number":514,"context_line":"                             % self.current_section)"},{"line_number":515,"context_line":"        if \u0027*\u0027 in name:"},{"line_number":516,"context_line":"            raise ValueError(\u0027Cannot write section with wildcard: %s\u0027 % name)"},{"line_number":517,"context_line":"        if name in self.index:"},{"line_number":518,"context_line":"            raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)"},{"line_number":519,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"d4256358_629bc84c","line":516,"in_reply_to":"fe8787fd_c92fba92","updated":"2025-06-13 21:04:18.000000000","message":"those are all neat ideas - I 100% support the more restrictive:\n\n`string.ascii_letters + string.digits + \u0027/-\u0027`","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"e4182ff6_0eb6f240","line":523,"updated":"2023-03-29 00:40:00.000000000","message":"wat?  how many checksum_methods do we support!?\n\ni mis-read this as `checksum_method() instead of checksum_method)()`\n\nI think this could be easier to read if you:\n\n1) made self.checksum_method \u003d\u003e CHECKSUM_METHOD_NAME\n2) did the getattr and call on two lines\n\n    checksum_method \u003d getattr(hashlib, CHECKSUM_METHOD_NAME)\n    self.checker \u003d checksum_method()","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"d4e27326_6c812e5c","line":523,"in_reply_to":"397f720f_9f7c20e4","updated":"2025-08-04 07:10:27.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":true,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"f2bb6d39_b210eb43","line":523,"in_reply_to":"8abf8bcf_8340cc68","updated":"2025-07-30 07:49:51.000000000","message":"Seems future-proofing is good. I mean we\u0027ve now got support for crc64nvme, so I think it\u0027s good we can see far enough in the future give us the ability to change the checksum algorithm.\n\nMaybe RingWrite does need a checksum_method kwarg and for the section contextmanager too:\n\n```\ndiff --git i/swift/common/ring/io.py w/swift/common/ring/io.py\nindex 0e5adabfb..6dbe3cf92 100644\n--- i/swift/common/ring/io.py\n+++ w/swift/common/ring/io.py\n@@ -520,9 +520,11 @@ class RingWriter(GzipWriter):\n         self.index \u003d {}\n         self.current_section \u003d None\n         self.checksum \u003d None\n+        if kw.get(\u0027checksum_method\u0027):\n+            self.checksum_method \u003d kw[\u0027checksum_method\u0027]\n \n     @contextlib.contextmanager\n-    def section(self, name):\n+    def section(self, name, checksum_method\u003dNone):\n         \"\"\"\n         Define a named section.\n \n@@ -542,10 +544,12 @@ class RingWriter(GzipWriter):\n             raise ValueError(\u0027Section has invalid name: %s\u0027 % name)\n         if name in self.index:\n             raise ValueError(\u0027Cannot write duplicate section: %s\u0027 % name)\n+        if not checksum_method:\n+            checksum_method \u003d self.checksum_method\n         self.flush()\n         self.current_section \u003d name\n         self.index[name] \u003d IndexEntry(self.tell(), self.pos)\n-        checksum_class \u003d getattr(hashlib, self.checksum_method)\n+        checksum_class \u003d getattr(hashlib, checksum_method)\n         self.checksum \u003d checksum_class()\n         try:\n             yield self\n@@ -554,7 +558,7 @@ class RingWriter(GzipWriter):\n                 self.index[name],\n                 compressed_end\u003dself.tell(),\n                 uncompressed_end\u003dself.pos,\n-                checksum_method\u003dself.checksum_method,\n+                checksum_method\u003dchecksum_method,\n                 checksum_value\u003dself.checksum.hexdigest(),\n             )\n         finally:\n```","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"c61315a7_e03492ee","line":523,"in_reply_to":"a3b8d7d9_29fa49a7","updated":"2025-05-06 20:28:41.000000000","message":"With https://review.opendev.org/c/openstack/swift/+/909800 and https://review.opendev.org/c/openstack/swift/+/946141 on the horizon, I stand by my decision to want to be able to eat unrecognized methods with a warning. Arguably this is still _too prescriptive_.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"8abf8bcf_8340cc68","line":523,"in_reply_to":"c61315a7_e03492ee","updated":"2025-06-13 21:04:18.000000000","message":"\u003e be able to eat unrecognized methods with a warning\n\nthat might be reasonable, - IMHO, \"forward compat\" is always difficult to get right and only very rarely helpful - but I don\u0027t think that has anything to do with how you spell the creation of the checksum hasher in this method.\n\nAFAICT RingWriter is never subclassed to override the class attribute `checksum_method` so it seems like mis-direction to not just create a hashlib.sha256 instance right here to state plainly what we\u0027re using.  If later we want to add different checksuming; AND it turns out that\u0027s easier to do with a subclass then we can simply refactor to an attribute to DRY it out at that point.  For now, YAGNI - better to keep all the relevant context local.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"946c1fb770d4bd769708dfd24306ba0f043f753b","unresolved":true,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"7d0d8c30_fa0fb4f6","line":523,"in_reply_to":"c61315a7_e03492ee","updated":"2025-05-28 01:04:57.000000000","message":"Now that we\u0027ve got that first patch merged, I wouldn\u0027t be opposed to using crc32/crc32c for these checksums...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":true,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"a3b8d7d9_29fa49a7","line":523,"in_reply_to":"e4182ff6_0eb6f240","updated":"2025-03-26 19:12:40.000000000","message":"why do any of this?  srly, how many checksum_methods do we need?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":true,"context_lines":[{"line_number":520,"context_line":"        self.current_section \u003d name"},{"line_number":521,"context_line":"        self.index[name] \u003d IndexEntry("},{"line_number":522,"context_line":"            self.tell(), self.pos, None, None, None, None)"},{"line_number":523,"context_line":"        self.checksum \u003d getattr(hashlib, self.checksum_method)()"},{"line_number":524,"context_line":"        try:"},{"line_number":525,"context_line":"            yield self"},{"line_number":526,"context_line":"            self.flush()"}],"source_content_type":"text/x-python","patch_set":17,"id":"397f720f_9f7c20e4","line":523,"in_reply_to":"f2bb6d39_b210eb43","updated":"2025-08-01 00:08:57.000000000","message":"\u003e Seems future-proofing is good\n\nthat should tell us that we should write down what checksum we\u0027re using in-case we need to chagne - not that we should ignore the checksum algo if it\u0027s not what we expect?\n\n\u003e does need a checksum_method kwarg and for the section contextmanager\n\nheck no - that\u0027s taking a bad idea even further!?  We only use sha256 - sha256 is great - write that down in the data structure!  When some future us needs to deal with it we know what we have *explicitly* - the future is safe and easy to reason about because there\u0027s no need to guess about what we\u0027re looking at.\n\nWhat we should NOT do is see \u0027very-important-checksum-do-not-ignore\u0027 and say \"ok, here\u0027s your data w/o checksumming!\" or encourage out-of-tree extension to use crc or md5?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":542,"context_line":"        \"\"\""},{"line_number":543,"context_line":"        if self.pos !\u003d 0:"},{"line_number":544,"context_line":"            raise IOError(\"Magic must be written at the start of the file\")"},{"line_number":545,"context_line":"        self.write(struct.pack(\"!4sH\", b\"R1NG\", version))"},{"line_number":546,"context_line":""},{"line_number":547,"context_line":"    def write_size(self, size, fmt\u003d\"!Q\"):"},{"line_number":548,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"7af6509d_8828723b","line":545,"updated":"2023-03-29 00:40:00.000000000","message":"it\u0027s weird for me this method isn\u0027t called anywhere from w/i RingWriter - seems leaky, i guess the RingWriter doesn\u0027t really know if it\u0027s v1 or v2 and we pass this instance to either serialize_v1 or serialize_v2.\n\nI\u0027m not sure what you would call this design pattern, but despite having a lot of classes it doesn\u0027t seem to be exactly what I would expect from an OO pattern that has a common/abstract serialization/compression primative that\u0027s extended by two different file format versions.\n\n    BaseRingWriter--\u003e RingWriterV1\n                  \\-\u003e RingWriterV2","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":542,"context_line":"        \"\"\""},{"line_number":543,"context_line":"        if self.pos !\u003d 0:"},{"line_number":544,"context_line":"            raise IOError(\"Magic must be written at the start of the file\")"},{"line_number":545,"context_line":"        self.write(struct.pack(\"!4sH\", b\"R1NG\", version))"},{"line_number":546,"context_line":""},{"line_number":547,"context_line":"    def write_size(self, size, fmt\u003d\"!Q\"):"},{"line_number":548,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"7a378a07_0965ce4c","line":545,"in_reply_to":"7af6509d_8828723b","updated":"2025-06-13 21:04:18.000000000","message":"I\u0027ve actually decided that a \"RingGzWriter\" should be a simple wrapper around GzFile (flush, atomic rename, etc), and the [de]serialization is an operation ON a GzWriter - so I have some \"codecs\" that have a RingGz[Reader|Writer] and provide [de]serialize methods.\n\nIMHO, it\u0027s handy to have the [de]serialize methods next to each other in each class - instead split between a Reader|Writer","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":601,"context_line":""},{"line_number":602,"context_line":"        Callers should not need to use this themselves; it will be done"},{"line_number":603,"context_line":"        automatically when using the writer as a context manager."},{"line_number":604,"context_line":"        \"\"\""},{"line_number":605,"context_line":"        with self.section(\u0027swift/index\u0027):"},{"line_number":606,"context_line":"            self.write_json(self.index)"},{"line_number":607,"context_line":"        # switch to uncompressed"}],"source_content_type":"text/x-python","patch_set":17,"id":"c245c4f3_1811d2e6","line":604,"updated":"2023-03-29 00:40:00.000000000","message":"a lot of these methods are never called in serialize_v1 I hope?  so we\u0027re passing in a somewhat \"overpowered\" writer to that method","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":601,"context_line":""},{"line_number":602,"context_line":"        Callers should not need to use this themselves; it will be done"},{"line_number":603,"context_line":"        automatically when using the writer as a context manager."},{"line_number":604,"context_line":"        \"\"\""},{"line_number":605,"context_line":"        with self.section(\u0027swift/index\u0027):"},{"line_number":606,"context_line":"            self.write_json(self.index)"},{"line_number":607,"context_line":"        # switch to uncompressed"}],"source_content_type":"text/x-python","patch_set":17,"id":"d544ec62_df176983","line":604,"in_reply_to":"370ace37_445f643c","updated":"2025-06-13 21:04:18.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":601,"context_line":""},{"line_number":602,"context_line":"        Callers should not need to use this themselves; it will be done"},{"line_number":603,"context_line":"        automatically when using the writer as a context manager."},{"line_number":604,"context_line":"        \"\"\""},{"line_number":605,"context_line":"        with self.section(\u0027swift/index\u0027):"},{"line_number":606,"context_line":"            self.write_json(self.index)"},{"line_number":607,"context_line":"        # switch to uncompressed"}],"source_content_type":"text/x-python","patch_set":17,"id":"370ace37_445f643c","line":604,"in_reply_to":"c245c4f3_1811d2e6","updated":"2023-04-03 23:05:30.000000000","message":"Yup -- only calls are\n\n writer.write_magic(version\u003d1)\n writer.write_json(_text, \u0027!I\u0027)\n writer.write(part2dev_id.tostring())","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"11b7880a1275f04ac271690978ed36008dde2021","unresolved":true,"context_lines":[{"line_number":414,"context_line":"    def close(self):"},{"line_number":415,"context_line":"        if self.index:"},{"line_number":416,"context_line":"            # only write index if we made use of any sections"},{"line_number":417,"context_line":"            self.write_index()"},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"        # This does three things:"},{"line_number":420,"context_line":"        #   * Flush the underlying compressobj (with Z_FINISH) and write"}],"source_content_type":"text/x-python","patch_set":20,"id":"554731a7_a4a5e408","line":417,"updated":"2025-01-10 04:45:55.000000000","message":"`index` only makes sense in a `RingWriter` -- this function needs to be split across the classes. (Or the classes need to be merged.)","commit_id":"1ff766bcff07c6a038a19518cb569418abe32cbd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":false,"context_lines":[{"line_number":414,"context_line":"    def close(self):"},{"line_number":415,"context_line":"        if self.index:"},{"line_number":416,"context_line":"            # only write index if we made use of any sections"},{"line_number":417,"context_line":"            self.write_index()"},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"        # This does three things:"},{"line_number":420,"context_line":"        #   * Flush the underlying compressobj (with Z_FINISH) and write"}],"source_content_type":"text/x-python","patch_set":20,"id":"da01d03a_c5cd5d84","line":417,"in_reply_to":"554731a7_a4a5e408","updated":"2025-01-28 17:30:08.000000000","message":"Done","commit_id":"1ff766bcff07c6a038a19518cb569418abe32cbd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"58f759cd839f5087aa1db3fea5109ea81b434b5d","unresolved":true,"context_lines":[{"line_number":78,"context_line":"        \"\"\""},{"line_number":79,"context_line":"        Open the ring file ``filename``"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"        :returns: a context manager that provides a ``RingReader`` instance"},{"line_number":82,"context_line":"        \"\"\""},{"line_number":83,"context_line":"        with open(filename, \u0027rb\u0027) as fp:"},{"line_number":84,"context_line":"            yield cls(fp)"}],"source_content_type":"text/x-python","patch_set":21,"id":"9069089b_ad04790e","line":81,"range":{"start_line":81,"start_character":52,"end_line":81,"end_character":69},"updated":"2025-01-15 09:35:33.000000000","message":"Does it return a RingReader instance or a GZipReader instance. I guess maybe the former only when using a RingReader that has inherited this class. Do we maybe want to make this class abstract so it can\u0027t be used except the way we want?\n\nIf this isn\u0027t suppose to be a concrete class I guess it could move to RingReader maybe. but shrug","commit_id":"f7e05f4b57c3701f1232c361db111486a1b13907"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":false,"context_lines":[{"line_number":78,"context_line":"        \"\"\""},{"line_number":79,"context_line":"        Open the ring file ``filename``"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"        :returns: a context manager that provides a ``RingReader`` instance"},{"line_number":82,"context_line":"        \"\"\""},{"line_number":83,"context_line":"        with open(filename, \u0027rb\u0027) as fp:"},{"line_number":84,"context_line":"            yield cls(fp)"}],"source_content_type":"text/x-python","patch_set":21,"id":"b2b0737a_0964f71b","line":81,"range":{"start_line":81,"start_character":52,"end_line":81,"end_character":69},"in_reply_to":"9069089b_ad04790e","updated":"2025-01-28 17:30:08.000000000","message":"I\u0027ll reword to \"a context manager that provides an instance of this class\"","commit_id":"f7e05f4b57c3701f1232c361db111486a1b13907"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a6566844d2ef7139db296d2328bd10aa2dc546b5","unresolved":true,"context_lines":[{"line_number":426,"context_line":"        if not data:"},{"line_number":427,"context_line":"            return 0"},{"line_number":428,"context_line":"        self.flushed \u003d False"},{"line_number":429,"context_line":"        if self.checksum:"},{"line_number":430,"context_line":"            self.checksum.update(data)"},{"line_number":431,"context_line":"        self.pos +\u003d len(data)"},{"line_number":432,"context_line":"        return self.gzip_fp.write(data)"}],"source_content_type":"text/x-python","patch_set":22,"id":"30829149_05e62992","line":429,"updated":"2025-02-04 20:59:16.000000000","message":"This ought to be an `AttributeError`, except for how this only gets used by way of `RingWriter`...","commit_id":"338e28fee15c574a5b45f83b81c2f68fce164614"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e595aaf1343b73d033b89cae2831cb63558fb15b","unresolved":false,"context_lines":[{"line_number":426,"context_line":"        if not data:"},{"line_number":427,"context_line":"            return 0"},{"line_number":428,"context_line":"        self.flushed \u003d False"},{"line_number":429,"context_line":"        if self.checksum:"},{"line_number":430,"context_line":"            self.checksum.update(data)"},{"line_number":431,"context_line":"        self.pos +\u003d len(data)"},{"line_number":432,"context_line":"        return self.gzip_fp.write(data)"}],"source_content_type":"text/x-python","patch_set":22,"id":"a851ccb7_1a29f836","line":429,"in_reply_to":"30829149_05e62992","updated":"2025-03-08 18:27:17.000000000","message":"Done","commit_id":"338e28fee15c574a5b45f83b81c2f68fce164614"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cc2db0a28617560f498d54857747e9591d06abe5","unresolved":true,"context_lines":[{"line_number":290,"context_line":"        # where this 31 (\u003d 18 + 13) came from."},{"line_number":291,"context_line":"        self.seek(-31, os.SEEK_END)"},{"line_number":292,"context_line":"        try:"},{"line_number":293,"context_line":"            index_start, \u003d struct.unpack(\"!Q\", self.read(8))"},{"line_number":294,"context_line":"        except zlib.error:"},{"line_number":295,"context_line":"            # TODO: we can still fix this if we\u0027re willing to read everything"},{"line_number":296,"context_line":"            raise IOError(\"Could not read index offset \""}],"source_content_type":"text/x-python","patch_set":28,"id":"c9970dd8_dc5dead4","line":293,"updated":"2025-04-17 15:36:10.000000000","message":"yeah I don\u0027t think write_index is using write_size to write the *size*","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":290,"context_line":"        # where this 31 (\u003d 18 + 13) came from."},{"line_number":291,"context_line":"        self.seek(-31, os.SEEK_END)"},{"line_number":292,"context_line":"        try:"},{"line_number":293,"context_line":"            index_start, \u003d struct.unpack(\"!Q\", self.read(8))"},{"line_number":294,"context_line":"        except zlib.error:"},{"line_number":295,"context_line":"            # TODO: we can still fix this if we\u0027re willing to read everything"},{"line_number":296,"context_line":"            raise IOError(\"Could not read index offset \""}],"source_content_type":"text/x-python","patch_set":28,"id":"a5909803_86d24f0a","line":293,"in_reply_to":"752b0d53_dbcc337a","updated":"2025-06-13 21:04:18.000000000","message":"yes, I needed to make that connection - thank you.\n\n```\n    def write_size(self, size, fmt\u003d\"!Q\"):\n        \"\"\"\n        Write a BLOB-length.\n```\n\n^ you may forgive me being confused tho.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":false,"context_lines":[{"line_number":290,"context_line":"        # where this 31 (\u003d 18 + 13) came from."},{"line_number":291,"context_line":"        self.seek(-31, os.SEEK_END)"},{"line_number":292,"context_line":"        try:"},{"line_number":293,"context_line":"            index_start, \u003d struct.unpack(\"!Q\", self.read(8))"},{"line_number":294,"context_line":"        except zlib.error:"},{"line_number":295,"context_line":"            # TODO: we can still fix this if we\u0027re willing to read everything"},{"line_number":296,"context_line":"            raise IOError(\"Could not read index offset \""}],"source_content_type":"text/x-python","patch_set":28,"id":"8e48b2fc_fa24b1bb","line":293,"in_reply_to":"a5909803_86d24f0a","updated":"2025-08-01 00:08:57.000000000","message":"Done","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":290,"context_line":"        # where this 31 (\u003d 18 + 13) came from."},{"line_number":291,"context_line":"        self.seek(-31, os.SEEK_END)"},{"line_number":292,"context_line":"        try:"},{"line_number":293,"context_line":"            index_start, \u003d struct.unpack(\"!Q\", self.read(8))"},{"line_number":294,"context_line":"        except zlib.error:"},{"line_number":295,"context_line":"            # TODO: we can still fix this if we\u0027re willing to read everything"},{"line_number":296,"context_line":"            raise IOError(\"Could not read index offset \""}],"source_content_type":"text/x-python","patch_set":28,"id":"752b0d53_dbcc337a","line":293,"in_reply_to":"c9970dd8_dc5dead4","updated":"2025-05-06 20:28:41.000000000","message":"I think of it kind of like `size_t` -- can be used for lengths _or_ offsets.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":true,"context_lines":[{"line_number":356,"context_line":"            if entry.checksum_method is not None:"},{"line_number":357,"context_line":"                logging.getLogger(\u0027swift.ring\u0027).warning("},{"line_number":358,"context_line":"                    \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":359,"context_line":"                    entry.checksum_method, entry.checksum_value, section)"},{"line_number":360,"context_line":"            checksum \u003d checksum_value \u003d None"},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"        with SectionReader("}],"source_content_type":"text/x-python","patch_set":28,"id":"7e3e7825_1fb1597e","line":359,"updated":"2025-03-26 19:12:40.000000000","message":"why is THIS the right answer?  How about we just have one checksum method and we always write it and it\u0027s never invalid?","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":356,"context_line":"            if entry.checksum_method is not None:"},{"line_number":357,"context_line":"                logging.getLogger(\u0027swift.ring\u0027).warning("},{"line_number":358,"context_line":"                    \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":359,"context_line":"                    entry.checksum_method, entry.checksum_value, section)"},{"line_number":360,"context_line":"            checksum \u003d checksum_value \u003d None"},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"        with SectionReader("}],"source_content_type":"text/x-python","patch_set":28,"id":"130530d4_9b06f8fe","line":359,"in_reply_to":"70e60d9b_55c38142","updated":"2025-08-04 07:10:27.000000000","message":"Acknowledged","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":356,"context_line":"            if entry.checksum_method is not None:"},{"line_number":357,"context_line":"                logging.getLogger(\u0027swift.ring\u0027).warning("},{"line_number":358,"context_line":"                    \"Ignoring unsupported checksum %s:%s for section %s\","},{"line_number":359,"context_line":"                    entry.checksum_method, entry.checksum_value, section)"},{"line_number":360,"context_line":"            checksum \u003d checksum_value \u003d None"},{"line_number":361,"context_line":""},{"line_number":362,"context_line":"        with SectionReader("}],"source_content_type":"text/x-python","patch_set":28,"id":"70e60d9b_55c38142","line":359,"in_reply_to":"7e3e7825_1fb1597e","updated":"2025-06-13 21:04:18.000000000","message":"Tim explains this is an attempt at forward compat.  If a new swift started writing ring\u0027s with crc checksum old swift\u0027s could (probably?) still load them and only print a warning that there\u0027s no guarantee they\u0027ve correctly read the unknown/unexpected data.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":565,"context_line":""},{"line_number":566,"context_line":"    def write_size(self, size, fmt\u003d\"!Q\"):"},{"line_number":567,"context_line":"        \"\"\""},{"line_number":568,"context_line":"        Write a BLOB-length."},{"line_number":569,"context_line":""},{"line_number":570,"context_line":"        :param data: the size to write"},{"line_number":571,"context_line":"        :param fmt: the struct format to use when writing the length."}],"source_content_type":"text/x-python","patch_set":28,"id":"b07320f6_f939a32e","line":568,"updated":"2025-06-13 21:04:18.000000000","message":"this helper is used for more than just writing a BLOB-length.  e.g. write_index uses for writing *offsets*\n\nconsider updating the doc string.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":false,"context_lines":[{"line_number":565,"context_line":""},{"line_number":566,"context_line":"    def write_size(self, size, fmt\u003d\"!Q\"):"},{"line_number":567,"context_line":"        \"\"\""},{"line_number":568,"context_line":"        Write a BLOB-length."},{"line_number":569,"context_line":""},{"line_number":570,"context_line":"        :param data: the size to write"},{"line_number":571,"context_line":"        :param fmt: the struct format to use when writing the length."}],"source_content_type":"text/x-python","patch_set":28,"id":"6e68d6c4_38430907","line":568,"in_reply_to":"b07320f6_f939a32e","updated":"2025-08-01 00:08:57.000000000","message":"Done","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cc2db0a28617560f498d54857747e9591d06abe5","unresolved":true,"context_lines":[{"line_number":621,"context_line":"        # switch to uncompressed"},{"line_number":622,"context_line":"        self._set_compression_level(0)"},{"line_number":623,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":624,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027][0])"},{"line_number":625,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":28,"id":"0febca96_c93dea33","line":624,"updated":"2025-04-17 15:36:10.000000000","message":"self.index is an OrderedDict - which holds IndexEntry elements - which are namedtupled - so [0] here is... compressed_start?  Why is that the size?  I thought the size \"will be exactly 18 bytes\"\n\nAre we [ab]using `write_size` for something other than \"write a length prefix ahead of a blog\" here?","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e9d48f8117603f9f7f5ce7a241796b479bbe1719","unresolved":true,"context_lines":[{"line_number":621,"context_line":"        # switch to uncompressed"},{"line_number":622,"context_line":"        self._set_compression_level(0)"},{"line_number":623,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":624,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027][0])"},{"line_number":625,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":28,"id":"2b61268c_6a3aeb22","line":624,"in_reply_to":"0febca96_c93dea33","updated":"2025-06-09 20:13:40.000000000","message":"\u003e I thought the size \"will be exactly 18 bytes\"\n\nThat\u0027s the length we\u0027re adding to the compressed stream between the `write_size` and the `flush` (five bytes for block info, size, and inverted size; 8 bytes of uncompressed data; 5 byte flush marker of an empty uncompressed block [`\\x00\\x00\\x00\\xff\\xff`]).","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":621,"context_line":"        # switch to uncompressed"},{"line_number":622,"context_line":"        self._set_compression_level(0)"},{"line_number":623,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":624,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027][0])"},{"line_number":625,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":28,"id":"c5d81489_96bcd668","line":624,"in_reply_to":"0febca96_c93dea33","updated":"2025-06-13 21:04:18.000000000","message":"it\u0027s more use than abuse - the write_size helper is just \"write int\" - the fmt is (almost?) always `!Q`","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":false,"context_lines":[{"line_number":621,"context_line":"        # switch to uncompressed"},{"line_number":622,"context_line":"        self._set_compression_level(0)"},{"line_number":623,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":624,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027][0])"},{"line_number":625,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":28,"id":"ed0e7340_c5a09c23","line":624,"in_reply_to":"c5d81489_96bcd668","updated":"2025-06-16 23:28:49.000000000","message":"\u003e the fmt is (almost?) always `!Q`\n\nThe exception being v1 rings, which still need `!I`.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":350,"context_line":"            raise IOError(\"Inconsistent section size\")"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"        if entry.checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":353,"context_line":"            checksum \u003d getattr(hashlib, entry.checksum_method)(prefix)"},{"line_number":354,"context_line":"            checksum_value \u003d entry.checksum_value"},{"line_number":355,"context_line":"        else:"},{"line_number":356,"context_line":"            if entry.checksum_method is not None:"}],"source_content_type":"text/x-python","patch_set":34,"id":"a9a3848b_a8ad0137","line":353,"updated":"2025-06-13 21:04:18.000000000","message":"oic, and HERE is where we ensure the prefix does get fed into the read checksum.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":false,"context_lines":[{"line_number":350,"context_line":"            raise IOError(\"Inconsistent section size\")"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"        if entry.checksum_method in (\u0027md5\u0027, \u0027sha1\u0027, \u0027sha256\u0027, \u0027sha512\u0027):"},{"line_number":353,"context_line":"            checksum \u003d getattr(hashlib, entry.checksum_method)(prefix)"},{"line_number":354,"context_line":"            checksum_value \u003d entry.checksum_value"},{"line_number":355,"context_line":"        else:"},{"line_number":356,"context_line":"            if entry.checksum_method is not None:"}],"source_content_type":"text/x-python","patch_set":34,"id":"d53f782f_f7f0be00","line":353,"in_reply_to":"a9a3848b_a8ad0137","updated":"2025-08-04 07:10:27.000000000","message":"Acknowledged","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":483,"context_line":""},{"line_number":484,"context_line":"    Note that the index will only be written if named sections were defined."},{"line_number":485,"context_line":"    \"\"\""},{"line_number":486,"context_line":"    checksum_method \u003d \u0027sha256\u0027"},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"    def __init__(self, *a, **kw):"},{"line_number":489,"context_line":"        super(RingWriter, self).__init__(*a, **kw)"}],"source_content_type":"text/x-python","patch_set":34,"id":"214589e1_a2b3c625","line":486,"updated":"2025-06-13 21:04:18.000000000","message":"I think it\u0027s weird that this is a class attribute.  AFAIK only v2 does checksumming and it\u0027s always sha256 - there\u0027s no subclass and GzipWriter doesn\u0027t look at it.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ef4242d4b8cf95ca87cb59725093fd81a19dbfc5","unresolved":true,"context_lines":[{"line_number":483,"context_line":""},{"line_number":484,"context_line":"    Note that the index will only be written if named sections were defined."},{"line_number":485,"context_line":"    \"\"\""},{"line_number":486,"context_line":"    checksum_method \u003d \u0027sha256\u0027"},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"    def __init__(self, *a, **kw):"},{"line_number":489,"context_line":"        super(RingWriter, self).__init__(*a, **kw)"}],"source_content_type":"text/x-python","patch_set":34,"id":"b8e147ab_8fcd0121","line":486,"in_reply_to":"214589e1_a2b3c625","updated":"2025-06-17 02:56:47.000000000","message":"I\u0027m mainly trying to keep the reference somewhat local -- if I made it module level, it\u0027s moving up about 500 lines when the only place that would use it is in `RingWriter.section`.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":false,"context_lines":[{"line_number":483,"context_line":""},{"line_number":484,"context_line":"    Note that the index will only be written if named sections were defined."},{"line_number":485,"context_line":"    \"\"\""},{"line_number":486,"context_line":"    checksum_method \u003d \u0027sha256\u0027"},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"    def __init__(self, *a, **kw):"},{"line_number":489,"context_line":"        super(RingWriter, self).__init__(*a, **kw)"}],"source_content_type":"text/x-python","patch_set":34,"id":"5a97ee88_e727f71f","line":486,"in_reply_to":"b8e147ab_8fcd0121","updated":"2025-06-18 22:21:44.000000000","message":"Acknowledged","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":571,"context_line":"        :param fmt: the struct format to use when writing the length."},{"line_number":572,"context_line":"                    All v2 BLOBs should use ``!Q``."},{"line_number":573,"context_line":"        \"\"\""},{"line_number":574,"context_line":"        self.write(struct.pack(fmt, size))"},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"    def write_blob(self, data, fmt\u003d\"!Q\"):"},{"line_number":577,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":34,"id":"545ccb8c_fef1e0b7","line":574,"updated":"2025-06-13 21:04:18.000000000","message":"I feel like this should update the checksum to include the size/bytes","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":571,"context_line":"        :param fmt: the struct format to use when writing the length."},{"line_number":572,"context_line":"                    All v2 BLOBs should use ``!Q``."},{"line_number":573,"context_line":"        \"\"\""},{"line_number":574,"context_line":"        self.write(struct.pack(fmt, size))"},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"    def write_blob(self, data, fmt\u003d\"!Q\"):"},{"line_number":577,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":34,"id":"bd6fee5b_dcc3b8aa","line":574,"in_reply_to":"545ccb8c_fef1e0b7","updated":"2025-06-16 23:28:49.000000000","message":"...it does, in `write`.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":571,"context_line":"        :param fmt: the struct format to use when writing the length."},{"line_number":572,"context_line":"                    All v2 BLOBs should use ``!Q``."},{"line_number":573,"context_line":"        \"\"\""},{"line_number":574,"context_line":"        self.write(struct.pack(fmt, size))"},{"line_number":575,"context_line":""},{"line_number":576,"context_line":"    def write_blob(self, data, fmt\u003d\"!Q\"):"},{"line_number":577,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":34,"id":"2615fd60_2132b437","line":574,"in_reply_to":"bd6fee5b_dcc3b8aa","updated":"2025-06-18 22:21:44.000000000","message":"right, *if self.checksum*\n\nI eventually found I preferred to have the checksumming always happen unconditionally when you\u0027re in a section.  This change has a SectionReader, but no SectionWriter - where the SectionReader provides a clear responsibility boundary the writer just installs/removes check-summing on itself directly.\n\nI did waffle on:\n\n```\nwith self.create_section() as writer:\n    write_length_prefixed_blob(writer, blob)\n```\n\nvs. \n\n```\nwith self.create_section(name):\n    write_length_prefixed_blob(self.fp, blob)\n```\n\nand ultimately opt\u0027d to support both - similar to how you temporarily install a checksumer while in a section, I would swap out `orig_fp, self.fp \u003d self.fp, SectionWriter(self.fp)` with-in the context manager.\n\n... it\u0027s not needed, but reduced some test churn and removed any ambiguity on if writing to `self.fp` w/i the `create_section` context manager should or shouldn\u0027t update checksumming.\n\nUltimately I\u0027m not sure why we want to support ANY writing into the ringv2 format outside of a section sans the index pointers; but I did eventually learn that the length prefix which sections MUST have IS part of their checksum both in read and write.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":49,"context_line":"        self.pos \u003d self.fp.tell()"},{"line_number":50,"context_line":"        if self.pos \u003d\u003d 0:"},{"line_number":51,"context_line":"            # Expect gzip header"},{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"}],"source_content_type":"text/x-python","patch_set":36,"id":"1a0fdc99_b7dce7b0","line":52,"updated":"2025-06-13 21:04:18.000000000","message":"\u003e Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection\n\nhttps://zlib.net/manual.html\n\nthe original implementation before this code moved was:\n\n```\nself._decomp \u003d zlib.decompressobj(32 + zlib.MAX_WBITS)\n```\n\n... why is 16 the right answer now?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":49,"context_line":"        self.pos \u003d self.fp.tell()"},{"line_number":50,"context_line":"        if self.pos \u003d\u003d 0:"},{"line_number":51,"context_line":"            # Expect gzip header"},{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"}],"source_content_type":"text/x-python","patch_set":36,"id":"40a22878_d3ddc1a0","line":52,"in_reply_to":"1a0fdc99_b7dce7b0","updated":"2025-06-16 23:28:49.000000000","message":"Because encountering a raw deflate stream when we were expecting something with gzip header/tail *should* be an error.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4b26e6feb579985c28c6f80724cbceab1814d0a4","unresolved":false,"context_lines":[{"line_number":49,"context_line":"        self.pos \u003d self.fp.tell()"},{"line_number":50,"context_line":"        if self.pos \u003d\u003d 0:"},{"line_number":51,"context_line":"            # Expect gzip header"},{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"}],"source_content_type":"text/x-python","patch_set":36,"id":"8d7a659d_cadeb41f","line":52,"in_reply_to":"1bd04c01_3bd8ffca","updated":"2025-07-30 18:38:26.000000000","message":"Acknowledged","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":49,"context_line":"        self.pos \u003d self.fp.tell()"},{"line_number":50,"context_line":"        if self.pos \u003d\u003d 0:"},{"line_number":51,"context_line":"            # Expect gzip header"},{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"}],"source_content_type":"text/x-python","patch_set":36,"id":"1bd04c01_3bd8ffca","line":52,"in_reply_to":"40a22878_d3ddc1a0","updated":"2025-06-18 22:21:44.000000000","message":"\u003e add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR).\n\nok, so the original optimization was just lazy and we could fix that independently of implementing ringv2 if we wanted.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"},{"line_number":56,"context_line":"        self.decompressor \u003d zlib.decompressobj(wbits)"},{"line_number":57,"context_line":"        self.buffer \u003d self.compressed_buffer \u003d b\"\""},{"line_number":58,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"46d363fe_59835207","line":55,"updated":"2025-06-13 21:04:18.000000000","message":"I get that pos !\u003d 0 would be a \"raw stream\" - I don\u0027t know why `-zlib.MAX_WBITS` is the correct answer:\n\n```\nThe wbits argument controls the size of the history buffer (or the “window\nsize”) used when compressing data, and whether a header and trailer is included\nin the output. It can take several ranges of values, defaulting to 15\n(MAX_WBITS):\n\n+9 to +15: The base-two logarithm of the window size, which therefore ranges\nbetween 512 and 32768. Larger values produce better compression at the expense\nof greater memory usage. The resulting output will include a zlib-specific\nheader and trailer.\n\n−9 to −15: Uses the absolute value of wbits as the window size logarithm, while\nproducing a raw output stream with no header or trailing checksum.\n\n+25 to +31 \u003d 16 + (9 to 15): Uses the low 4 bits of the value as the window\nsize logarithm, while including a basic gzip header and trailing checksum in\nthe output.\n\n\n```\n\nhttps://docs.python.org/3/library/zlib.html#zlib.compress\n\nIs gzip not ok with the zlib headers/trailer in the deflate blocks?  Is using an absolute value of 15 better compression than \"ranges between 512 and 32768\"","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"},{"line_number":56,"context_line":"        self.decompressor \u003d zlib.decompressobj(wbits)"},{"line_number":57,"context_line":"        self.buffer \u003d self.compressed_buffer \u003d b\"\""},{"line_number":58,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"c7ee1254_e32e47a5","line":55,"in_reply_to":"2420254f_816ed369","updated":"2025-06-18 22:21:44.000000000","message":"Thank you!  For decompression of the individual deflate stream blocks w/i a gzip file I\u0027m convinced `-zlib.MAX_WBITS` as the most correct answer.\n\n```\n\u003e\u003e\u003e for i in range(-31, 32):\n...     try:\n...         v \u003d len(zlib.compress(data * 20, wbits\u003di))\n...     except Exception as e:\n...         v \u003d e\n...     print(i, v)\n... \n-31 Bad compression level\n-30 Bad compression level\n-29 Bad compression level\n-28 Bad compression level\n-27 Bad compression level\n-26 Bad compression level\n-25 Bad compression level\n-24 Bad compression level\n-23 Bad compression level\n-22 Bad compression level\n-21 Bad compression level\n-20 Bad compression level\n-19 Bad compression level\n-18 Bad compression level\n-17 Bad compression level\n-16 Bad compression level\n-15 1224\n-14 1224\n-13 1224\n-12 1224\n-11 1224\n-10 20196\n-9 20196\n-8 Bad compression level\n-7 Bad compression level\n-6 Bad compression level\n-5 Bad compression level\n-4 Bad compression level\n-3 Bad compression level\n-2 Bad compression level\n-1 Bad compression level\n0 Bad compression level\n1 Bad compression level\n2 Bad compression level\n3 Bad compression level\n4 Bad compression level\n5 Bad compression level\n6 Bad compression level\n7 Bad compression level\n8 20202\n9 20202\n10 20202\n11 1230\n12 1230\n13 1230\n14 1230\n15 1230\n16 Bad compression level\n17 Bad compression level\n18 Bad compression level\n19 Bad compression level\n20 Bad compression level\n21 Bad compression level\n22 Bad compression level\n23 Bad compression level\n24 Bad compression level\n25 20214\n26 20214\n27 1242\n28 1242\n29 1242\n30 1242\n31 1242\n\u003e\u003e\u003e -zlib.MAX_WBITS\n-15\n```","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"},{"line_number":56,"context_line":"        self.decompressor \u003d zlib.decompressobj(wbits)"},{"line_number":57,"context_line":"        self.buffer \u003d self.compressed_buffer \u003d b\"\""},{"line_number":58,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"2420254f_816ed369","line":55,"in_reply_to":"46d363fe_59835207","updated":"2025-06-16 23:28:49.000000000","message":"\u003e used when compressing data\n\nRight, so you can use larger `wbits` to trade increased memory usage during compression for smaller compressed output:\n```\n\u003e\u003e\u003e len(zlib.compress(os.urandom(2**10) * 20, wbits\u003d9))\n20189\n\u003e\u003e\u003e len(zlib.compress(os.urandom(2**10) * 20, wbits\u003d15))\n1230\n```\nDuring *decompression*, though, you need to either know the `wbits` used, or err on the side of too-large, so *something* involving `MAX_WBITS` is definitely good:\n```\n\u003e\u003e\u003e len(zlib.decompress(zlib.compress(os.urandom(2**10) * 20, wbits\u003d9), wbits\u003d15))\n20480\n\u003e\u003e\u003e len(zlib.decompress(zlib.compress(os.urandom(2**10) * 20, wbits\u003d15), wbits\u003d9))\nTraceback (most recent call last):\n  File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\nzlib.error: Error -3 while decompressing data: invalid window size\n```\nSo, should it be positive or negative? It doesn\u0027t work great to mix the two:\n```\n\u003e\u003e\u003e len(zlib.decompress(zlib.compress(os.urandom(2**10) * 20, wbits\u003d9), wbits\u003d-15))\nTraceback (most recent call last):\n  File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\nzlib.error: Error -3 while decompressing data: invalid stored block lengths\n\u003e\u003e\u003e len(zlib.decompress(zlib.compress(os.urandom(2**10) * 20, wbits\u003d-9), wbits\u003d15))\nTraceback (most recent call last):\n  File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\nzlib.error: Error -3 while decompressing data: incorrect header check\n```\nThat\u0027s because a positive `wbits` produces a zlib stream (see [RFC 1951](https://www.rfc-editor.org/rfc/rfc1951)), while negative produces the raw deflate stream (see [RFC 1950](https://www.rfc-editor.org/rfc/rfc1950)).\n\n\u003e Is gzip not ok with the zlib headers/trailer in the deflate blocks?\n\nFrom [RFC 1952](https://www.rfc-editor.org/rfc/rfc1952):\n```\n         CM (Compression Method)\n            This identifies the compression method used in the file.  CM\n            \u003d 0-7 are reserved.  CM \u003d 8 denotes the \"deflate\"\n            compression method, which is the one customarily used by\n            gzip and which is documented elsewhere.\n```\nso, deflate stream, not zlib stream -- which matches [what python\u0027s doing](https://github.com/python/cpython/blob/v3.13.5/Lib/gzip.py#L231-L235) (and has been doing for decades).","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":false,"context_lines":[{"line_number":52,"context_line":"            wbits \u003d 16 + zlib.MAX_WBITS"},{"line_number":53,"context_line":"        else:"},{"line_number":54,"context_line":"            # Raw stream"},{"line_number":55,"context_line":"            wbits \u003d -zlib.MAX_WBITS"},{"line_number":56,"context_line":"        self.decompressor \u003d zlib.decompressobj(wbits)"},{"line_number":57,"context_line":"        self.buffer \u003d self.compressed_buffer \u003d b\"\""},{"line_number":58,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"399536e2_6490e6ad","line":55,"in_reply_to":"c7ee1254_e32e47a5","updated":"2025-07-30 07:49:51.000000000","message":"This seems resolved then","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":203,"context_line":"            self._checksum.update(data)"},{"line_number":204,"context_line":"        return data"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"    def read_ring_table(self, itemsize, partition_count):"},{"line_number":207,"context_line":"        max_row_len \u003d itemsize * partition_count"},{"line_number":208,"context_line":"        type_code \u003d BYTES_TO_TYPE_CODE[itemsize]"},{"line_number":209,"context_line":"        return ["}],"source_content_type":"text/x-python","patch_set":36,"id":"cf61825b_611d5431","line":206,"updated":"2025-06-13 21:04:18.000000000","message":"looking at `write_ring_table` it\u0027s pretty clear that method writes the size:\n\n`assignments * dev_id_bytes`\n\n... but I guess those bytes get consumed as part of opening the section.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":203,"context_line":"            self._checksum.update(data)"},{"line_number":204,"context_line":"        return data"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"    def read_ring_table(self, itemsize, partition_count):"},{"line_number":207,"context_line":"        max_row_len \u003d itemsize * partition_count"},{"line_number":208,"context_line":"        type_code \u003d BYTES_TO_TYPE_CODE[itemsize]"},{"line_number":209,"context_line":"        return ["}],"source_content_type":"text/x-python","patch_set":36,"id":"7cf48823_7e938f6d","line":206,"in_reply_to":"c16974b5_f75e969d","updated":"2025-06-18 22:21:44.000000000","message":"\u003e ...and? \n\nit was a comment about symmetry - you have two methods right next to each other, one writes size then blob - one reads just the blob; which raises the question \"where did the size bytes go?\"\n\n... the answer turns out some other code somewhere else already used them - and you just have to know that and accept that these two methods don\u0027t have symmetry.\n\nAnother observation was that the byte enforcing section reader didn\u0027t seem strictly necessary when we\u0027re using length prefixed blobs anyway.  The one place it was actually useful to wrap the blob in a length limiting reader was when reading a ring table who\u0027s final row was truncated by fractional replicas.  This one specific blob really wants to be able to try to read more bytes than are available when populating the array.  OTOH, length prefixed json blobs didn\u0027t really ever try to read more bytes than was available anyway.\n\nSo it actually seemed not just symmetrical but USEFUL to let a method/function who\u0027s parsing a section have it\u0027s length prefix.\n\nFWIW I did end up enforcing section length using the difference between compressed_start and compressed_end in the index.  The only section that trick didn\u0027t work with is the index itself ... if you consider that a \"section\".","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":203,"context_line":"            self._checksum.update(data)"},{"line_number":204,"context_line":"        return data"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"    def read_ring_table(self, itemsize, partition_count):"},{"line_number":207,"context_line":"        max_row_len \u003d itemsize * partition_count"},{"line_number":208,"context_line":"        type_code \u003d BYTES_TO_TYPE_CODE[itemsize]"},{"line_number":209,"context_line":"        return ["}],"source_content_type":"text/x-python","patch_set":36,"id":"c16974b5_f75e969d","line":206,"in_reply_to":"cf61825b_611d5431","updated":"2025-06-16 23:28:49.000000000","message":"...and? We use that overall length to cap our overall reads, sure, but you *need* this other context to know how to interpret the data. It\u0027s not enough to say \"Here\u0027s a ring table, it\u0027s got 1,600,000 bytes\" -- you need to know two out of three of\n\n- item size\n- partition count (or part power)\n- ceil(replicas)","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        self.close()"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"@dataclasses.dataclass(frozen\u003dTrue)"},{"line_number":238,"context_line":"class IndexEntry:"},{"line_number":239,"context_line":"    compressed_start: int"},{"line_number":240,"context_line":"    uncompressed_start: int"}],"source_content_type":"text/x-python","patch_set":36,"id":"2714f5d5_7c4e292b","line":237,"updated":"2025-06-13 21:04:18.000000000","message":"https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass\n\n\u003e frozen: If true (the default is False), assigning to fields will generate an exception. This emulates read-only frozen instances. If __setattr__() or __delattr__() is defined in the class, then TypeError is raised\n\nhttps://docs.python.org/3/library/dataclasses.html#frozen-instances\n\n\u003e There is a tiny performance penalty when using frozen\u003dTrue: __init__() cannot use simple assignment to initialize fields, and must use object.__setattr__().\n\nI think this is why we see a lot of `dataclasses.replace` in tests...\n\nhttps://docs.python.org/3/library/dataclasses.html#dataclasses.replace\n\nAlso it apparently makes the dataclass objects hashable: but I don\u0027t think they\u0027re used dict/index keys.\n\nApparently dataclasses buy you automatic equality comparison, but not hashibility - which I don\u0027t think *strictly* requires immutability but is probably a lot easier to get right with immutable objects.\n\n```\n\u003e\u003e\u003e class MyObject:\n...     foo: int\n... \n\u003e\u003e\u003e {MyObject()}\n{\u003c__main__.MyObject object at 0x751609a63c80\u003e}\n\u003e\u003e\u003e import dataclasses\n\u003e\u003e\u003e @dataclasses.dataclass\n... class MyStruct:\n...     foo: int\n... \n\u003e\u003e\u003e {MyStruct(1)}\nTraceback (most recent call last):\n  File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\nTypeError: unhashable type: \u0027MyStruct\u0027\n```\n\nThere\u0027s some marginal boilerplate reduction, AFAIK the only place we compare IndexEntry equality is in the round-trip tests.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        self.close()"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"@dataclasses.dataclass(frozen\u003dTrue)"},{"line_number":238,"context_line":"class IndexEntry:"},{"line_number":239,"context_line":"    compressed_start: int"},{"line_number":240,"context_line":"    uncompressed_start: int"}],"source_content_type":"text/x-python","patch_set":36,"id":"3bf929a0_9813a6ea","line":237,"in_reply_to":"2671531f_c375c4d0","updated":"2025-06-18 22:21:44.000000000","message":"\u003e in general, it shouldn\u0027t be updated\n\nthat doesn\u0027t sound right, in the create_section helper you *always* make an anemic IndexEntry instance and insert it to the index before you yield and then you update the end bits after the fact (or rather replace the value in self.index with a new IndexEntry based of the existing one but with the originally anemic values replaced with the new correct values)\n\nFWIW in the other implementation I just wait to insert the complete/correct IndexEntry at the end of the context manager and that seemed to work great.  I\u0027m not sure I understand the value/motivation of having the unfinished IndexEntry in self.index while you\u0027re writing to it and wouldn\u0027t mind seeing that extra code/behavior removed as YAGNI.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        self.close()"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"@dataclasses.dataclass(frozen\u003dTrue)"},{"line_number":238,"context_line":"class IndexEntry:"},{"line_number":239,"context_line":"    compressed_start: int"},{"line_number":240,"context_line":"    uncompressed_start: int"}],"source_content_type":"text/x-python","patch_set":36,"id":"2671531f_c375c4d0","line":237,"in_reply_to":"2714f5d5_7c4e292b","updated":"2025-06-16 23:28:49.000000000","message":"I made it frozen because, in general, it shouldn\u0027t be updated. If you find yourself wanting to make manual updates to an existing `IndexEntry`, you\u0027re probably holding the tool wrong.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":244,"context_line":"    checksum_value: Optional[str] \u003d None"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    @property"},{"line_number":247,"context_line":"    def uncompressed_length(self) -\u003e Optional[int]:"},{"line_number":248,"context_line":"        if self.uncompressed_end is None:"},{"line_number":249,"context_line":"            return None"},{"line_number":250,"context_line":"        return self.uncompressed_end - self.uncompressed_start"}],"source_content_type":"text/x-python","patch_set":36,"id":"bb74a10a_29cf51b8","line":247,"updated":"2025-06-13 21:04:18.000000000","message":"is this the ONLY type hint in all of swift now?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":true,"context_lines":[{"line_number":244,"context_line":"    checksum_value: Optional[str] \u003d None"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    @property"},{"line_number":247,"context_line":"    def uncompressed_length(self) -\u003e Optional[int]:"},{"line_number":248,"context_line":"        if self.uncompressed_end is None:"},{"line_number":249,"context_line":"            return None"},{"line_number":250,"context_line":"        return self.uncompressed_end - self.uncompressed_start"}],"source_content_type":"text/x-python","patch_set":36,"id":"33d7acc4_5fde6715","line":247,"in_reply_to":"690f958a_fcee0912","updated":"2025-08-05 07:23:12.000000000","message":"\u003e I\u0027m not sure how I want to tackle type hints. Presumably with some kind of mypy oriented linting?\n\nProbably the right direction. FWIW, `mypy swift` currently reports 168 errors on master; tacking on a `--ignore-missing-imports` brings it down to 56. But tacking on `--check-untyped-defs` (which seems more like what I\u0027d want to be targeting, if not full-on `--disallow-untyped-defs`) brings it up to 945.\n\nHaven\u0027t looked deeply into it, but I\u0027d guess we\u0027d want some kind of ratchet going like we did for py3. Now I want to go try it out in pyeclib or something first and see how it goes...\n\n\u003e With out that it\u0027ll probably rust like any other comment/doc ...\n\nBetter to have it and have it rust than to never have it at all. 🤷","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":244,"context_line":"    checksum_value: Optional[str] \u003d None"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    @property"},{"line_number":247,"context_line":"    def uncompressed_length(self) -\u003e Optional[int]:"},{"line_number":248,"context_line":"        if self.uncompressed_end is None:"},{"line_number":249,"context_line":"            return None"},{"line_number":250,"context_line":"        return self.uncompressed_end - self.uncompressed_start"}],"source_content_type":"text/x-python","patch_set":36,"id":"f501ca28_82e29f34","line":247,"in_reply_to":"bb74a10a_29cf51b8","updated":"2025-06-16 23:28:49.000000000","message":"Probably -- I can drop it if you really don\u0027t want it, but I view it basically as docs.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":244,"context_line":"    checksum_value: Optional[str] \u003d None"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    @property"},{"line_number":247,"context_line":"    def uncompressed_length(self) -\u003e Optional[int]:"},{"line_number":248,"context_line":"        if self.uncompressed_end is None:"},{"line_number":249,"context_line":"            return None"},{"line_number":250,"context_line":"        return self.uncompressed_end - self.uncompressed_start"}],"source_content_type":"text/x-python","patch_set":36,"id":"690f958a_fcee0912","line":247,"in_reply_to":"f501ca28_82e29f34","updated":"2025-06-18 22:21:44.000000000","message":"I\u0027m not sure how I want to tackle type hints.  Presumably with some kind of mypy oriented linting?  With out that it\u0027ll probably rust like any other comment/doc ...","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":287,"context_line":"        do this themselves."},{"line_number":288,"context_line":"        \"\"\""},{"line_number":289,"context_line":"        if self.version !\u003d 2:"},{"line_number":290,"context_line":"            return"},{"line_number":291,"context_line":""},{"line_number":292,"context_line":"        # See notes in RingWriter.write_index and RingWriter.__exit__ for"},{"line_number":293,"context_line":"        # where this 31 (\u003d 18 + 13) came from."}],"source_content_type":"text/x-python","patch_set":36,"id":"6cb96aff_8c301370","line":290,"updated":"2025-06-13 21:04:18.000000000","message":"it\u0027s so strange to me to see the v2 [de]serialization logic in a embedded in a class used by v1 [de]serialization","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":325,"context_line":"    def read_section(self, section):"},{"line_number":326,"context_line":"        \"\"\""},{"line_number":327,"context_line":"        Read an entire section\u0027s data"},{"line_number":328,"context_line":"        \"\"\""},{"line_number":329,"context_line":"        with self.open_section(section) as reader:"},{"line_number":330,"context_line":"            return reader.read()"},{"line_number":331,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"9030ba96_b65c58f0","line":328,"updated":"2025-06-13 21:04:18.000000000","message":"I struggled a little bit to keep straight the difference between read_blob and read_section - fundamentally I think it\u0027s about the responsibility of reading the size.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":325,"context_line":"    def read_section(self, section):"},{"line_number":326,"context_line":"        \"\"\""},{"line_number":327,"context_line":"        Read an entire section\u0027s data"},{"line_number":328,"context_line":"        \"\"\""},{"line_number":329,"context_line":"        with self.open_section(section) as reader:"},{"line_number":330,"context_line":"            return reader.read()"},{"line_number":331,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"aa99b09d_c21dbac9","line":328,"in_reply_to":"9030ba96_b65c58f0","updated":"2025-06-16 23:28:49.000000000","message":"I\u0027d view it more as being about how you found the data to read. Are you trying to find a specific named section? Use `read_section`; it\u0027ll take care of looking it up in the index, seeking to the appropriate position, etc. Did you already make sure you\u0027re at the appropriate position? Use the lower-level `read_blob`.\n\nMaybe the feedback is to get rid of `read_blob` and instead inline it in the two places we use it? But I rather like the symmetry with `write_blob`...","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":325,"context_line":"    def read_section(self, section):"},{"line_number":326,"context_line":"        \"\"\""},{"line_number":327,"context_line":"        Read an entire section\u0027s data"},{"line_number":328,"context_line":"        \"\"\""},{"line_number":329,"context_line":"        with self.open_section(section) as reader:"},{"line_number":330,"context_line":"            return reader.read()"},{"line_number":331,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"b9d732a7_7a0cb412","line":328,"in_reply_to":"aa99b09d_c21dbac9","updated":"2025-06-18 22:21:44.000000000","message":"\u003eDid you already make sure you\u0027re at the appropriate position? Use the lower-level read_blob.\n\nok, that\u0027s maybe helpful; there\u0027s fundamentally two different ways to use a RingReader and at least in tests I saw both getting used - even for v2 rings.\n\nIn the actual implementation read_section doesn\u0027t work with v1 and in v2 read_blob doesn\u0027t work with open_section - but ... they\u0027re both hanging off the same class.  This RingReader doesn\u0027t feel so much like an abstraction as a swiss-army-knife with lots of pointy tools sticking out that we can pass around to different context because they don\u0027t currently have in bugs that use the wrong methods and cut themselves.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b8c3e0c58b98e63c98af654b2c1596bd751e97ae","unresolved":true,"context_lines":[{"line_number":325,"context_line":"    def read_section(self, section):"},{"line_number":326,"context_line":"        \"\"\""},{"line_number":327,"context_line":"        Read an entire section\u0027s data"},{"line_number":328,"context_line":"        \"\"\""},{"line_number":329,"context_line":"        with self.open_section(section) as reader:"},{"line_number":330,"context_line":"            return reader.read()"},{"line_number":331,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"a724e6d9_43790481","line":328,"in_reply_to":"b9d732a7_7a0cb412","updated":"2025-08-04 07:10:27.000000000","message":"The `read_blob` does mention:\n```\nNote that the RingReader needs to already be positioned correctly.\n```\n\nBut fair point talking about possible conflict using both method danging off the same class. However, one would hope if you used open section then you\u0027ll use the returned reader to read that section and not then go call another method.\n\nMaybe we could have a RawRingReader with lower level class and methods and a RingReader with just the sections of something. But that\u0027s an API change so would rather that a follow up about that goes in before a swift release so we don\u0027t have to think about supporting the old way.\n\nBut also arguable the existing doc already alludes to the method needing to be positioned properly.\n\nHappy to land now and all this is a follow up. But if we do decide to tweak this API I\u0027d like it done before the next release, I beleve Tim is working on one now, so maybe wait until he has that release done?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":347,"context_line":"        self.seek(entry.compressed_start)"},{"line_number":348,"context_line":""},{"line_number":349,"context_line":"        prefix \u003d self.read(8)"},{"line_number":350,"context_line":"        blob_length, \u003d struct.unpack(\"!Q\", prefix)"},{"line_number":351,"context_line":"        if entry.compressed_end is not None and \\"},{"line_number":352,"context_line":"                blob_length + 8 !\u003d entry.uncompressed_length:"},{"line_number":353,"context_line":"            raise IOError(\"Inconsistent section size\")"}],"source_content_type":"text/x-python","patch_set":36,"id":"b7802bd6_5dcd38cc","line":350,"updated":"2025-06-13 21:04:18.000000000","message":"this 8 should probably come from `struct.calcsize(self.blob_length_fmt)`\n\nIn this implementation when you open a section the size is automatically consumed - in contrast to when you create a section (where it\u0027s the caller\u0027s responsibility to write the size).\n\nI think fundamentally, as convenient as it might be to enforce that every section you open start with a size and prevent reading beyond that size the structure of the index offers sections an alternative for size limiting and even blob length encoding.\n\nIf you call open_section and read_blob you\u0027re going to trying to double account for the size field - I think it\u0027s a footgun to try and offer both abstractions on the same class.\n\nMy recommendation is that open section should leave length reading to the caller (just like create_section does for length writing) and if we want to enforce length limiting it should be calculated from the end index w/o consuming the size_t/length-field from the check-summed section body.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":false,"context_lines":[{"line_number":347,"context_line":"        self.seek(entry.compressed_start)"},{"line_number":348,"context_line":""},{"line_number":349,"context_line":"        prefix \u003d self.read(8)"},{"line_number":350,"context_line":"        blob_length, \u003d struct.unpack(\"!Q\", prefix)"},{"line_number":351,"context_line":"        if entry.compressed_end is not None and \\"},{"line_number":352,"context_line":"                blob_length + 8 !\u003d entry.uncompressed_length:"},{"line_number":353,"context_line":"            raise IOError(\"Inconsistent section size\")"}],"source_content_type":"text/x-python","patch_set":36,"id":"b55a9937_70ed115a","line":350,"in_reply_to":"b7802bd6_5dcd38cc","updated":"2025-08-01 00:08:57.000000000","message":"the 8 is now calculated from a constant `V2_SIZE_FORMAT \u003d \"!Q\"` which is perfect.  I moved the rest of the comment back into context.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":459,"context_line":"        and start reading, flush the writer first."},{"line_number":460,"context_line":""},{"line_number":461,"context_line":"        If you want the position within the *uncompressed* stream, use the"},{"line_number":462,"context_line":"        ``pos`` attribute."},{"line_number":463,"context_line":"        \"\"\""},{"line_number":464,"context_line":"        self.flush()"},{"line_number":465,"context_line":"        return self.raw_fp.tell()"}],"source_content_type":"text/x-python","patch_set":36,"id":"f3463d6e_b9ca0d8d","line":462,"updated":"2025-06-13 21:04:18.000000000","message":"I think this is confusingly backwards - a gzip.GzipFile() has a tell and it returns the pos in the uncompressed stream.  I think it could be argued it\u0027d be most reasonable for seek and tell to stay consistent with the GzipFile interfaces and this wrapper object would expose a *new* api for the new interface:\n\ni.e. compressed_tell()\n\n... which would pair nicely with a RingGzReader.compressed_seek()","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":465,"context_line":"        return self.raw_fp.tell()"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"    def _set_compression_level(self, lvl):"},{"line_number":468,"context_line":"        # two valid deflate streams may be concentated to produce another"},{"line_number":469,"context_line":"        # valid deflate stream, so finish the one stream..."},{"line_number":470,"context_line":"        self.flush()"},{"line_number":471,"context_line":"        # ... so we can start up another with whatever level we want"}],"source_content_type":"text/x-python","patch_set":36,"id":"6e8ed4be_eea141fe","line":468,"range":{"start_line":468,"start_character":43,"end_line":468,"end_character":54},"updated":"2025-06-13 21:04:18.000000000","message":"concatenated?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":false,"context_lines":[{"line_number":465,"context_line":"        return self.raw_fp.tell()"},{"line_number":466,"context_line":""},{"line_number":467,"context_line":"    def _set_compression_level(self, lvl):"},{"line_number":468,"context_line":"        # two valid deflate streams may be concentated to produce another"},{"line_number":469,"context_line":"        # valid deflate stream, so finish the one stream..."},{"line_number":470,"context_line":"        self.flush()"},{"line_number":471,"context_line":"        # ... so we can start up another with whatever level we want"}],"source_content_type":"text/x-python","patch_set":36,"id":"91cd56a5_b1a141d9","line":468,"range":{"start_line":468,"start_character":43,"end_line":468,"end_character":54},"in_reply_to":"6e8ed4be_eea141fe","updated":"2025-08-01 00:08:57.000000000","message":"Done","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":628,"context_line":"        # switch to uncompressed"},{"line_number":629,"context_line":"        self._set_compression_level(0)"},{"line_number":630,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":631,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027].compressed_start)"},{"line_number":632,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":36,"id":"887de10f_f7b551c1","line":631,"updated":"2025-06-13 21:04:18.000000000","message":"this comment confuses me - it reads like the value of \"compressed_start\" will be \"18 bytes\"\n\nmaybe \"... which ensures we can always look at the start of the index at exactly an 18 byte offset\"","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":629,"context_line":"        self._set_compression_level(0)"},{"line_number":630,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":631,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027].compressed_start)"},{"line_number":632,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":36,"id":"906916c3_7b2cd9b8","line":632,"updated":"2025-06-13 21:04:18.000000000","message":"when looking at reading a ringv2 using a unmodified-buffered-gzip-fileobj it turned it it would have been useful to also include the \"uncompressed_start\"\n\nI haven\u0027t personally tried to review Charles golang ringv2 reader (he needs a programmatic way to get ring.devs out of a v2 .ring.gz so he can do essentially a \"what are my devs\" lookup based on ip similar to the code in servers-per-port) - but I imagine in a pinch someone might wish to forego most of the fancy \"only read what you need!\" stuff just to get a \"quick and dirty read the whole thing and pull out what I want\"...","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":false,"context_lines":[{"line_number":629,"context_line":"        self._set_compression_level(0)"},{"line_number":630,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":631,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027].compressed_start)"},{"line_number":632,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":36,"id":"a3833576_79757ccd","line":632,"in_reply_to":"906916c3_7b2cd9b8","updated":"2025-06-16 23:28:49.000000000","message":"Done in PS38.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d88180c6f6bb817acb32adfb66b34ee4ead0101b","unresolved":true,"context_lines":[{"line_number":260,"context_line":"    def uncompressed_length(self) -\u003e Optional[int]:"},{"line_number":261,"context_line":"        if self.uncompressed_end is None:"},{"line_number":262,"context_line":"            return None"},{"line_number":263,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"class RingReader(GzipReader):"}],"source_content_type":"text/x-python","patch_set":37,"id":"cdb00032_b0fb3939","line":263,"updated":"2025-06-10 17:48:31.000000000","message":"Might should add\n```\n    @property\n    def compressed_length(self) -\u003e Optional[int]:\n        if self.compressed_end is None:\n            return None\n        return self.compressed_end - self.compressed_start\n\n    @property\n    def compression_ratio(self) -\u003e Optional[float]:\n        if self.uncompressed_end is None:\n            return None\n        return 1 - self.compressed_length / self.uncompressed_length\n```","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9e5734a3b621024408970e6b4e87178824d45db6","unresolved":false,"context_lines":[{"line_number":260,"context_line":"    def uncompressed_length(self) -\u003e Optional[int]:"},{"line_number":261,"context_line":"        if self.uncompressed_end is None:"},{"line_number":262,"context_line":"            return None"},{"line_number":263,"context_line":"        return self.uncompressed_end - self.uncompressed_start"},{"line_number":264,"context_line":""},{"line_number":265,"context_line":""},{"line_number":266,"context_line":"class RingReader(GzipReader):"}],"source_content_type":"text/x-python","patch_set":37,"id":"c15ef0be_32189439","line":263,"in_reply_to":"cdb00032_b0fb3939","updated":"2025-06-11 21:00:49.000000000","message":"Done","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c17ac17c0f8a906b6f1b48db79fe079fff14dbc1","unresolved":true,"context_lines":[{"line_number":640,"context_line":"        # switch to uncompressed"},{"line_number":641,"context_line":"        self._set_compression_level(0)"},{"line_number":642,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":643,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027].compressed_start)"},{"line_number":644,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":37,"id":"926633bd_dd8d12d1","line":643,"updated":"2025-06-10 18:06:55.000000000","message":"We might should change this to write both `compressed_start` and `uncompressed_start` -- we\u0027ve got some SRE tooling in golang that wants to crack open a ring and pull out the local devices. Currently it makes no use of the index; instead it buffers the whole uncompressed content and relies on order in which sections are written. It\u0027d be nice if that could find and use the index fairly easily.\n\nNote that we\u0027d probably want to implement it as\n```\n        self.write_size(self.index[\u0027swift/index\u0027].uncompressed_start)\n        self.flush()\n        self.write_size(self.index[\u0027swift/index\u0027].compressed_start)\n        self.flush()\n```\nso we can use the same code to read rings written with and without the uncompressed start.","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9e5734a3b621024408970e6b4e87178824d45db6","unresolved":false,"context_lines":[{"line_number":640,"context_line":"        # switch to uncompressed"},{"line_number":641,"context_line":"        self._set_compression_level(0)"},{"line_number":642,"context_line":"        # ... which allows us to know that this will be exactly 18 bytes"},{"line_number":643,"context_line":"        self.write_size(self.index[\u0027swift/index\u0027].compressed_start)"},{"line_number":644,"context_line":"        self.flush()"}],"source_content_type":"text/x-python","patch_set":37,"id":"6fd5f9eb_f1b1d514","line":643,"in_reply_to":"926633bd_dd8d12d1","updated":"2025-06-11 21:00:49.000000000","message":"Done","commit_id":"dec9e7a6bd96aae8b712b298c147c66fa1bb0b0b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        type_code \u003d BYTES_TO_TYPE_CODE[itemsize]"},{"line_number":222,"context_line":"        return ["},{"line_number":223,"context_line":"            read_network_order_array(type_code, row)"},{"line_number":224,"context_line":"            for row in iter(lambda: self.read(max_row_len), b\u0027\u0027)"},{"line_number":225,"context_line":"        ]"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"    def close(self):"}],"source_content_type":"text/x-python","patch_set":38,"id":"092e6896_f37feb3a","line":224,"updated":"2025-06-13 21:04:18.000000000","message":"I don\u0027t have any reason to suspect reading the data into a python string is significantly less efficient than `fromfile`\n\nhttps://docs.python.org/3/library/array.html#array.array.fromfile\n\n... but I doubt it\u0027s MORE memory efficient.","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        type_code \u003d BYTES_TO_TYPE_CODE[itemsize]"},{"line_number":222,"context_line":"        return ["},{"line_number":223,"context_line":"            read_network_order_array(type_code, row)"},{"line_number":224,"context_line":"            for row in iter(lambda: self.read(max_row_len), b\u0027\u0027)"},{"line_number":225,"context_line":"        ]"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"    def close(self):"}],"source_content_type":"text/x-python","patch_set":38,"id":"73db8b0e_69fa30fd","line":224,"in_reply_to":"092e6896_f37feb3a","updated":"2025-06-16 23:28:49.000000000","message":"Yeah, `fromfile` [looks just like this](https://github.com/python/cpython/blob/v3.13.5/Modules/arraymodule.c#L1526-L1556) only implemented in C and throwing errors. Maybe that would help with something? It\u0027s gonna be messier, though; something along the lines of\n```\nres \u003d []\nwhile True:  # always a red flag\n   row \u003d array.array(type_code)\n   try:\n       row.fromfile(self, max_row_len)\n   except EOFError:\n       if row:\n           res.append(row)\n       break\n   else:\n       res.append(row)\nreturn res\n```","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":false,"context_lines":[{"line_number":221,"context_line":"        type_code \u003d BYTES_TO_TYPE_CODE[itemsize]"},{"line_number":222,"context_line":"        return ["},{"line_number":223,"context_line":"            read_network_order_array(type_code, row)"},{"line_number":224,"context_line":"            for row in iter(lambda: self.read(max_row_len), b\u0027\u0027)"},{"line_number":225,"context_line":"        ]"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"    def close(self):"}],"source_content_type":"text/x-python","patch_set":38,"id":"f894096a_79c99c5c","line":224,"in_reply_to":"73db8b0e_69fa30fd","updated":"2025-06-18 22:21:44.000000000","message":"yeah that\u0027s exactly what I ended up with, but from your link it looks like the C code is still allocating a PyObject bytes buffer from the read so ... yeah, probably six-one-way-half-a-dozen-the-other.","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b13669c8a31b75ab11c0e49ef46834f140cff4e9","unresolved":true,"context_lines":[{"line_number":364,"context_line":"        entry \u003d self.index[section]"},{"line_number":365,"context_line":"        self.seek(entry.compressed_start)"},{"line_number":366,"context_line":"        size_len \u003d struct.calcsize(V2_SIZE_FORMAT)"},{"line_number":367,"context_line":"        prefix \u003d self.read(size_len)"},{"line_number":368,"context_line":"        blob_length, \u003d struct.unpack(V2_SIZE_FORMAT, prefix)"},{"line_number":369,"context_line":"        if entry.compressed_end is not None and \\"},{"line_number":370,"context_line":"                size_len + blob_length !\u003d entry.uncompressed_length:"}],"source_content_type":"text/x-python","patch_set":42,"id":"8fdb63f9_e3211a72","line":367,"updated":"2025-08-01 00:08:57.000000000","message":"In this implementation when you open a section the size is automatically consumed - in contrast to when you create a section (where it\u0027s the caller\u0027s responsibility to write the size).\n\nI think fundamentally, as convenient as it might be to enforce that every section you open start with a size and prevent reading beyond that size the structure of the index offers sections an alternative for size limiting and even blob length encoding.\n\nIf you call open_section followed by read_blob you\u0027re going to be trying to double account for the size field - I think it\u0027s a footgun to try and offer both abstractions on the same class.\n\nMy recommendation is that open section should leave length reading of the size to the caller (just like create_section does for length writing) and since we want to enforce length limiting - it should be calculated from the end index w/o consuming the size_t/length-field from the check-summed section body.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"}],"swift/common/ring/ring.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c9b68e79237f7f7698652e1af6d54c43da7fdde1","unresolved":false,"context_lines":[{"line_number":199,"context_line":"                                 ring_data[\u0027devs\u0027], ring_data[\u0027part_shift\u0027],"},{"line_number":200,"context_line":"                                 ring_data.get(\u0027next_part_power\u0027),"},{"line_number":201,"context_line":"                                 ring_data.get(\u0027version\u0027))"},{"line_number":202,"context_line":"        for attr in (\u0027md5\u0027, \u0027size\u0027, \u0027raw_size\u0027):"},{"line_number":203,"context_line":"            setattr(ring_data, attr, getattr(gz_file, attr))"},{"line_number":204,"context_line":"        return ring_data"},{"line_number":205,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"66d629e1_86c928ed","side":"PARENT","line":202,"range":{"start_line":202,"start_character":22,"end_line":202,"end_character":25},"updated":"2022-03-20 23:33:01.000000000","message":"Good riddence, this was the bane of my existence when getthing PTG edition version fo the patch written. So glad to see it gone, especially as we\u0027re not using it.","commit_id":"cb8b3cdab262af1d223c0536220400b13c1d0a9a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c9b68e79237f7f7698652e1af6d54c43da7fdde1","unresolved":true,"context_lines":[{"line_number":155,"context_line":"        for part2dev_id in ring[\u0027replica2part2dev_id\u0027]:"},{"line_number":156,"context_line":"            if six.PY2:"},{"line_number":157,"context_line":"                # Can\u0027t just use tofile() because a GzipFile apparently"},{"line_number":158,"context_line":"                # doesn\u0027t count as an \u0027open file\u0027"},{"line_number":159,"context_line":"                writer.write(part2dev_id.tostring())"},{"line_number":160,"context_line":"            else:"},{"line_number":161,"context_line":"                part2dev_id.tofile(writer)"}],"source_content_type":"text/x-python","patch_set":1,"id":"f999f0ba_16fef899","line":158,"updated":"2022-03-20 23:33:01.000000000","message":"Now that we\u0027re dealing with our own writer, can we address this comment and can it be treaded as an open file. Or maybe it doesn\u0027t matter as we\u0027re about to deprecate py2 anyway.","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9104a2463c032f42a0c3b7a044fe180661f3df57","unresolved":false,"context_lines":[{"line_number":155,"context_line":"        for part2dev_id in ring[\u0027replica2part2dev_id\u0027]:"},{"line_number":156,"context_line":"            if six.PY2:"},{"line_number":157,"context_line":"                # Can\u0027t just use tofile() because a GzipFile apparently"},{"line_number":158,"context_line":"                # doesn\u0027t count as an \u0027open file\u0027"},{"line_number":159,"context_line":"                writer.write(part2dev_id.tostring())"},{"line_number":160,"context_line":"            else:"},{"line_number":161,"context_line":"                part2dev_id.tofile(writer)"}],"source_content_type":"text/x-python","patch_set":1,"id":"8a8f6aae_f1aca0b1","line":158,"in_reply_to":"d3efa73d_8e814ce0","updated":"2022-05-17 01:39:46.000000000","message":"Ack","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"51e6e7c9da4b3859d48cd5d75611a10dffe57cf5","unresolved":true,"context_lines":[{"line_number":155,"context_line":"        for part2dev_id in ring[\u0027replica2part2dev_id\u0027]:"},{"line_number":156,"context_line":"            if six.PY2:"},{"line_number":157,"context_line":"                # Can\u0027t just use tofile() because a GzipFile apparently"},{"line_number":158,"context_line":"                # doesn\u0027t count as an \u0027open file\u0027"},{"line_number":159,"context_line":"                writer.write(part2dev_id.tostring())"},{"line_number":160,"context_line":"            else:"},{"line_number":161,"context_line":"                part2dev_id.tofile(writer)"}],"source_content_type":"text/x-python","patch_set":1,"id":"d3efa73d_8e814ce0","line":158,"in_reply_to":"f999f0ba_16fef899","updated":"2022-03-23 18:39:54.000000000","message":"If I\u0027m reading this right, I think it\u0027d need to inherit from `file`... might be doable, but it\u0027s also not a real high priority for me...","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"c9b68e79237f7f7698652e1af6d54c43da7fdde1","unresolved":false,"context_lines":[{"line_number":168,"context_line":"        :param mtime: time used to override mtime for gzip, default or None"},{"line_number":169,"context_line":"                      if the caller wants to include time"},{"line_number":170,"context_line":"        \"\"\""},{"line_number":171,"context_line":"        if format_version not in (0, 1):"},{"line_number":172,"context_line":"            raise ValueError(\"format_version must be one of 0, 1\")"},{"line_number":173,"context_line":"        # Override the timestamp so that the same ring data creates"},{"line_number":174,"context_line":"        # the same bytes on disk. This makes a checksum comparison a"}],"source_content_type":"text/x-python","patch_set":1,"id":"bd348cd4_9a9005b5","line":171,"range":{"start_line":171,"start_character":34,"end_line":171,"end_character":38},"updated":"2022-03-20 23:33:01.000000000","message":"NIT: I wonder if we should turn this into a constant, because we know are about to add a new one. OTHO hopefully the new v2 format will last... FOREVER!!","commit_id":"88fc7aeb6046ecf31ea2e445999dcd6b187cae71"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":108,"context_line":"                                    accomodate all dev ids in use"},{"line_number":109,"context_line":"        \"\"\""},{"line_number":110,"context_line":"        if new_dev_id_bytes not in BYTES_TO_TYPE_CODE:"},{"line_number":111,"context_line":"            raise ValueError(\u0027dev_id_bytes must be one of %r\u0027"},{"line_number":112,"context_line":"                             % (tuple(BYTES_TO_TYPE_CODE),))"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"        if new_dev_id_bytes \u003d\u003d self.dev_id_bytes:"}],"source_content_type":"text/x-python","patch_set":3,"id":"260db968_e3c417ed","line":111,"updated":"2022-04-23 05:36:19.000000000","message":"Needs coverage","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"723785b2ab4f1c3edce0166c061a932717aa7a81","unresolved":false,"context_lines":[{"line_number":108,"context_line":"                                    accomodate all dev ids in use"},{"line_number":109,"context_line":"        \"\"\""},{"line_number":110,"context_line":"        if new_dev_id_bytes not in BYTES_TO_TYPE_CODE:"},{"line_number":111,"context_line":"            raise ValueError(\u0027dev_id_bytes must be one of %r\u0027"},{"line_number":112,"context_line":"                             % (tuple(BYTES_TO_TYPE_CODE),))"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"        if new_dev_id_bytes \u003d\u003d self.dev_id_bytes:"}],"source_content_type":"text/x-python","patch_set":3,"id":"7c072fc5_2c597052","line":111,"in_reply_to":"260db968_e3c417ed","updated":"2022-06-29 09:15:15.000000000","message":"Now covered","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":115,"context_line":"            return  # Already done!"},{"line_number":116,"context_line":"        min_dev_id_bytes \u003d calc_dev_id_bytes(self.max_dev_id)"},{"line_number":117,"context_line":"        if min_dev_id_bytes \u003e new_dev_id_bytes:"},{"line_number":118,"context_line":"            raise DevIdBytesTooSmall(\u0027Too many devices for %d-byte \u0027"},{"line_number":119,"context_line":"                                     \u0027device ids\u0027 % new_dev_id_bytes)"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"        self._replica2part2dev_id \u003d ["}],"source_content_type":"text/x-python","patch_set":3,"id":"6b543b93_b80a8c45","line":118,"updated":"2022-04-23 05:36:19.000000000","message":"Needs coverage","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"723785b2ab4f1c3edce0166c061a932717aa7a81","unresolved":false,"context_lines":[{"line_number":115,"context_line":"            return  # Already done!"},{"line_number":116,"context_line":"        min_dev_id_bytes \u003d calc_dev_id_bytes(self.max_dev_id)"},{"line_number":117,"context_line":"        if min_dev_id_bytes \u003e new_dev_id_bytes:"},{"line_number":118,"context_line":"            raise DevIdBytesTooSmall(\u0027Too many devices for %d-byte \u0027"},{"line_number":119,"context_line":"                                     \u0027device ids\u0027 % new_dev_id_bytes)"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"        self._replica2part2dev_id \u003d ["}],"source_content_type":"text/x-python","patch_set":3,"id":"3fdaf83e_66fc9483","line":118,"in_reply_to":"6b543b93_b80a8c45","updated":"2022-06-29 09:15:15.000000000","message":"Now covered","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":130,"context_line":"        Note that this may be larger than what\u0027s actually used in the"},{"line_number":131,"context_line":"        replica2part2dev_id table, but it can\u0027t be lower."},{"line_number":132,"context_line":"        \"\"\""},{"line_number":133,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":134,"context_line":"            if dev is not None:"},{"line_number":135,"context_line":"                break"},{"line_number":136,"context_line":"        return len(self.devs) - 1 - i"}],"source_content_type":"text/x-python","patch_set":3,"id":"868d7700_a4265012","line":133,"updated":"2022-04-23 05:36:19.000000000","message":"What about when devs is empty? Ought to return 0, but complains about an unbound local.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"86f15c7640ac7f8bebe0a3a47d284392cddffb6e","unresolved":false,"context_lines":[{"line_number":130,"context_line":"        Note that this may be larger than what\u0027s actually used in the"},{"line_number":131,"context_line":"        replica2part2dev_id table, but it can\u0027t be lower."},{"line_number":132,"context_line":"        \"\"\""},{"line_number":133,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":134,"context_line":"            if dev is not None:"},{"line_number":135,"context_line":"                break"},{"line_number":136,"context_line":"        return len(self.devs) - 1 - i"}],"source_content_type":"text/x-python","patch_set":3,"id":"6b8e0fdc_13f3548a","line":133,"in_reply_to":"868d7700_a4265012","updated":"2022-04-29 00:14:56.000000000","message":"Done","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":132,"context_line":"        \"\"\""},{"line_number":133,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":134,"context_line":"            if dev is not None:"},{"line_number":135,"context_line":"                break"},{"line_number":136,"context_line":"        return len(self.devs) - 1 - i"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    @classmethod"}],"source_content_type":"text/x-python","patch_set":3,"id":"b377e561_571b37ac","line":135,"updated":"2022-04-23 05:36:19.000000000","message":"I think we always hit this on the first pass in tests...","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":132,"context_line":"        \"\"\""},{"line_number":133,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":134,"context_line":"            if dev is not None:"},{"line_number":135,"context_line":"                break"},{"line_number":136,"context_line":"        return len(self.devs) - 1 - i"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    @classmethod"}],"source_content_type":"text/x-python","patch_set":3,"id":"59031ba0_a6fe8e05","line":135,"in_reply_to":"b377e561_571b37ac","updated":"2025-05-20 19:58:47.000000000","message":"Mooted.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":149,"context_line":"                            consumed the 6 bytes of magic and version."},{"line_number":150,"context_line":"        :param bool metadata_only: If True, only load `devs` and `part_shift`"},{"line_number":151,"context_line":"        :param bool read_version: If True, read the \"magic\" first 6 bytes;"},{"line_number":152,"context_line":"                                  default is False for legacy third-parties"},{"line_number":153,"context_line":"        :returns: A dict containing `devs`, `part_shift`, and"},{"line_number":154,"context_line":"                  `replica2part2dev_id`"},{"line_number":155,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"e389a462_c76f7fc4","line":152,"updated":"2022-04-23 05:36:19.000000000","message":"Ought to test this default. Or say screw it, and always read version.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"48cba41492652b5d76421ff3e931b62cf020ab8e","unresolved":true,"context_lines":[{"line_number":149,"context_line":"                            consumed the 6 bytes of magic and version."},{"line_number":150,"context_line":"        :param bool metadata_only: If True, only load `devs` and `part_shift`"},{"line_number":151,"context_line":"        :param bool read_version: If True, read the \"magic\" first 6 bytes;"},{"line_number":152,"context_line":"                                  default is False for legacy third-parties"},{"line_number":153,"context_line":"        :returns: A dict containing `devs`, `part_shift`, and"},{"line_number":154,"context_line":"                  `replica2part2dev_id`"},{"line_number":155,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"a54114b9_2e0465e5","line":152,"in_reply_to":"7f3ee494_dcbdd7a4","updated":"2022-05-17 06:32:42.000000000","message":"In the testing follow up I\u0027ve ripped out \u0027read_version\u0027 and now just do a tell() to check to see if we\u0027re at the start of the file. Could actaully check to see if we\u0027re 6 bytes in but :shrug:","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":149,"context_line":"                            consumed the 6 bytes of magic and version."},{"line_number":150,"context_line":"        :param bool metadata_only: If True, only load `devs` and `part_shift`"},{"line_number":151,"context_line":"        :param bool read_version: If True, read the \"magic\" first 6 bytes;"},{"line_number":152,"context_line":"                                  default is False for legacy third-parties"},{"line_number":153,"context_line":"        :returns: A dict containing `devs`, `part_shift`, and"},{"line_number":154,"context_line":"                  `replica2part2dev_id`"},{"line_number":155,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"c224a2fa_a5a02aae","line":152,"in_reply_to":"a54114b9_2e0465e5","updated":"2022-06-08 23:25:18.000000000","message":"Seems reasonable, good idea!","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9104a2463c032f42a0c3b7a044fe180661f3df57","unresolved":true,"context_lines":[{"line_number":149,"context_line":"                            consumed the 6 bytes of magic and version."},{"line_number":150,"context_line":"        :param bool metadata_only: If True, only load `devs` and `part_shift`"},{"line_number":151,"context_line":"        :param bool read_version: If True, read the \"magic\" first 6 bytes;"},{"line_number":152,"context_line":"                                  default is False for legacy third-parties"},{"line_number":153,"context_line":"        :returns: A dict containing `devs`, `part_shift`, and"},{"line_number":154,"context_line":"                  `replica2part2dev_id`"},{"line_number":155,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"7f3ee494_dcbdd7a4","line":152,"in_reply_to":"e389a462_c76f7fc4","updated":"2022-05-17 01:39:46.000000000","message":"Yeah the starting of reading a few bytes in was always kinda annoying. Can just rip it out and see who complains ;) Oh I guess it\u0027s a classmethod.\n\nMaybe we can do a reader.tell() to see how far we\u0027re in, and if we\u0027re 6 bytes skip the version? That way we\u0027re legecy third-party safe and we don\u0027t have to grow a new param?","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":156,"context_line":"        if read_version:"},{"line_number":157,"context_line":"            magic \u003d reader.read(6)"},{"line_number":158,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":159,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":162,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"}],"source_content_type":"text/x-python","patch_set":3,"id":"727947b4_91a3fb8f","line":159,"updated":"2022-04-23 05:36:19.000000000","message":"Needs coverage","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"723785b2ab4f1c3edce0166c061a932717aa7a81","unresolved":false,"context_lines":[{"line_number":156,"context_line":"        if read_version:"},{"line_number":157,"context_line":"            magic \u003d reader.read(6)"},{"line_number":158,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":159,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":162,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"}],"source_content_type":"text/x-python","patch_set":3,"id":"6fdaa83c_f9a50011","line":159,"in_reply_to":"727947b4_91a3fb8f","updated":"2022-06-29 09:15:15.000000000","message":"Now covered","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":163,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"        if metadata_only:"},{"line_number":166,"context_line":"            return ring_dict"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"        byteswap \u003d (ring_dict.get(\u0027byteorder\u0027, sys.byteorder) !\u003d sys.byteorder)"},{"line_number":169,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"060bf166_26492558","line":166,"updated":"2022-04-23 05:36:19.000000000","message":"Is this tested?","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"723785b2ab4f1c3edce0166c061a932717aa7a81","unresolved":false,"context_lines":[{"line_number":163,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"        if metadata_only:"},{"line_number":166,"context_line":"            return ring_dict"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"        byteswap \u003d (ring_dict.get(\u0027byteorder\u0027, sys.byteorder) !\u003d sys.byteorder)"},{"line_number":169,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"b42f24e6_8e7386b6","line":166,"in_reply_to":"060bf166_26492558","updated":"2022-06-29 09:15:15.000000000","message":"is now","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":195,"context_line":"        :param bool metadata_only: If True, skip loading"},{"line_number":196,"context_line":"                                   ``replica2part2dev_id``"},{"line_number":197,"context_line":"        :param bool include_devices: If False and ``metadata_only`` is True,"},{"line_number":198,"context_line":"                                     skip loading ``devs``"},{"line_number":199,"context_line":"        :returns: A dict containing ``devs``, ``part_shift``,"},{"line_number":200,"context_line":"                  ``dev_id_bytes``, and ``replica2part2dev_id``"},{"line_number":201,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"54d58fc4_a1a7d027","line":198,"updated":"2022-04-23 05:36:19.000000000","message":"We never test with this False.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"723785b2ab4f1c3edce0166c061a932717aa7a81","unresolved":false,"context_lines":[{"line_number":195,"context_line":"        :param bool metadata_only: If True, skip loading"},{"line_number":196,"context_line":"                                   ``replica2part2dev_id``"},{"line_number":197,"context_line":"        :param bool include_devices: If False and ``metadata_only`` is True,"},{"line_number":198,"context_line":"                                     skip loading ``devs``"},{"line_number":199,"context_line":"        :returns: A dict containing ``devs``, ``part_shift``,"},{"line_number":200,"context_line":"                  ``dev_id_bytes``, and ``replica2part2dev_id``"},{"line_number":201,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"2d98a2cf_a561da1a","line":198,"in_reply_to":"54d58fc4_a1a7d027","updated":"2022-06-29 09:15:15.000000000","message":"Do now","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        with RingReader(filename) as reader:"},{"line_number":235,"context_line":"            if reader.version not in RING_CODECS:"},{"line_number":236,"context_line":"                raise Exception(\u0027Unknown ring format version %d\u0027 %"},{"line_number":237,"context_line":"                                reader.version)"},{"line_number":238,"context_line":"            ring_data \u003d RING_CODECS[reader.version][\u0027deserialize\u0027]("},{"line_number":239,"context_line":"                cls, reader, metadata_only)"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"84765598_c2cc2b8f","line":237,"updated":"2022-04-23 05:36:19.000000000","message":"Needs coverage","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"723785b2ab4f1c3edce0166c061a932717aa7a81","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        with RingReader(filename) as reader:"},{"line_number":235,"context_line":"            if reader.version not in RING_CODECS:"},{"line_number":236,"context_line":"                raise Exception(\u0027Unknown ring format version %d\u0027 %"},{"line_number":237,"context_line":"                                reader.version)"},{"line_number":238,"context_line":"            ring_data \u003d RING_CODECS[reader.version][\u0027deserialize\u0027]("},{"line_number":239,"context_line":"                cls, reader, metadata_only)"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"bc152202_6603d341","line":237,"in_reply_to":"84765598_c2cc2b8f","updated":"2022-06-29 09:15:15.000000000","message":"We are now testing: \u0027Unsupported ring version\u0027 which is triggered by the RingReader on line 234 above.. so doesn\u0027t get here to check the RING_CODECS. might have to make sure we can actaully catch or trigger this one.","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"357c90f3903834ee1797f4742e6ff026de707982","unresolved":true,"context_lines":[{"line_number":234,"context_line":"        with RingReader(filename) as reader:"},{"line_number":235,"context_line":"            if reader.version not in RING_CODECS:"},{"line_number":236,"context_line":"                raise Exception(\u0027Unknown ring format version %d\u0027 %"},{"line_number":237,"context_line":"                                reader.version)"},{"line_number":238,"context_line":"            ring_data \u003d RING_CODECS[reader.version][\u0027deserialize\u0027]("},{"line_number":239,"context_line":"                cls, reader, metadata_only)"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"bc6d1a9a_79feab05","line":237,"in_reply_to":"bc152202_6603d341","updated":"2022-06-30 02:05:07.000000000","message":"So the check on 235 checks to see if the reader.version is in RING_CODECS. but there is a check in the RingReader contructor that has a similar check. So it currently can only be 0, 1 or 2. Consequently it seems impossible for this exception to fire.\nUnless of course we add a version to the RingReader or change the versions supported in the RING_CODECS constant.\n\nIt\u0027s not a bad check to have as scaffolding I guess, as if something does happen or something changes or corrupts the reader version (once it\u0027s loaded). I\u0027ll can force it to fire in a test I guess.\n\nI\u0027ve managed to trigger it with:\n\n  diff --git a/test/unit/common/ring/test_ring.py \n  b/test/unit/common/ring/test_ring.py\n  index a96962ab2..17f5206b8 100644\n  --- a/test/unit/common/ring/test_ring.py\n  +++ b/test/unit/common/ring/test_ring.py\n  @@ -337,6 +337,18 @@ class TestRingData(unittest.TestCase):\n               ring.RingData.load(ring_fname_bad_version)\n           self.assertEqual(\u0027Unsupported ring version: 5\u0027, str(ex.exception))\n   \n  +        orig_load_index \u003d io.RingReader.load_index\n  +\n  +        def mock_load_index(cls):\n  +            cls.version \u003d 5\n  +            orig_load_index(cls)\n  +\n  +        with mock.patch(\u0027swift.common.ring.io.RingReader.load_index\u0027,\n  +                        mock_load_index):\n  +            with self.assertRaises(Exception) as ex:\n  +                ring.RingData.load(ring_fname_1)\n  +        self.assertEqual(\u0027Unknown ring format version 5\u0027, str(ex.exception))\n  +\n           expected_r2p2d \u003d [\n               array.array(\u0027B\u0027, [0, 1, 0, 1]),\n               array.array(\u0027B\u0027, [0, 1, 0, 1])]","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"227d72b0c53ee36a1a6725586bcfeddd13995720","unresolved":false,"context_lines":[{"line_number":234,"context_line":"        with RingReader(filename) as reader:"},{"line_number":235,"context_line":"            if reader.version not in RING_CODECS:"},{"line_number":236,"context_line":"                raise Exception(\u0027Unknown ring format version %d\u0027 %"},{"line_number":237,"context_line":"                                reader.version)"},{"line_number":238,"context_line":"            ring_data \u003d RING_CODECS[reader.version][\u0027deserialize\u0027]("},{"line_number":239,"context_line":"                cls, reader, metadata_only)"},{"line_number":240,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"eb72e493_43bcecd6","line":237,"in_reply_to":"bc6d1a9a_79feab05","updated":"2022-07-20 10:24:42.000000000","message":"Done","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0aa469fa92c2a18a3fcfdc4ae02ab230398e3c82","unresolved":true,"context_lines":[{"line_number":324,"context_line":"        \"\"\""},{"line_number":325,"context_line":"        if format_version not in RING_CODECS:"},{"line_number":326,"context_line":"            raise ValueError(\"format_version must be one of %r\" % tuple("},{"line_number":327,"context_line":"                RING_CODECS.keys()))"},{"line_number":328,"context_line":"        # Override the timestamp so that the same ring data creates"},{"line_number":329,"context_line":"        # the same bytes on disk. This makes a checksum comparison a"},{"line_number":330,"context_line":"        # good way to see if two rings are identical."}],"source_content_type":"text/x-python","patch_set":3,"id":"66d92ba0_626936bb","line":327,"updated":"2022-04-23 05:36:19.000000000","message":"Needs coverage","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":324,"context_line":"        \"\"\""},{"line_number":325,"context_line":"        if format_version not in RING_CODECS:"},{"line_number":326,"context_line":"            raise ValueError(\"format_version must be one of %r\" % tuple("},{"line_number":327,"context_line":"                RING_CODECS.keys()))"},{"line_number":328,"context_line":"        # Override the timestamp so that the same ring data creates"},{"line_number":329,"context_line":"        # the same bytes on disk. This makes a checksum comparison a"},{"line_number":330,"context_line":"        # good way to see if two rings are identical."}],"source_content_type":"text/x-python","patch_set":3,"id":"7ae723a4_f2b68750","line":327,"in_reply_to":"66d92ba0_626936bb","updated":"2022-10-21 20:02:17.000000000","message":"Done","commit_id":"d8d41f770fb6d445539a23ebb78f942c3d49a770"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"86f15c7640ac7f8bebe0a3a47d284392cddffb6e","unresolved":true,"context_lines":[{"line_number":130,"context_line":"        replica2part2dev_id table, but it can\u0027t be lower."},{"line_number":131,"context_line":"        \"\"\""},{"line_number":132,"context_line":"        if not self.devs:"},{"line_number":133,"context_line":"            return 0"},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":136,"context_line":"            if dev is not None:"}],"source_content_type":"text/x-python","patch_set":4,"id":"d24a8b5c_a709c1d4","line":133,"updated":"2022-04-29 00:14:56.000000000","message":"Oh, or -1... hmm...\n\nMaybe better to just raise ValueError?","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":130,"context_line":"        replica2part2dev_id table, but it can\u0027t be lower."},{"line_number":131,"context_line":"        \"\"\""},{"line_number":132,"context_line":"        if not self.devs:"},{"line_number":133,"context_line":"            return 0"},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":136,"context_line":"            if dev is not None:"}],"source_content_type":"text/x-python","patch_set":4,"id":"1385b2ac_08c5c02f","line":133,"in_reply_to":"11fb0626_3a710b9c","updated":"2025-05-20 19:58:47.000000000","message":"Mooted.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9104a2463c032f42a0c3b7a044fe180661f3df57","unresolved":true,"context_lines":[{"line_number":130,"context_line":"        replica2part2dev_id table, but it can\u0027t be lower."},{"line_number":131,"context_line":"        \"\"\""},{"line_number":132,"context_line":"        if not self.devs:"},{"line_number":133,"context_line":"            return 0"},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":136,"context_line":"            if dev is not None:"}],"source_content_type":"text/x-python","patch_set":4,"id":"11fb0626_3a710b9c","line":133,"in_reply_to":"d24a8b5c_a709c1d4","updated":"2022-05-17 01:39:46.000000000","message":"0 is at least the lowest unssigned int, so if there are no devs we only need to use the lowest dev_id size :shrug:","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"48cba41492652b5d76421ff3e931b62cf020ab8e","unresolved":true,"context_lines":[{"line_number":325,"context_line":"                               for the sake of clusters on older versions"},{"line_number":326,"context_line":"        \"\"\""},{"line_number":327,"context_line":"        if format_version not in RING_CODECS:"},{"line_number":328,"context_line":"            raise ValueError(\"format_version must be one of %r\" % tuple("},{"line_number":329,"context_line":"                RING_CODECS.keys()))"},{"line_number":330,"context_line":"        # Override the timestamp so that the same ring data creates"},{"line_number":331,"context_line":"        # the same bytes on disk. This makes a checksum comparison a"},{"line_number":332,"context_line":"        # good way to see if two rings are identical."}],"source_content_type":"text/x-python","patch_set":4,"id":"bc9a35cf_bf1328bd","line":329,"range":{"start_line":328,"start_character":66,"end_line":329,"end_character":36},"updated":"2022-05-17 06:32:42.000000000","message":"This needs to be something like:\n\n  % (tuple(RING_CODECS.keys()),)\n\nLike you did further up in this file. Its fixed in the testing follow up.","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":false,"context_lines":[{"line_number":325,"context_line":"                               for the sake of clusters on older versions"},{"line_number":326,"context_line":"        \"\"\""},{"line_number":327,"context_line":"        if format_version not in RING_CODECS:"},{"line_number":328,"context_line":"            raise ValueError(\"format_version must be one of %r\" % tuple("},{"line_number":329,"context_line":"                RING_CODECS.keys()))"},{"line_number":330,"context_line":"        # Override the timestamp so that the same ring data creates"},{"line_number":331,"context_line":"        # the same bytes on disk. This makes a checksum comparison a"},{"line_number":332,"context_line":"        # good way to see if two rings are identical."}],"source_content_type":"text/x-python","patch_set":4,"id":"e4029d66_00dcf5fd","line":329,"range":{"start_line":328,"start_character":66,"end_line":329,"end_character":36},"in_reply_to":"bc9a35cf_bf1328bd","updated":"2022-06-08 23:25:18.000000000","message":"Done","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"227d72b0c53ee36a1a6725586bcfeddd13995720","unresolved":true,"context_lines":[{"line_number":259,"context_line":"        ring \u003d cls(ring_data[\u0027replica2part2dev_id\u0027],"},{"line_number":260,"context_line":"                   ring_data[\u0027devs\u0027], ring_data[\u0027part_shift\u0027],"},{"line_number":261,"context_line":"                   ring_data.get(\u0027next_part_power\u0027),"},{"line_number":262,"context_line":"                   ring_data.get(\u0027version\u0027))"},{"line_number":263,"context_line":"        # For loading with metadata_only\u003dTrue"},{"line_number":264,"context_line":"        if \u0027replica_count\u0027 in ring_data:"},{"line_number":265,"context_line":"            ring._replica_count \u003d ring_data[\u0027replica_count\u0027]"}],"source_content_type":"text/x-python","patch_set":7,"id":"519f6f87_4bb10d66","line":262,"updated":"2022-07-20 10:24:42.000000000","message":"This is much better. I do wonder if we could somehow:\n\n  ring \u003d cls(**ring_data)\n  \nBut then I guess we\u0027ll need a **kwargs in the ringdata constructor to catch any extra options which is a little more dirty. So being more deliberate here I guess is for the best.","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"acd97a2ec90ff7e1c972b8a0f9def93a0d2b223c","unresolved":false,"context_lines":[{"line_number":259,"context_line":"        ring \u003d cls(ring_data[\u0027replica2part2dev_id\u0027],"},{"line_number":260,"context_line":"                   ring_data[\u0027devs\u0027], ring_data[\u0027part_shift\u0027],"},{"line_number":261,"context_line":"                   ring_data.get(\u0027next_part_power\u0027),"},{"line_number":262,"context_line":"                   ring_data.get(\u0027version\u0027))"},{"line_number":263,"context_line":"        # For loading with metadata_only\u003dTrue"},{"line_number":264,"context_line":"        if \u0027replica_count\u0027 in ring_data:"},{"line_number":265,"context_line":"            ring._replica_count \u003d ring_data[\u0027replica_count\u0027]"}],"source_content_type":"text/x-python","patch_set":7,"id":"c2be587a_35f4fab5","line":262,"in_reply_to":"519f6f87_4bb10d66","updated":"2025-07-28 09:02:29.000000000","message":"Acknowledged","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f442fa852cb4439b1376f4c427b928d6dd5135ca","unresolved":true,"context_lines":[{"line_number":53,"context_line":"        \"deserialize\": lambda cls, reader, metadata_only: cls.deserialize_v2("},{"line_number":54,"context_line":"            reader, metadata_only\u003dmetadata_only),"},{"line_number":55,"context_line":"    },"},{"line_number":56,"context_line":"}"},{"line_number":57,"context_line":"DEFAULT_RING_FORMAT_VERSION \u003d 1"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"9a4f2267_3545799d","line":56,"updated":"2022-07-27 22:25:32.000000000","message":"I like the CODECS map - but I find the lambdas a little difficult to read visually... maybe something a little more verbose (like a singleton?) would be more obvious.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":53,"context_line":"        \"deserialize\": lambda cls, reader, metadata_only: cls.deserialize_v2("},{"line_number":54,"context_line":"            reader, metadata_only\u003dmetadata_only),"},{"line_number":55,"context_line":"    },"},{"line_number":56,"context_line":"}"},{"line_number":57,"context_line":"DEFAULT_RING_FORMAT_VERSION \u003d 1"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"d19093c0_f8dbd716","line":56,"in_reply_to":"9a4f2267_3545799d","updated":"2025-06-13 21:04:18.000000000","message":"FWIW I\u0027m working with\n\n```\nRING_CODECS \u003d {\n    0: LegacyPickleCodec,\n    1: JsonAndArrayCodec,\n    2: IndexedSectionCodec,\n}\n```","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"acd97a2ec90ff7e1c972b8a0f9def93a0d2b223c","unresolved":true,"context_lines":[{"line_number":53,"context_line":"        \"deserialize\": lambda cls, reader, metadata_only: cls.deserialize_v2("},{"line_number":54,"context_line":"            reader, metadata_only\u003dmetadata_only),"},{"line_number":55,"context_line":"    },"},{"line_number":56,"context_line":"}"},{"line_number":57,"context_line":"DEFAULT_RING_FORMAT_VERSION \u003d 1"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"e28aaf84_f1599180","line":56,"in_reply_to":"d19093c0_f8dbd716","updated":"2025-07-28 09:02:29.000000000","message":"Well now pickle isn\u0027t needed anymore ;)","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":false,"context_lines":[{"line_number":53,"context_line":"        \"deserialize\": lambda cls, reader, metadata_only: cls.deserialize_v2("},{"line_number":54,"context_line":"            reader, metadata_only\u003dmetadata_only),"},{"line_number":55,"context_line":"    },"},{"line_number":56,"context_line":"}"},{"line_number":57,"context_line":"DEFAULT_RING_FORMAT_VERSION \u003d 1"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"1c4e972f_11ea6bd9","line":56,"in_reply_to":"e28aaf84_f1599180","updated":"2025-07-30 07:49:51.000000000","message":"Done","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2119793e74683f93952e83057ea56e3fd12c955f","unresolved":true,"context_lines":[{"line_number":38,"context_line":"DEFAULT_RELOAD_TIME \u003d 15"},{"line_number":39,"context_line":"RING_CODECS \u003d {"},{"line_number":40,"context_line":"    0: {"},{"line_number":41,"context_line":"        \"serialize\": lambda ring_data, writer: pickle.dump("},{"line_number":42,"context_line":"            ring_data.to_dict(), writer),"},{"line_number":43,"context_line":"        # NB: Old-style pickled ring can\u0027t respect metadata_only"},{"line_number":44,"context_line":"        \"deserialize\": lambda cls, reader, metadata_only: pickle.load(reader),"}],"source_content_type":"text/x-python","patch_set":12,"id":"b3470c8c_87601745","line":41,"range":{"start_line":41,"start_character":47,"end_line":41,"end_character":58},"updated":"2022-08-31 16:22:38.000000000","message":"We need to specify a protocol version here -- probably 2, judging by object/diskfile.py and common/db.py","commit_id":"00949fa087706962da9315a1ba7b7a313c46ecf0"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":38,"context_line":"DEFAULT_RELOAD_TIME \u003d 15"},{"line_number":39,"context_line":"RING_CODECS \u003d {"},{"line_number":40,"context_line":"    0: {"},{"line_number":41,"context_line":"        \"serialize\": lambda ring_data, writer: pickle.dump("},{"line_number":42,"context_line":"            ring_data.to_dict(), writer),"},{"line_number":43,"context_line":"        # NB: Old-style pickled ring can\u0027t respect metadata_only"},{"line_number":44,"context_line":"        \"deserialize\": lambda cls, reader, metadata_only: pickle.load(reader),"}],"source_content_type":"text/x-python","patch_set":12,"id":"e458867b_7aa5dad8","line":41,"range":{"start_line":41,"start_character":47,"end_line":41,"end_character":58},"in_reply_to":"b3470c8c_87601745","updated":"2022-10-21 20:02:17.000000000","message":"Done","commit_id":"00949fa087706962da9315a1ba7b7a313c46ecf0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    @property"},{"line_number":112,"context_line":"    def md5(self):"},{"line_number":113,"context_line":"        return self._md5.hexdigest()"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"class RingData(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"a8dd1ef7_3fa1b3c5","side":"PARENT","line":113,"updated":"2023-03-29 00:40:00.000000000","message":"maybe as a pre-factor we could \"just\" move this class into ring.io\n\nand try to make a RingWriter that just handles ringv1","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    @property"},{"line_number":112,"context_line":"    def md5(self):"},{"line_number":113,"context_line":"        return self._md5.hexdigest()"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"class RingData(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"dfcb1c48_5b07b2ef","side":"PARENT","line":113,"in_reply_to":"650486ec_d01844c2","updated":"2025-06-13 21:04:18.000000000","message":"I was commenting on the whole red hunk to the existing RingReader interface - moreso than just the md5 change, which I hadn\u0027t really groked but would agree we can drop if it\u0027s not being used.\n\nI imagine (but haven\u0027t proven) that once we settle the interface(s) in ring.io *some* of them will probably seem reasonable \"just\" for v1-rings/master - and if we can move the introduction of a new module to a pre-factor patch it will probably make understanding/reviewing the code that\u0027s specific to just the v2 [de]serialization easier.","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cc2db0a28617560f498d54857747e9591d06abe5","unresolved":true,"context_lines":[{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    @property"},{"line_number":112,"context_line":"    def md5(self):"},{"line_number":113,"context_line":"        return self._md5.hexdigest()"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"class RingData(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bc3158ed_8bab837f","side":"PARENT","line":113,"in_reply_to":"a8dd1ef7_3fa1b3c5","updated":"2025-04-17 15:36:10.000000000","message":"as an exercise of doing so it might make it more obvious this property is going away - so we can decide and express more explicitly if/why that\u0027s a good idea.","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    @property"},{"line_number":112,"context_line":"    def md5(self):"},{"line_number":113,"context_line":"        return self._md5.hexdigest()"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"class RingData(object):"}],"source_content_type":"text/x-python","patch_set":17,"id":"650486ec_d01844c2","side":"PARENT","line":113,"in_reply_to":"bc3158ed_8bab837f","updated":"2025-05-06 20:28:41.000000000","message":"I didn\u0027t particularly try to hide it; from the commit message:\n```\n- Drop MD5 tracking in RingReader. It was brittle at best anyway, and\n  nothing uses it. YAGNI\n```\nAFAICT it provides no value: there\u0027s nothing to cross-check it against and it doesn\u0027t work with `metadata_only\u003dTrue`.","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":132,"context_line":"    @property"},{"line_number":133,"context_line":"    def replica_count(self):"},{"line_number":134,"context_line":"        \"\"\"Number of replicas (full or partial) used in the ring.\"\"\""},{"line_number":135,"context_line":"        return calc_replica_count(self._replica2part2dev_id)"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"    @classmethod"},{"line_number":138,"context_line":"    def deserialize_v1(cls, gz_file, metadata_only\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":17,"id":"eee048e9_7a716f58","side":"PARENT","line":135,"updated":"2023-03-29 00:40:00.000000000","message":"i\u0027d like to understand better when we want/need the explicit _replica_count from a RingData object that doesn\u0027t have a _replica2part2dev_id yet - it seems like that\u0027s a new requirement somewhere?","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":false,"context_lines":[{"line_number":132,"context_line":"    @property"},{"line_number":133,"context_line":"    def replica_count(self):"},{"line_number":134,"context_line":"        \"\"\"Number of replicas (full or partial) used in the ring.\"\"\""},{"line_number":135,"context_line":"        return calc_replica_count(self._replica2part2dev_id)"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"    @classmethod"},{"line_number":138,"context_line":"    def deserialize_v1(cls, gz_file, metadata_only\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":17,"id":"0548bad7_bec6710e","side":"PARENT","line":135,"in_reply_to":"8ade7999_9eb183fd","updated":"2025-03-26 19:12:40.000000000","message":"Acknowledged","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":132,"context_line":"    @property"},{"line_number":133,"context_line":"    def replica_count(self):"},{"line_number":134,"context_line":"        \"\"\"Number of replicas (full or partial) used in the ring.\"\"\""},{"line_number":135,"context_line":"        return calc_replica_count(self._replica2part2dev_id)"},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"    @classmethod"},{"line_number":138,"context_line":"    def deserialize_v1(cls, gz_file, metadata_only\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":17,"id":"8ade7999_9eb183fd","side":"PARENT","line":135,"in_reply_to":"eee048e9_7a716f58","updated":"2023-04-03 23:05:30.000000000","message":"Mainly? It bugs the crap out of me that loading with `metadata_only\u003dTrue` gives me a worthless `replica_count`.","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":78,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":79,"context_line":"        self.next_part_power \u003d next_part_power"},{"line_number":80,"context_line":"        self.version \u003d version"},{"line_number":81,"context_line":"        self.format_version \u003d None"},{"line_number":82,"context_line":"        self.size \u003d self.raw_size \u003d None"},{"line_number":83,"context_line":"        # Next two are used when replica2part2dev is empty"},{"line_number":84,"context_line":"        self._dev_id_bytes \u003d 0"}],"source_content_type":"text/x-python","patch_set":17,"id":"4b26f7b9_6f72c9f6","line":81,"updated":"2023-03-29 00:40:00.000000000","message":"is this state relevant to the RingData per se or just when we\u0027re serializing it?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":78,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":79,"context_line":"        self.next_part_power \u003d next_part_power"},{"line_number":80,"context_line":"        self.version \u003d version"},{"line_number":81,"context_line":"        self.format_version \u003d None"},{"line_number":82,"context_line":"        self.size \u003d self.raw_size \u003d None"},{"line_number":83,"context_line":"        # Next two are used when replica2part2dev is empty"},{"line_number":84,"context_line":"        self._dev_id_bytes \u003d 0"}],"source_content_type":"text/x-python","patch_set":17,"id":"6be378c4_e6ad54ff","line":81,"in_reply_to":"4b26f7b9_6f72c9f6","updated":"2025-05-06 20:28:41.000000000","message":"When I do `RingData.load(\u003cring file\u003e, metadata\u003dTrue)`, I want _all_ the metadata, including whether this was a v1 or v2 ring I just loaded.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":78,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":79,"context_line":"        self.next_part_power \u003d next_part_power"},{"line_number":80,"context_line":"        self.version \u003d version"},{"line_number":81,"context_line":"        self.format_version \u003d None"},{"line_number":82,"context_line":"        self.size \u003d self.raw_size \u003d None"},{"line_number":83,"context_line":"        # Next two are used when replica2part2dev is empty"},{"line_number":84,"context_line":"        self._dev_id_bytes \u003d 0"}],"source_content_type":"text/x-python","patch_set":17,"id":"c236d1ae_4171e52c","line":81,"in_reply_to":"6be378c4_e6ad54ff","updated":"2025-06-13 21:04:18.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":136,"context_line":""},{"line_number":137,"context_line":"        self._replica2part2dev_id \u003d ["},{"line_number":138,"context_line":"            resize_array(part2dev_id, new_dev_id_bytes)"},{"line_number":139,"context_line":"            for part2dev_id in self._replica2part2dev_id]"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"    @property"},{"line_number":142,"context_line":"    def max_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"e6e02391_63017a70","line":139,"updated":"2023-03-29 00:40:00.000000000","message":"so all this dev_id_bytes handling is kinda different from over on the builder (this particular line is pretty similar)\n\n... but it\u0027s surprsing to me they aren\u0027t more alike, even tho they\u0027re different objects it seems like they\u0027d both have the same \"guts\" more or less (esp wrt the replica2part2dev_id array), just hydrated from (unfortunately) different on-disk file formats.\n\nmaybe there\u0027s room for to create a common Replica2Part2DevBase in a prefactor","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":136,"context_line":""},{"line_number":137,"context_line":"        self._replica2part2dev_id \u003d ["},{"line_number":138,"context_line":"            resize_array(part2dev_id, new_dev_id_bytes)"},{"line_number":139,"context_line":"            for part2dev_id in self._replica2part2dev_id]"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"    @property"},{"line_number":142,"context_line":"    def max_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"3c98d027_bf4d1303","line":139,"in_reply_to":"1fd95bbb_374d68a8","updated":"2025-06-13 21:04:18.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":true,"context_lines":[{"line_number":136,"context_line":""},{"line_number":137,"context_line":"        self._replica2part2dev_id \u003d ["},{"line_number":138,"context_line":"            resize_array(part2dev_id, new_dev_id_bytes)"},{"line_number":139,"context_line":"            for part2dev_id in self._replica2part2dev_id]"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"    @property"},{"line_number":142,"context_line":"    def max_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"1fd95bbb_374d68a8","line":139,"in_reply_to":"ac6fe1c4_ad7a9f4c","updated":"2025-03-26 19:12:40.000000000","message":"re-reading 865065: ring: Create ReplicaToPartToDev helper | https://review.opendev.org/c/openstack/swift/+/865065 one thing that jumps out from using a single array is we don\u0027t have to consider different itemsize per element.\n\nif the memoryview for each \"block\" (i.e. replica) was created only once I assume most of the performance concerns go away and it\u0027d might look like a more obvious win.\n\nI don\u0027t know what the py2 issue was but maybe we can re-visit.\n\nI\u0027ve grown confused as to why a RingData object would ever even NEED a set_dev_id_bytes function - this seems like all this code belongs on the RingBuilder class.  RingData should be basically read-only or created from a Builder\u0027s arrays directly for serialization.  If anyone should resize these arrays it should happen when adding/removing devices by the builder class.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":136,"context_line":""},{"line_number":137,"context_line":"        self._replica2part2dev_id \u003d ["},{"line_number":138,"context_line":"            resize_array(part2dev_id, new_dev_id_bytes)"},{"line_number":139,"context_line":"            for part2dev_id in self._replica2part2dev_id]"},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"    @property"},{"line_number":142,"context_line":"    def max_dev_id(self):"}],"source_content_type":"text/x-python","patch_set":17,"id":"ac6fe1c4_ad7a9f4c","line":139,"in_reply_to":"e6e02391_63017a70","updated":"2023-04-03 23:05:30.000000000","message":"\u003e maybe there\u0027s room for to create a common Replica2Part2DevBase in a prefactor\n\nI started to do something like that in https://review.opendev.org/c/openstack/swift/+/865065 but didn\u0027t much care for it -- at least, not until we can drop py2.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":173,"context_line":"        if reader.tell() \u003d\u003d 0:"},{"line_number":174,"context_line":"            magic \u003d reader.read(6)"},{"line_number":175,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":176,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":179,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"}],"source_content_type":"text/x-python","patch_set":17,"id":"1ffc980b_a1af010e","line":176,"updated":"2023-03-29 00:40:00.000000000","message":"Already consumed the what you say?\n\n    :param file reader: An opened file-like object which has already\n                        consumed the 6 bytes of magic and version.\n                        \nI writer has a write_magic, why doesn\u0027t reader have a read_magic (it calls exposes magic through .version)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":173,"context_line":"        if reader.tell() \u003d\u003d 0:"},{"line_number":174,"context_line":"            magic \u003d reader.read(6)"},{"line_number":175,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":176,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":179,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"}],"source_content_type":"text/x-python","patch_set":17,"id":"c22833c7_5d9d547f","line":176,"in_reply_to":"1ffc980b_a1af010e","updated":"2023-04-03 23:05:30.000000000","message":"Still can -- I don\u0027t want to go breaking the API! But it also bugs the snot out of me that serialize_v1 and deserialize_v1 aren\u0027t closer to inverses of each other.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":true,"context_lines":[{"line_number":173,"context_line":"        if reader.tell() \u003d\u003d 0:"},{"line_number":174,"context_line":"            magic \u003d reader.read(6)"},{"line_number":175,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":176,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":179,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"}],"source_content_type":"text/x-python","patch_set":17,"id":"eeea492a_435d700a","line":176,"in_reply_to":"9ad3121c_21c18dab","updated":"2025-08-05 07:23:12.000000000","message":"This was all wildly inaccurate currently:\n\n- this is `deserialize_v1`, there\u0027s no index to load\n- the last thing `RingReader` does in `__init__` is seek back to the start\n\nThe cross-check is still a little funky -- we can\u0027t get here without already knowing this read will give us `b\u0027R1NG\\x00\\x01\u0027` -- but as long as we need to discard those bytes anyway, I\u0027d rather validate than just drop it on the floor.\n\nThanks for the nudge to rip out the `tell`.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":173,"context_line":"        if reader.tell() \u003d\u003d 0:"},{"line_number":174,"context_line":"            magic \u003d reader.read(6)"},{"line_number":175,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":176,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":179,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"}],"source_content_type":"text/x-python","patch_set":17,"id":"9ad3121c_21c18dab","line":176,"in_reply_to":"c22833c7_5d9d547f","updated":"2025-06-13 21:04:18.000000000","message":"\u003e breaking the API\n\nwhich API?  I *hope* the only interface people consume should be `RingData.load` b/c if they\u0027re using `deserialize_v1` directly they\u0027re going to have to re-implement load to get v2 handling correct.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":175,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":176,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":179,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":180,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":181,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"95598a35_fb4b1542","line":178,"updated":"2023-03-29 00:40:00.000000000","message":"so I think under the hood in our ring.io class read_blob must take a size format string that tells it the length, then it reads that length\n\nI think in the ideal ringv2 patch the serialize_v1 method wouldn\u0027t change - any new interfaces from ringv2 you want to re-use in ringv1 would be a prefactor - alternatively you could just leave v1 alone and save \"combining v1/v2 to be more elegant\" as a follow on change (or just hope everyone adopts ringv2 and we can leave the old/working code alone until we remove ringv1).","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":175,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":176,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":179,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":180,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":181,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"806907b9_0f8dc33f","line":178,"in_reply_to":"49dedffe_9fef5da7","updated":"2025-06-13 21:04:18.000000000","message":"\u003e so many conflicting requests to break this up\n\nI think this is common with big changes that are doing a lot of refactoring and introducing new modules/classes\n\nbasically as a reviewer you want to understand the whole and make sure it all makes sense (with a +2K line diff there\u0027s going to be more than one way to organize things) then once everyone is on board with a direction/style/architecture we\u0027ll want to break things up into pieces that are easier to consume for people that weren\u0027t along for the ride (and hopefully towards the end of the line approaching \"obviously correct\" w/ some small amount of context).\n\n\u003e I think in the ideal ringv2 patch the serialize_v1 method wouldn\u0027t change\n\nthis should be a fairly uncontroversial claim I think.  However digging in I *also* think that the serialize_v1 method on the RingData class needs to change (move?) in order for the new ring.io module structure to make the most sense.  So I think (hope?) there will be an obvious \"just write ringv1 rings same as always but with slightly easier to understand/maintain code\" change that we can merge before adding a change that says: \"take advantage of all that sweet refactoring to plug-in a new ringv2 format - super obvious, no new modules, hopefully just one new class - so easy!\"","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":175,"context_line":"            if magic !\u003d b\u0027R1NG\\x00\\x01\u0027:"},{"line_number":176,"context_line":"                raise ValueError(\u0027unexpected magic: %r\u0027 % magic)"},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":179,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":180,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":181,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"49dedffe_9fef5da7","line":178,"in_reply_to":"95598a35_fb4b1542","updated":"2023-04-03 23:05:30.000000000","message":"I feel like I\u0027ve gotten *so many* conflicting requests to break this up into more changes or squash it down to a single change...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":186,"context_line":""},{"line_number":187,"context_line":"        partition_count \u003d 1 \u003c\u003c (32 - ring_dict[\u0027part_shift\u0027])"},{"line_number":188,"context_line":"        for x in range(ring_dict[\u0027replica_count\u0027]):"},{"line_number":189,"context_line":"            part2dev \u003d array.array(\u0027H\u0027, reader.read(2 * partition_count))"},{"line_number":190,"context_line":"            if byteswap:"},{"line_number":191,"context_line":"                part2dev.byteswap()"},{"line_number":192,"context_line":"            ring_dict[\u0027replica2part2dev_id\u0027].append(part2dev)"}],"source_content_type":"text/x-python","patch_set":17,"id":"d592c976_20a0b685","line":189,"updated":"2023-03-29 00:40:00.000000000","message":"is this H related to the dev_id_bytes you added above?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":186,"context_line":""},{"line_number":187,"context_line":"        partition_count \u003d 1 \u003c\u003c (32 - ring_dict[\u0027part_shift\u0027])"},{"line_number":188,"context_line":"        for x in range(ring_dict[\u0027replica_count\u0027]):"},{"line_number":189,"context_line":"            part2dev \u003d array.array(\u0027H\u0027, reader.read(2 * partition_count))"},{"line_number":190,"context_line":"            if byteswap:"},{"line_number":191,"context_line":"                part2dev.byteswap()"},{"line_number":192,"context_line":"            ring_dict[\u0027replica2part2dev_id\u0027].append(part2dev)"}],"source_content_type":"text/x-python","patch_set":17,"id":"e6c30850_1a726b90","line":189,"in_reply_to":"d592c976_20a0b685","updated":"2023-04-03 23:05:30.000000000","message":"Of course -- as is the `2 *`\n\nDo we want more changes to `deserialize_v1` or do we want to not touch it? I\u0027m confused.\n\nI *do* know that I want to explicitly write down `\"dev_id_bytes\": 2` even for v1 rings, though -- we have an established history of adding new fields to the metadata dict and \"explicit is better than implicit\".","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":186,"context_line":""},{"line_number":187,"context_line":"        partition_count \u003d 1 \u003c\u003c (32 - ring_dict[\u0027part_shift\u0027])"},{"line_number":188,"context_line":"        for x in range(ring_dict[\u0027replica_count\u0027]):"},{"line_number":189,"context_line":"            part2dev \u003d array.array(\u0027H\u0027, reader.read(2 * partition_count))"},{"line_number":190,"context_line":"            if byteswap:"},{"line_number":191,"context_line":"                part2dev.byteswap()"},{"line_number":192,"context_line":"            ring_dict[\u0027replica2part2dev_id\u0027].append(part2dev)"}],"source_content_type":"text/x-python","patch_set":17,"id":"b7ed149e_49f16f44","line":189,"in_reply_to":"e6c30850_1a726b90","updated":"2025-06-13 21:04:18.000000000","message":"\u003e I want to explicitly write down \"dev_id_bytes\": 2 ... explicit is better than implicit\n\nfair enough, I think I was just noticing the relationship that we make explicit in `BYTES_TO_TYPE_CODE` was made here again (implicitly?)\n\nIt might make a little strange in a ring.io prefactor to have a BYTES_TO_TYPE_CODE that only has the one value/key - but I think if the commit message calls it out as a prefactor preparing for wide devices it won\u0027t seem any worse than a bunch of magic 2\u0027s and H\u0027s laying around.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":222,"context_line":"        ring_dict[\u0027devs\u0027] \u003d []"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        if not metadata_only or include_devices:"},{"line_number":225,"context_line":"            ring_dict[\u0027devs\u0027] \u003d json.loads("},{"line_number":226,"context_line":"                reader.read_section(\u0027swift/ring/devices\u0027))"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"6aac92c4_f245670a","line":224,"updated":"2023-03-29 00:40:00.000000000","message":"this conditional has non-obvious binding and it trips me up everytime, so metadata_only DOES include_devices?  you can\u0027t include_devices\u003dFalse with deserialize_v1, so this is a new interface??? who uses it?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":222,"context_line":"        ring_dict[\u0027devs\u0027] \u003d []"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        if not metadata_only or include_devices:"},{"line_number":225,"context_line":"            ring_dict[\u0027devs\u0027] \u003d json.loads("},{"line_number":226,"context_line":"                reader.read_section(\u0027swift/ring/devices\u0027))"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"8ece8079_d5979f7e","line":224,"in_reply_to":"084c7cb9_3fabbadb","updated":"2025-08-05 07:23:12.000000000","message":"\u003e Regardless I think there\u0027s some kind of double-negative that\u0027s annoying me about \u0027metadata_only\u003dFalse, include_devices\u003dFalse` - I think that gets you metadata, devices and assignments, no?\n\nYeah, I think it was mainly because of some combination of\n\n* `metadata_only` is indicating intent w.r.t. (v1) metadata, which necessarily includes devices\n* I have a hard time seeing why anyone would want the assignments with no context for what those dev ids mean.\n\nThat said, if you want four ways of loading a ring instead of three, I can do that. Makes it more obvious that `metatdata_only\u003dFalse` should have been something like `include_assignments\u003dTrue`, but hindsight\u0027s 20/20.\n\n\u003e FWIW I think SRE needs an interface to read locally assigned devices w/o metadata for their telemetry stuff\n\nMeanwhile I can readily imagine applications for reading metadata without devices: Matt\u0027s ELECT idea, yes, but also a self-distributing ring system where ring changes on one node disseminate to the rest almost like a call tree -- just gotta check in the the rack or cluster leader to see whether our versions match.\n\nMaybe the *real* solution should be [a ring service](https://review.opendev.org/c/openstack/swift/+/670674) which *always* reads the whole thing, then exposes the pieces via an API.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":222,"context_line":"        ring_dict[\u0027devs\u0027] \u003d []"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        if not metadata_only or include_devices:"},{"line_number":225,"context_line":"            ring_dict[\u0027devs\u0027] \u003d json.loads("},{"line_number":226,"context_line":"                reader.read_section(\u0027swift/ring/devices\u0027))"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"c3a4dcf7_37148d3c","line":224,"in_reply_to":"1e9137f1_2f6f8c5d","updated":"2025-06-18 22:21:44.000000000","message":"\u003e Nope, we just go ahead and return the data you asked to not get (just like how loading a v0 ring with metadata_only\u003dTrue still gets you assignments).\n\nthat\u0027s lame, we don\u0027t need to hold arrays we don\u0027t need in memory forever just because we can\u0027t NOT read it.  Good I guess hopefully no one has format0 rings.\n\nBut maybe the question I\u0027m asking is \"if we add an include_devices kwarg to serialize_v2 should it be exposed via RingData.load and if so what should happen when you have a v1 format ring\"\n\nI think if we add it to serialize_v2 we *should* expose it through `RingData.load` (but don\u0027t?) and I think we *could* ignore it on v1 format rings (you cite prior art on format0 w/ metadata_only) - but it would also seem justifiable if v1 (or load) decided to `del ring_dict[\u0027devs\u0027]` when include_devices\u003dFalse?\n\nRegardless I think there\u0027s some kind of double-negative that\u0027s annoying me about \u0027metadata_only\u003dFalse, include_devices\u003dFalse` - I think that gets you metadata, devices and assignments, no?\n\nI don\u0027t know what any of this means except that adding a new un-used interface because it \"seems weird to break it out but still have no way to prevent its loading\" might have more complexity than we need to tackle right now.  I\u0027d personally have no push back on the v2 format writing metdata and devs into different sections without trying to add any new interface to read metadata w/o devices until we need it.  (FWIW I think SRE needs an interface to read locally assigned devices w/o metadata for their telemetry stuff)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":222,"context_line":"        ring_dict[\u0027devs\u0027] \u003d []"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        if not metadata_only or include_devices:"},{"line_number":225,"context_line":"            ring_dict[\u0027devs\u0027] \u003d json.loads("},{"line_number":226,"context_line":"                reader.read_section(\u0027swift/ring/devices\u0027))"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"1e9137f1_2f6f8c5d","line":224,"in_reply_to":"4b962572_ebdc307c","updated":"2025-06-16 23:28:49.000000000","message":"\u003e I feel like we\u0027re going to have to add a bunch of kwargs that will only make sense for ring-v2\n\nWe already have a similar situation with `metadata_only`\n\n\u003e how much data gets thrown away (or made-up/stubbed out?) in ringv1\n\nNope, we just go ahead and return the data you asked to not get (just like how loading a v0 ring with `metadata_only\u003dTrue` still gets you assignments).\n\n\u003e Like, presumably it\u0027ll be a couple of years before servers/deamons can refuse to start when they encounter old rings\n\nWe still haven\u0027t thrown away v0 loading so... yeah, seems like a safe bet.\n\n\u003e \"I need this much info from a ring if you have it\"\n\nWe always have it (unless/until we merge builders and rings!? and you\u0027re looking for builder data); the question is how much extra stuff we need to load to get it.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":222,"context_line":"        ring_dict[\u0027devs\u0027] \u003d []"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        if not metadata_only or include_devices:"},{"line_number":225,"context_line":"            ring_dict[\u0027devs\u0027] \u003d json.loads("},{"line_number":226,"context_line":"                reader.read_section(\u0027swift/ring/devices\u0027))"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"9e727cd0_ad9b29ec","line":224,"in_reply_to":"6aac92c4_f245670a","updated":"2025-05-06 20:28:41.000000000","message":"So part of why we broke out `devs` as its own section was that it\u0027s _orders of magnitude_ larger than all the other metadata -- seems weird to break it out but still have no way to prevent its loading.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":222,"context_line":"        ring_dict[\u0027devs\u0027] \u003d []"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        if not metadata_only or include_devices:"},{"line_number":225,"context_line":"            ring_dict[\u0027devs\u0027] \u003d json.loads("},{"line_number":226,"context_line":"                reader.read_section(\u0027swift/ring/devices\u0027))"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"4b962572_ebdc307c","line":224,"in_reply_to":"9e727cd0_ad9b29ec","updated":"2025-06-13 21:04:18.000000000","message":"\u003e seems weird to break it out but still have no way to prevent its loading\n\nthat\u0027s fair; maybe part of what was tripping me up was the lack of plumbing through load?  I feel like we\u0027re going to have to add a bunch of kwargs that will only make sense for ring-v2 but mostly have to with how much data gets thrown away (or made-up/stubbed out?) in ringv1\n\nLike, presumably it\u0027ll be a couple of years before servers/deamons can refuse to start when they encounter old rings; so the interface will have to be something like: \"I need this much info from a ring if you have it\" and the abstractions we provide will do their best with what they\u0027ve got (and their best will be SO AWESOME when they have ringv2!!!)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"acd97a2ec90ff7e1c972b8a0f9def93a0d2b223c","unresolved":true,"context_lines":[{"line_number":221,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":222,"context_line":"        ring_dict[\u0027devs\u0027] \u003d []"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        if not metadata_only or include_devices:"},{"line_number":225,"context_line":"            ring_dict[\u0027devs\u0027] \u003d json.loads("},{"line_number":226,"context_line":"                reader.read_section(\u0027swift/ring/devices\u0027))"},{"line_number":227,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"084c7cb9_3fabbadb","line":224,"in_reply_to":"c3a4dcf7_37148d3c","updated":"2025-07-28 09:02:29.000000000","message":"I think its the metadata_only which is an old api what make it confusing, but that\u0027s what makes it hard to grok. Ie the double negitive and how one takes a positive to exclude and the other a positive to include.. yet v1 rings metadata_only included devs. But like Tim said, devices being part of metadata was just wrong.. they\u0027re too big.. so I like that we seperated them and their not apart of metadata anymore.\n\nSidenote: One example from the past where I\u0027d love to have this is in the improved auto-sharding leader ellection patch (from ages ago) I wanted to have an ELECT verb where container servers could respond with which version of the ring they had to break deadlocks. But I also don\u0027t want to load a full ring into the container server, a metadata_only is the answer esp one that doesn\u0027t have devices. I want ring metadata not assignment details.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":232,"context_line":""},{"line_number":233,"context_line":"        with reader.open_section(\u0027swift/ring/assignments\u0027) as section:"},{"line_number":234,"context_line":"            ring_dict[\u0027replica2part2dev_id\u0027] \u003d section.read_ring_table("},{"line_number":235,"context_line":"                ring_dict[\u0027dev_id_bytes\u0027], partition_count)"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"        return ring_dict"},{"line_number":238,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"0404f711_32b8097a","line":235,"updated":"2023-03-29 00:40:00.000000000","message":"does open_section return different *types* of sections, or could I call open(\u0027swift/ring/metdata\u0027) and then section.read_ring_table?\n\nare we trying to make the ringv2 format support arbitrary sections or arbitrary section *types*","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":232,"context_line":""},{"line_number":233,"context_line":"        with reader.open_section(\u0027swift/ring/assignments\u0027) as section:"},{"line_number":234,"context_line":"            ring_dict[\u0027replica2part2dev_id\u0027] \u003d section.read_ring_table("},{"line_number":235,"context_line":"                ring_dict[\u0027dev_id_bytes\u0027], partition_count)"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"        return ring_dict"},{"line_number":238,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"21a303db_a4c688c9","line":235,"in_reply_to":"0404f711_32b8097a","updated":"2023-04-03 23:05:30.000000000","message":"\u003e could I call open(\u0027swift/ring/metdata\u0027) and then section.read_ring_table?\n\nSure, if you like, I guess. Can\u0027t imagine why you\u0027d want to, though.\n\n\u003e are we trying to make the ringv2 format support arbitrary sections or arbitrary section *types*\n\nI\u0027m not sure what you mean by the question. Maybe it\u0027ll help though if I point out that *this code*, right here and no where else, is what defines the `section name \u003d\u003e deserialization method` map.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":232,"context_line":""},{"line_number":233,"context_line":"        with reader.open_section(\u0027swift/ring/assignments\u0027) as section:"},{"line_number":234,"context_line":"            ring_dict[\u0027replica2part2dev_id\u0027] \u003d section.read_ring_table("},{"line_number":235,"context_line":"                ring_dict[\u0027dev_id_bytes\u0027], partition_count)"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"        return ring_dict"},{"line_number":238,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"60e0e00c_0b950650","line":235,"in_reply_to":"21a303db_a4c688c9","updated":"2025-06-13 21:04:18.000000000","message":"\u003e  this code, ... is what defines the section name \u003d\u003e deserialization method map.\n\nYes!  I think that\u0027s why I\u0027m so confused about the `open_section` and `ring_ring_table` methods living over in the reader - they have to be used together so carefully and they\u0027re only relevant/useful/understandable in the context of this very specific deserialize_v2 method!  There\u0027s this very necessary coupling - that\u0027s spread the implementation specifics across two modules.  I don\u0027t think we\u0027re hiding anything over in ringv2 - open_section/read_type_section go together with deserializev2 (and serializev2!)\n\n... and also \"this code\" does *not* define what the last swift/index section looks like - that\u0027s implemented over in another class and instances of which are also passed to ringv1\u0027s deserialize method.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":248,"context_line":"        with RingReader(filename) as reader:"},{"line_number":249,"context_line":"            if reader.version not in RING_CODECS:"},{"line_number":250,"context_line":"                raise Exception(\u0027Unknown ring format version %d\u0027 %"},{"line_number":251,"context_line":"                                reader.version)"},{"line_number":252,"context_line":"            ring_data \u003d RING_CODECS[reader.version][\u0027deserialize\u0027]("},{"line_number":253,"context_line":"                cls, reader, metadata_only)"},{"line_number":254,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"61970ca1_00cb4d43","line":251,"updated":"2023-03-29 00:40:00.000000000","message":"RE: reader.read_magic()\n\noic, it has a RingReader.version","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":true,"context_lines":[{"line_number":89,"context_line":"        self.devs \u003d devs"},{"line_number":90,"context_line":"        for i, part2dev_id in enumerate(replica2part2dev_id):"},{"line_number":91,"context_line":"            if not isinstance(part2dev_id, array.array):"},{"line_number":92,"context_line":"                replica2part2dev_id[i] \u003d array.array(\u0027I\u0027, part2dev_id)"},{"line_number":93,"context_line":"        self._replica2part2dev_id \u003d replica2part2dev_id"},{"line_number":94,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":95,"context_line":"        self.next_part_power \u003d next_part_power"}],"source_content_type":"text/x-python","patch_set":28,"id":"b01b530a_ebdf85aa","line":92,"updated":"2025-03-26 19:12:40.000000000","message":"this in-line mutation of the passed in arg is a little sneaky and confused me\n\nwe have some tests that say:\n\n```\nr2p2d \u003d [[0, 1, 0, 1], [1, 0, 1, 0]]\nrd \u003d RingData(r2p2d, ...)\nassertEqual(rd._replica2part2dev_id, r2p2d)\n```\n\npreviously I\u0027d always read this as a value equality assertion, which was a reasonable veneer b/c the passed in table was un-modified.\n\nIt turns out `[0, 1, 0, 1] !\u003d array.array(\u0027I\u0027, [0, 1, 0, 1])` but because we\u0027ve modified the passed in list in-place as opposed to a more natural looking:\n\n```\nself._replica2part2dev_id \u003d [\n    resize_array(part2dev_id, size) for part2dev_id in r2p2d]\n```\n\nThe equality check in the test still \"works\", but is NOT making it obvious what\u0027s going on is the constructor is replacing all the elements of the passed in object with new values.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":89,"context_line":"        self.devs \u003d devs"},{"line_number":90,"context_line":"        for i, part2dev_id in enumerate(replica2part2dev_id):"},{"line_number":91,"context_line":"            if not isinstance(part2dev_id, array.array):"},{"line_number":92,"context_line":"                replica2part2dev_id[i] \u003d array.array(\u0027I\u0027, part2dev_id)"},{"line_number":93,"context_line":"        self._replica2part2dev_id \u003d replica2part2dev_id"},{"line_number":94,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":95,"context_line":"        self.next_part_power \u003d next_part_power"}],"source_content_type":"text/x-python","patch_set":28,"id":"f9c43f30_9b3a496b","line":92,"in_reply_to":"b01b530a_ebdf85aa","updated":"2025-05-06 20:28:41.000000000","message":"\u003e It turns out [0, 1, 0, 1] !\u003d array.array(\u0027I\u0027, [0, 1, 0, 1])\n\nAll the more reason we *should* be doing this! I\u0027m about at the point where I\u0027d argue `RingData([[0, 1, 0, 1], [1, 0, 1, 0]], ...)` should raise a `TypeError` -- it\u0027s not what the structure was made for, and it\u0027s not how it\u0027s used in practice.\n\nI should probably be using `H` here, though.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":89,"context_line":"        self.devs \u003d devs"},{"line_number":90,"context_line":"        for i, part2dev_id in enumerate(replica2part2dev_id):"},{"line_number":91,"context_line":"            if not isinstance(part2dev_id, array.array):"},{"line_number":92,"context_line":"                replica2part2dev_id[i] \u003d array.array(\u0027I\u0027, part2dev_id)"},{"line_number":93,"context_line":"        self._replica2part2dev_id \u003d replica2part2dev_id"},{"line_number":94,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":95,"context_line":"        self.next_part_power \u003d next_part_power"}],"source_content_type":"text/x-python","patch_set":28,"id":"88c898dd_b2322d50","line":92,"in_reply_to":"f9c43f30_9b3a496b","updated":"2025-05-20 19:58:47.000000000","message":"Done","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":163,"context_line":"        for i, dev in enumerate(reversed(self.devs)):"},{"line_number":164,"context_line":"            if dev is not None:"},{"line_number":165,"context_line":"                break"},{"line_number":166,"context_line":"        return len(self.devs) - 1 - i"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    @classmethod"},{"line_number":169,"context_line":"    def deserialize_v1(cls, reader, metadata_only\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":28,"id":"8b3ac06c_4cd0182c","line":166,"updated":"2025-06-13 21:04:18.000000000","message":"i\u0027m pretty sure all this getting gone is a good thing.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":138,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":139,"context_line":"        self.next_part_power \u003d next_part_power"},{"line_number":140,"context_line":"        self.version \u003d version"},{"line_number":141,"context_line":"        self.md5 \u003d self.size \u003d self.raw_size \u003d None"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"    @property"},{"line_number":144,"context_line":"    def replica_count(self):"}],"source_content_type":"text/x-python","patch_set":36,"id":"0e50043d_8d3c6681","side":"PARENT","line":141,"updated":"2025-06-13 21:04:18.000000000","message":"am I correct to understand that md5 was not used?\n\n... but size and raw_size were!?","commit_id":"afacfb6cea88d1edd71c7a70a5532e3e12bd8304"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":88,"context_line":"        self.devs \u003d devs"},{"line_number":89,"context_line":"        for i, part2dev_id in enumerate(replica2part2dev_id):"},{"line_number":90,"context_line":"            if not isinstance(part2dev_id, array.array):"},{"line_number":91,"context_line":"                replica2part2dev_id[i] \u003d array.array(\u0027H\u0027, part2dev_id)"},{"line_number":92,"context_line":"        self._replica2part2dev_id \u003d replica2part2dev_id"},{"line_number":93,"context_line":"        self._part_shift \u003d part_shift"},{"line_number":94,"context_line":"        self.next_part_power \u003d next_part_power"}],"source_content_type":"text/x-python","patch_set":36,"id":"d39dded6_55d123dd","line":91,"updated":"2025-06-13 21:04:18.000000000","message":"ok, so we normalize arrays-of-ints to dev_id_bytes \u003d 2","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":96,"context_line":"        self.format_version \u003d None"},{"line_number":97,"context_line":"        self.size \u003d self.raw_size \u003d None"},{"line_number":98,"context_line":"        # Next two are used when replica2part2dev is empty"},{"line_number":99,"context_line":"        self._dev_id_bytes \u003d 0"},{"line_number":100,"context_line":"        self._replica_count \u003d 0"},{"line_number":101,"context_line":"        self._num_devs \u003d sum(1 if dev is not None else 0 for dev in self.devs)"},{"line_number":102,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"4f2d64b2_2764ac1c","line":99,"updated":"2025-06-13 21:04:18.000000000","message":"for consistency with from_dict I wonder if this should default to 2?  Or if from_dict w/o a replica2part2dev table should *also* set it 0?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":147,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":148,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"        if metadata_only:"},{"line_number":151,"context_line":"            return ring_dict"}],"source_content_type":"text/x-python","patch_set":36,"id":"9e8f8712_69121a6f","line":148,"updated":"2025-06-13 21:04:18.000000000","message":"this seems like an odd choice for deserialize to make; I could maybe understand a setdefault - but since to_dict grew a dev_id_bytes and v1 serialize explicitly chooses NOT to copy it into ring_dict it doesn\u0027t seem very symmetrical to add it here\n\nRingDict.from_dict is robust to the key missing and assumes 2 - but only for setting ._dev_id_bytes; which is only used when replica2part2dev_id is missing.\n\nI think either serialize should write dev_id_bytes and deserialize should setdefault or both should ignore it since dev_id_bytes isn\u0027t really a concept in the v1 serialization format.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":147,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":148,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"        if metadata_only:"},{"line_number":151,"context_line":"            return ring_dict"}],"source_content_type":"text/x-python","patch_set":36,"id":"70791887_a5f5a863","line":148,"in_reply_to":"9dfe92aa_7a9e399e","updated":"2025-06-18 22:21:44.000000000","message":"\u003e v1 serialize explicitly chooses NOT to copy it into ring_dict it doesn\u0027t seem very symmetrical to add it here\n\u003e some yahoo actually wrote down something other than 2 \n\u003e which no older Swift code would have done\n\nThe \"yahoo\" might be us on accident at some point.\n\n\u003e I guess you\u0027re advocating for always popping it off and ensuring deserialize_v1 never returns a dict with dev_id_bytes\n\nmeh, I think just continuing to NOT set it when loading a v1 ring will be fine - if the to_dict value that comes from the arrays ever leaks into a v1 format ring_dict hopefully it will still represent the size of the arrays during deserialize.\n\n\u003e if this needs to know about dev_id_bytes anyway, why not just set it appropriately?\n\nit doesn\u0027t tho - RingDict.from_dict is robust to the key missing and assumes 2 - I think that\u0027s the most obvious place for that normalization to happen.  Having it happen here \"as well, cause why not\" stands out in stark contrast to v1 serialize who explicitly ignores it.\n\n... but maybe it\u0027s just a stylistic thing - I don\u0027t like defensive code that tries to \"correctly\" handle conditions that can never happen.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ef4242d4b8cf95ca87cb59725093fd81a19dbfc5","unresolved":true,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"        ring_dict \u003d json.loads(reader.read_blob(\u0027!I\u0027))"},{"line_number":147,"context_line":"        ring_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":148,"context_line":"        ring_dict[\u0027dev_id_bytes\u0027] \u003d 2"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"        if metadata_only:"},{"line_number":151,"context_line":"            return ring_dict"}],"source_content_type":"text/x-python","patch_set":36,"id":"9dfe92aa_7a9e399e","line":148,"in_reply_to":"9e8f8712_69121a6f","updated":"2025-06-17 02:56:47.000000000","message":"You\u0027re reminding me of the heart of the reason for why we needed a v2 format.\n\nIf we did a `setdefault`, and some yahoo actually *wrote down* something other than `2` -- what should we do? Try to use an `array.array(\u0027I\u0027)` (which no older Swift code would have done)? Continue to use `array.array(\u0027H\u0027)` and just *lie* about `dev_id_bytes`? I guess you\u0027re advocating for always popping it off and ensuring `deserialize_v1` *never* returns a dict with `dev_id_bytes` -- but if this **needs** to know about `dev_id_bytes` anyway, why not just set it appropriately?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":176,"context_line":"        the value ``[]``."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        :param file reader: An opened file-like object which has already"},{"line_number":179,"context_line":"                            consumed the 6 bytes of magic and version."},{"line_number":180,"context_line":"        :param bool metadata_only: If True, skip loading"},{"line_number":181,"context_line":"                                   ``replica2part2dev_id``"},{"line_number":182,"context_line":"        :param bool include_devices: If False and ``metadata_only`` is True,"}],"source_content_type":"text/x-python","patch_set":36,"id":"fb18632c_d18f9603","line":179,"updated":"2025-06-13 21:04:18.000000000","message":"\u003e consumed the 6 bytes of magic and version\n\n... and a bunch of bytes at the end for the index\n\nit\u0027s really strange to me that the [de]serialize_v2 methods make no mention of the index","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":176,"context_line":"        the value ``[]``."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        :param file reader: An opened file-like object which has already"},{"line_number":179,"context_line":"                            consumed the 6 bytes of magic and version."},{"line_number":180,"context_line":"        :param bool metadata_only: If True, skip loading"},{"line_number":181,"context_line":"                                   ``replica2part2dev_id``"},{"line_number":182,"context_line":"        :param bool include_devices: If False and ``metadata_only`` is True,"}],"source_content_type":"text/x-python","patch_set":36,"id":"bb8d4875_6d009712","line":179,"in_reply_to":"d406f8c6_d94dc49f","updated":"2025-06-18 22:21:44.000000000","message":"\u003e  I\u0027d gotten used to the idea of RingReader basically operating in two distinct modes\n\nI think i\u0027m still struggling with this.  Maybe if every method on a RingReader was decorated as \"this is for v1 mode\" and \"this is for v2 mode\" or \"used by either\"... \n\n\nAlternatively, where I landed was we move those \"format specific\" methods to somewhere else - closer to an abstraction that represents the different modes and leave a RingGzReader as some kind of basic primitive that\u0027s not responsible for encapsulating various format serialization version specific behaviors and instead only holds knowledge that applies to either format (like ring magic version parsing)","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":176,"context_line":"        the value ``[]``."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"        :param file reader: An opened file-like object which has already"},{"line_number":179,"context_line":"                            consumed the 6 bytes of magic and version."},{"line_number":180,"context_line":"        :param bool metadata_only: If True, skip loading"},{"line_number":181,"context_line":"                                   ``replica2part2dev_id``"},{"line_number":182,"context_line":"        :param bool include_devices: If False and ``metadata_only`` is True,"}],"source_content_type":"text/x-python","patch_set":36,"id":"d406f8c6_d94dc49f","line":179,"in_reply_to":"fb18632c_d18f9603","updated":"2025-06-16 23:28:49.000000000","message":"It\u0027s not \"consumed\" -- if you did a `reader.read()` you\u0027d still see it right there at the end.\n\n\u003e it\u0027s really strange to me that the [de]serialize_v2 methods make no mention of the index\n\nFair point -- I\u0027d gotten used to the idea of `RingReader` basically operating in two distinct modes, and took it as given the `deserialize_v2` would be in the v2 mode. Likely better as\n```\n        :param file reader: An opened file-like object which has already\n                            loaded the index at the end, gone back to the\n                            front, and consumed the 6 bytes of magic and\n                            version.\n```","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":205,"context_line":"        return ring_dict"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"    @classmethod"},{"line_number":208,"context_line":"    def load(cls, filename, metadata_only\u003dFalse):"},{"line_number":209,"context_line":"        \"\"\""},{"line_number":210,"context_line":"        Load ring data from a file."},{"line_number":211,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"0142a819_b5eb029c","line":208,"updated":"2025-06-13 21:04:18.000000000","message":"why not plumb include_devices out to here?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[{"line_number":205,"context_line":"        return ring_dict"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"    @classmethod"},{"line_number":208,"context_line":"    def load(cls, filename, metadata_only\u003dFalse):"},{"line_number":209,"context_line":"        \"\"\""},{"line_number":210,"context_line":"        Load ring data from a file."},{"line_number":211,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"af885cb2_7932bab5","line":208,"in_reply_to":"0142a819_b5eb029c","updated":"2025-08-05 07:23:12.000000000","message":"Done","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":220,"context_line":"            ring_data \u003d RING_CODECS[reader.version][\u0027deserialize\u0027]("},{"line_number":221,"context_line":"                cls, reader, metadata_only)"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"        if hasattr(ring_data, \u0027devs\u0027):"},{"line_number":224,"context_line":"            # pickled RingData; make sure we\u0027ve got region/replication info"},{"line_number":225,"context_line":"            normalize_devices(ring_data.devs)"},{"line_number":226,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":36,"id":"8948f81f_274b4463","line":223,"updated":"2025-06-13 21:04:18.000000000","message":"bit of a wart that legacy-ringv0-pickle-format normalization doesn\u0027t happen until way out here - the 0 codec is the only one that can return something so obtuse.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[{"line_number":220,"context_line":"            ring_data \u003d RING_CODECS[reader.version][\u0027deserialize\u0027]("},{"line_number":221,"context_line":"                cls, reader, metadata_only)"},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"        if hasattr(ring_data, \u0027devs\u0027):"},{"line_number":224,"context_line":"            # pickled RingData; make sure we\u0027ve got region/replication info"},{"line_number":225,"context_line":"            normalize_devices(ring_data.devs)"},{"line_number":226,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":36,"id":"82c20925_3be2de31","line":223,"in_reply_to":"8948f81f_274b4463","updated":"2025-08-05 07:23:12.000000000","message":"Done","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":239,"context_line":"        # For loading with metadata_only\u003dTrue"},{"line_number":240,"context_line":"        if \u0027replica_count\u0027 in ring_data:"},{"line_number":241,"context_line":"            ring._replica_count \u003d ring_data[\u0027replica_count\u0027]"},{"line_number":242,"context_line":"        # dev_id_bytes only included in v2 and above"},{"line_number":243,"context_line":"        ring._dev_id_bytes \u003d ring_data.get(\u0027dev_id_bytes\u0027, 2)"},{"line_number":244,"context_line":"        return ring"},{"line_number":245,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"7f27ad74_e0c56bcb","line":242,"updated":"2025-06-13 21:04:18.000000000","message":"not true, in this patch set, deserialize_v1 is adding the dev_id_bytes key (but FWIW I don\u0027t think it *should*)","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[{"line_number":239,"context_line":"        # For loading with metadata_only\u003dTrue"},{"line_number":240,"context_line":"        if \u0027replica_count\u0027 in ring_data:"},{"line_number":241,"context_line":"            ring._replica_count \u003d ring_data[\u0027replica_count\u0027]"},{"line_number":242,"context_line":"        # dev_id_bytes only included in v2 and above"},{"line_number":243,"context_line":"        ring._dev_id_bytes \u003d ring_data.get(\u0027dev_id_bytes\u0027, 2)"},{"line_number":244,"context_line":"        return ring"},{"line_number":245,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"c521e8dd_8b02079f","line":242,"in_reply_to":"7f27ad74_e0c56bcb","updated":"2025-08-05 07:23:12.000000000","message":"Done","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":240,"context_line":"        if \u0027replica_count\u0027 in ring_data:"},{"line_number":241,"context_line":"            ring._replica_count \u003d ring_data[\u0027replica_count\u0027]"},{"line_number":242,"context_line":"        # dev_id_bytes only included in v2 and above"},{"line_number":243,"context_line":"        ring._dev_id_bytes \u003d ring_data.get(\u0027dev_id_bytes\u0027, 2)"},{"line_number":244,"context_line":"        return ring"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    def serialize_v1(self, writer):"}],"source_content_type":"text/x-python","patch_set":36,"id":"ad358fc3_b748bff2","line":243,"updated":"2025-06-13 21:04:18.000000000","message":"I guess this is useful for v2 ring_dicts that didn\u0027t read their replica2part2dev table - so we\u0027d otherwise not know if it was serialized with wide-device-ids","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":false,"context_lines":[{"line_number":240,"context_line":"        if \u0027replica_count\u0027 in ring_data:"},{"line_number":241,"context_line":"            ring._replica_count \u003d ring_data[\u0027replica_count\u0027]"},{"line_number":242,"context_line":"        # dev_id_bytes only included in v2 and above"},{"line_number":243,"context_line":"        ring._dev_id_bytes \u003d ring_data.get(\u0027dev_id_bytes\u0027, 2)"},{"line_number":244,"context_line":"        return ring"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    def serialize_v1(self, writer):"}],"source_content_type":"text/x-python","patch_set":36,"id":"1e84d32f_7e939889","line":243,"in_reply_to":"2bbf1c85_05e2cf42","updated":"2025-06-18 22:21:44.000000000","message":"Acknowledged","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ef4242d4b8cf95ca87cb59725093fd81a19dbfc5","unresolved":true,"context_lines":[{"line_number":240,"context_line":"        if \u0027replica_count\u0027 in ring_data:"},{"line_number":241,"context_line":"            ring._replica_count \u003d ring_data[\u0027replica_count\u0027]"},{"line_number":242,"context_line":"        # dev_id_bytes only included in v2 and above"},{"line_number":243,"context_line":"        ring._dev_id_bytes \u003d ring_data.get(\u0027dev_id_bytes\u0027, 2)"},{"line_number":244,"context_line":"        return ring"},{"line_number":245,"context_line":""},{"line_number":246,"context_line":"    def serialize_v1(self, writer):"}],"source_content_type":"text/x-python","patch_set":36,"id":"2bbf1c85_05e2cf42","line":243,"in_reply_to":"ad358fc3_b748bff2","updated":"2025-06-17 02:56:47.000000000","message":"Yes, a persistent pet-peeve of mine is how *bad* `metadata_only` loading is. If I can add some reasonably useful information about the ring, I want to do that -- *especially* if we have to write it down anyway!\n\nNote that *we use this already*, for `swift-ring-builder \u003cring\u003e version`","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"}],"swift/common/ring/utils.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"9104a2463c032f42a0c3b7a044fe180661f3df57","unresolved":true,"context_lines":[{"line_number":33,"context_line":"    # on many platforms we care about. Use \u0027I\u0027 for now; hold off on writing"},{"line_number":34,"context_line":"    # custom array (de)serialization methods until someone actually complains."},{"line_number":35,"context_line":"    4: \u0027I\u0027,"},{"line_number":36,"context_line":"    # This just seems excessive; besides, array.array() only takes it on py33+"},{"line_number":37,"context_line":"    # 8: \u0027Q\u0027,"},{"line_number":38,"context_line":"}"},{"line_number":39,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"dfbc8a1b_576f43c2","line":36,"range":{"start_line":36,"start_character":73,"end_line":36,"end_character":78},"updated":"2022-05-17 01:39:46.000000000","message":"Now that are py3 only, which is a current minumum?","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"357c90f3903834ee1797f4742e6ff026de707982","unresolved":false,"context_lines":[{"line_number":33,"context_line":"    # on many platforms we care about. Use \u0027I\u0027 for now; hold off on writing"},{"line_number":34,"context_line":"    # custom array (de)serialization methods until someone actually complains."},{"line_number":35,"context_line":"    4: \u0027I\u0027,"},{"line_number":36,"context_line":"    # This just seems excessive; besides, array.array() only takes it on py33+"},{"line_number":37,"context_line":"    # 8: \u0027Q\u0027,"},{"line_number":38,"context_line":"}"},{"line_number":39,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"b22c37b0_608a1582","line":36,"range":{"start_line":36,"start_character":73,"end_line":36,"end_character":78},"in_reply_to":"77bd3377_3bf20f85","updated":"2022-06-30 02:05:07.000000000","message":"Ack","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"46fe482999fa9934eccdf0ebe507ad0bf1bb125b","unresolved":true,"context_lines":[{"line_number":33,"context_line":"    # on many platforms we care about. Use \u0027I\u0027 for now; hold off on writing"},{"line_number":34,"context_line":"    # custom array (de)serialization methods until someone actually complains."},{"line_number":35,"context_line":"    4: \u0027I\u0027,"},{"line_number":36,"context_line":"    # This just seems excessive; besides, array.array() only takes it on py33+"},{"line_number":37,"context_line":"    # 8: \u0027Q\u0027,"},{"line_number":38,"context_line":"}"},{"line_number":39,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"77bd3377_3bf20f85","line":36,"range":{"start_line":36,"start_character":73,"end_line":36,"end_character":78},"in_reply_to":"dfbc8a1b_576f43c2","updated":"2022-06-08 23:25:18.000000000","message":"It\u0027ll be py36, but we\u0027ve still got py27 a little longer. See https://github.com/openstack/swift/blob/master/setup.cfg#L17-L23","commit_id":"b3da7c22b17209960b62ef15476651a9754a5cda"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":67,"context_line":"    return array.array("},{"line_number":68,"context_line":"        BYTES_TO_TYPE_CODE[new_dev_id_bytes],"},{"line_number":69,"context_line":"        (new_none_dev if dev_id \u003d\u003d old_none_dev else dev_id"},{"line_number":70,"context_line":"         for dev_id in old_arr))"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"@contextlib.contextmanager"}],"source_content_type":"text/x-python","patch_set":17,"id":"70b8bdc8_66b9146c","line":70,"updated":"2023-03-29 00:40:00.000000000","message":"ok, so builder and ringdata both use this function when resizing their replica2part2dev rows\n\nmaybe we could move the abstraction up a layer and just say:\n\n    self._replica2part2dev \u003d resize_r2p2d(new_dev_id_bytes)\n    \nof if we they had a common HasReplica2Part2DevBase() maybe just self.resize_r2p2d()","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":67,"context_line":"    return array.array("},{"line_number":68,"context_line":"        BYTES_TO_TYPE_CODE[new_dev_id_bytes],"},{"line_number":69,"context_line":"        (new_none_dev if dev_id \u003d\u003d old_none_dev else dev_id"},{"line_number":70,"context_line":"         for dev_id in old_arr))"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"@contextlib.contextmanager"}],"source_content_type":"text/x-python","patch_set":17,"id":"b61d6488_a90a497a","line":70,"in_reply_to":"0a698e49_934eec15","updated":"2025-06-16 23:28:49.000000000","message":"*Today*, on master, you can say\n```\n\u003e\u003e\u003e rb\u003dRingBuilder(8, 3, 0)\n\u003e\u003e\u003e rb.add_dev({\u0027ip\u0027: \u0027127.0.0.1\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027d0\u0027, \u0027weight\u0027: 1,\n... \u0027region\u0027: 1, \u0027zone\u0027: 1})\n0\n\u003e\u003e\u003e rb.add_dev({\u0027ip\u0027: \u0027127.0.0.1\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027d1\u0027, \u0027weight\u0027: 1,\n... \u0027region\u0027: 1, \u0027zone\u0027: 1})\n1\n\u003e\u003e\u003e rb.add_dev({\u0027ip\u0027: \u0027127.0.0.1\u0027, \u0027port\u0027: 6200, \u0027device\u0027: \u0027d2\u0027, \u0027weight\u0027: 1,\n... \u0027region\u0027: 1, \u0027zone\u0027: 1})\n2\n\u003e\u003e\u003e rb.rebalance()\n(768, 0.0, 0)\n\u003e\u003e\u003e import array\n\u003e\u003e\u003e rb._replica2part2dev[1] \u003d array.array(\u0027I\u0027, rb._replica2part2dev[1])\n\u003e\u003e\u003e rb.save(...)\n```\n...and unsurprisingly, you\u0027ll be in for a bad time. I\u0027m just stating explicitly an assumption we\u0027re already making. Would it be better phrased like\n```\n# All the rows identically should be sized identically\n```\n?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e595aaf1343b73d033b89cae2831cb63558fb15b","unresolved":true,"context_lines":[{"line_number":67,"context_line":"    return array.array("},{"line_number":68,"context_line":"        BYTES_TO_TYPE_CODE[new_dev_id_bytes],"},{"line_number":69,"context_line":"        (new_none_dev if dev_id \u003d\u003d old_none_dev else dev_id"},{"line_number":70,"context_line":"         for dev_id in old_arr))"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"@contextlib.contextmanager"}],"source_content_type":"text/x-python","patch_set":17,"id":"f7428b66_c6bf381d","line":70,"in_reply_to":"70b8bdc8_66b9146c","updated":"2025-03-08 18:27:17.000000000","message":"Or we revive https://review.opendev.org/c/openstack/swift/+/865065 and have this operate on the one and only `ReplicaToPartToDev.array`","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":67,"context_line":"    return array.array("},{"line_number":68,"context_line":"        BYTES_TO_TYPE_CODE[new_dev_id_bytes],"},{"line_number":69,"context_line":"        (new_none_dev if dev_id \u003d\u003d old_none_dev else dev_id"},{"line_number":70,"context_line":"         for dev_id in old_arr))"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"@contextlib.contextmanager"}],"source_content_type":"text/x-python","patch_set":17,"id":"6046e627_be941997","line":70,"in_reply_to":"b61d6488_a90a497a","updated":"2025-06-18 22:21:44.000000000","message":"\u003e have this operate on the one and only ReplicaToPartToDev.array\n\nI think that\u0027s closer to the ideal than fixing a comment; I think if the abstraction took the whole \"list of arrays\" instead of individual rows it would be more palatable.\n\nSo far I haven\u0027t written anything I liked better for this; probably because I was mainly working in serialization instead of the builders.  I\u0027m looking at the builder/resize stuff now - if I come up with anything I\u0027ll let you know.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":67,"context_line":"    return array.array("},{"line_number":68,"context_line":"        BYTES_TO_TYPE_CODE[new_dev_id_bytes],"},{"line_number":69,"context_line":"        (new_none_dev if dev_id \u003d\u003d old_none_dev else dev_id"},{"line_number":70,"context_line":"         for dev_id in old_arr))"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"@contextlib.contextmanager"}],"source_content_type":"text/x-python","patch_set":17,"id":"0a698e49_934eec15","line":70,"in_reply_to":"f7428b66_c6bf381d","updated":"2025-06-13 21:04:18.000000000","message":"this still sounds pretty attractive; i\u0027m not sure how to prioritize - it seems like we could in theory do it as a follow-up b/c it\u0027s more about how we represent the array in memory after we read it than how we serialize it when we write it.\n\nBut the introduction of:\n\n```\ndef dev_id_bytes:\n    first_row \u003d self.replica2part2dev[0]\n    # all the other rows are probably the same size, right!?\n    return sizeof(first_row[0])\n```\n\n... annoys me","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"@contextlib.contextmanager"},{"line_number":74,"context_line":"def network_order_array(arr):"},{"line_number":75,"context_line":"    if sys.byteorder \u003d\u003d \u0027little\u0027:"},{"line_number":76,"context_line":"        # Switch to network-order for serialization"},{"line_number":77,"context_line":"        arr.byteswap()"}],"source_content_type":"text/x-python","patch_set":34,"id":"ce00cb4f_1fbe2b63","line":74,"updated":"2025-06-13 21:04:18.000000000","message":"i can add a NameError here and `pytest swift/test/unit/common/ring/test_utils.py` produces no failures.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":74,"context_line":"def network_order_array(arr):"},{"line_number":75,"context_line":"    if sys.byteorder \u003d\u003d \u0027little\u0027:"},{"line_number":76,"context_line":"        # Switch to network-order for serialization"},{"line_number":77,"context_line":"        arr.byteswap()"},{"line_number":78,"context_line":"    try:"},{"line_number":79,"context_line":"        yield arr"},{"line_number":80,"context_line":"    finally:"}],"source_content_type":"text/x-python","patch_set":34,"id":"7fb50979_87e3aed6","line":77,"updated":"2025-06-13 21:04:18.000000000","message":"so the default behavior on most systems with ringv2 is to byteswap the whole array everytime we serialize?\n\n```\nvagrant@saio:~$ python3\nPython 3.10.12 (main, Feb  4 2025, 14:57:36) [GCC 11.4.0] on linux\nType \"help\", \"copyright\", \"credits\" or \"license\" for more information.\n\u003e\u003e\u003e import sys\n\u003e\u003e\u003e sys.byteorder\n\u0027little\u0027\n```\n\nI get why we\u0027d prefer making the new on-disk format consistent regardless of the arch/build/host that is doing the serialization; and maybe `byteswap` is crazy optimized; and maybe there\u0027s even some RFC that says something like \"serialization of bytes SHOULD be network byte order\" - but... we should probably attempt to measure this difference anyway.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":false,"context_lines":[{"line_number":74,"context_line":"def network_order_array(arr):"},{"line_number":75,"context_line":"    if sys.byteorder \u003d\u003d \u0027little\u0027:"},{"line_number":76,"context_line":"        # Switch to network-order for serialization"},{"line_number":77,"context_line":"        arr.byteswap()"},{"line_number":78,"context_line":"    try:"},{"line_number":79,"context_line":"        yield arr"},{"line_number":80,"context_line":"    finally:"}],"source_content_type":"text/x-python","patch_set":34,"id":"a07aae1d_bc12e481","line":77,"in_reply_to":"4a89f120_51730668","updated":"2025-06-18 22:21:44.000000000","message":"\u003e `array.array(\"I\", os.urandom(2 ** 24))`\n\nHow big of an array is this in terms of part power?  \"I\" is 4 byte dev_ids, so `(2 ** 24) / 4` is 2 ** 22 devices?  I guess that makes sense...\n\nso there *is* some overhead of byteswap:\n\n```\n$ python -m timeit -s \u0027import array, os; bytes\u003dos.urandom(2 ** 24)\u0027 \u0027a\u003darray.array(\"I\", bytes); a.byteswap()\u0027\n50 loops, best of 5: 4.09 msec per loop\n$ python -m timeit -s \u0027import array, os; bytes\u003dos.urandom(2 ** 24)\u0027 \u0027a\u003darray.array(\"I\", bytes)\u0027\n500 loops, best of 5: 907 usec per loop\n```\n\n.. and upgrade concerns for nvidia aside - it probably would have been faster to load v2 rings if we\u0027d picked little as the \"always serialize format\" ... but only by a few ms so it doesn\u0027t matter.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":74,"context_line":"def network_order_array(arr):"},{"line_number":75,"context_line":"    if sys.byteorder \u003d\u003d \u0027little\u0027:"},{"line_number":76,"context_line":"        # Switch to network-order for serialization"},{"line_number":77,"context_line":"        arr.byteswap()"},{"line_number":78,"context_line":"    try:"},{"line_number":79,"context_line":"        yield arr"},{"line_number":80,"context_line":"    finally:"}],"source_content_type":"text/x-python","patch_set":34,"id":"4a89f120_51730668","line":77,"in_reply_to":"7fb50979_87e3aed6","updated":"2025-06-16 23:28:49.000000000","message":"So why\u0027d we use network order for the JSON length in [v1](https://github.com/openstack/swift/commit/f8ce43a2)?\n\n\u003e I get why we\u0027d prefer making the new on-disk format consistent regardless of the arch/build/host that is doing the serialization\n\nThat\u0027s exactly why -- I don\u0027t think we should need to specify ring-builder-host details [like we currently do](https://github.com/openstack/swift/commit/1ec6e2bb).\n\n\u003e maybe there\u0027s even some RFC that says something like \"serialization of bytes SHOULD be network byte order\"\n\nThere\u0027s https://www.rfc-editor.org/rfc/rfc1700 but that\u0027s very much geared towards network protocols:\n\n```\nSimilarly, whenever a multi-octet field represents a numeric quantity\nthe left most bit of the whole field is the most significant bit.  When\na multi-octet quantity is transmitted the most significant octet is\ntransmitted first.\n```\n\n\u003e we should probably attempt to measure this difference anyway\n\n```\n$ python -m timeit -s \u0027import array, os; a\u003darray.array(\"I\", os.urandom(2 ** 24))\u0027 \u0027a.byteswap()\u0027\n100 loops, best of 5: 3.36 msec per loop\n```\nDoesn\u0027t seem terrible (IMHO) especially for something infrequently done.\n\nOTOH, I could be convinced to go back to writing ring-builder-host details into the ring -- it just felt like a wart I\u0027d rather leave behind from ringv1. Or just always write little-endian -- though that will make *our* upgrade path trickier, now that we\u0027re carrying this.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":true,"context_lines":[{"line_number":33,"context_line":"    # on many platforms we care about. Use \u0027I\u0027 for now; hold off on writing"},{"line_number":34,"context_line":"    # custom array (de)serialization methods until someone actually complains."},{"line_number":35,"context_line":"    4: \u0027I\u0027,"},{"line_number":36,"context_line":"    # This just seems excessive; besides, array.array() only takes it on py33+"},{"line_number":37,"context_line":"    # 8: \u0027Q\u0027,"},{"line_number":38,"context_line":"}"},{"line_number":39,"context_line":""}],"source_content_type":"text/x-python","patch_set":42,"id":"aa1c80fa_b344c3bf","line":36,"range":{"start_line":36,"start_character":73,"end_line":36,"end_character":78},"updated":"2025-07-30 07:49:51.000000000","message":"Now that swift supports 37+ this means we can probablly support Q in arrays now. Again this is just a NIT and definitely can be a follow up.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[{"line_number":33,"context_line":"    # on many platforms we care about. Use \u0027I\u0027 for now; hold off on writing"},{"line_number":34,"context_line":"    # custom array (de)serialization methods until someone actually complains."},{"line_number":35,"context_line":"    4: \u0027I\u0027,"},{"line_number":36,"context_line":"    # This just seems excessive; besides, array.array() only takes it on py33+"},{"line_number":37,"context_line":"    # 8: \u0027Q\u0027,"},{"line_number":38,"context_line":"}"},{"line_number":39,"context_line":""}],"source_content_type":"text/x-python","patch_set":42,"id":"1db4ffbd_b48bde72","line":36,"range":{"start_line":36,"start_character":73,"end_line":36,"end_character":78},"in_reply_to":"aa1c80fa_b344c3bf","updated":"2025-08-05 07:23:12.000000000","message":"Ack -- but I suspect we\u0027ll hit some new challenges well before needing to tackle this one. 😜","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"}],"test/functional/__init__.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":288,"context_line":"                {\u0027id\u0027: 2, \u0027zone\u0027: 2, \u0027device\u0027: \u0027sdc1\u0027, \u0027ip\u0027: \u0027127.0.0.1\u0027,"},{"line_number":289,"context_line":"                 \u0027port\u0027: obj_sockets[2].getsockname()[1]}]"},{"line_number":290,"context_line":"        ring_data \u003d ring.RingData(replica2part2dev_id, devs, 30)"},{"line_number":291,"context_line":"        ring_data.save(ring_file_test, format_version\u003d0)"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"    for dev in ring_data.devs:"},{"line_number":294,"context_line":"        _debug(\u0027Ring file dev: %s\u0027 % dev)"}],"source_content_type":"text/x-python","patch_set":17,"id":"e7a90d9d_567629e5","line":291,"updated":"2023-03-29 00:40:00.000000000","message":"heh, old-skewl!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":288,"context_line":"                {\u0027id\u0027: 2, \u0027zone\u0027: 2, \u0027device\u0027: \u0027sdc1\u0027, \u0027ip\u0027: \u0027127.0.0.1\u0027,"},{"line_number":289,"context_line":"                 \u0027port\u0027: obj_sockets[2].getsockname()[1]}]"},{"line_number":290,"context_line":"        ring_data \u003d ring.RingData(replica2part2dev_id, devs, 30)"},{"line_number":291,"context_line":"        ring_data.save(ring_file_test, format_version\u003d0)"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":"    for dev in ring_data.devs:"},{"line_number":294,"context_line":"        _debug(\u0027Ring file dev: %s\u0027 % dev)"}],"source_content_type":"text/x-python","patch_set":17,"id":"53094fac_518c3d48","line":291,"in_reply_to":"e7a90d9d_567629e5","updated":"2023-04-03 23:05:30.000000000","message":"I was torn about doing it -- on the one hand, it\u0027s nice to get some validation of that `format_version\u003d0` plumbing, but it\u0027s also some unnecessary churn...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"}],"test/unit/cli/test_ringbuilder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"357c90f3903834ee1797f4742e6ff026de707982","unresolved":true,"context_lines":[{"line_number":2563,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\"]"},{"line_number":2564,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2565,"context_line":""},{"line_number":2566,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\", \"--format-version\", \"1\"]"},{"line_number":2567,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2568,"context_line":""},{"line_number":2569,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\", \"--format-version\", \"2\"]"}],"source_content_type":"text/x-python","patch_set":5,"id":"9df181fc_830cd8fe","line":2566,"updated":"2022-06-30 02:05:07.000000000","message":"for completeness we should probably have a `--format-version 0`","commit_id":"00bdb0a26cf52bce1305f86a617662e7bb696655"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"227d72b0c53ee36a1a6725586bcfeddd13995720","unresolved":false,"context_lines":[{"line_number":2563,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\"]"},{"line_number":2564,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2565,"context_line":""},{"line_number":2566,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\", \"--format-version\", \"1\"]"},{"line_number":2567,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2568,"context_line":""},{"line_number":2569,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\", \"--format-version\", \"2\"]"}],"source_content_type":"text/x-python","patch_set":5,"id":"055ace21_ed5324cc","line":2566,"in_reply_to":"9df181fc_830cd8fe","updated":"2022-07-20 10:24:42.000000000","message":"Done","commit_id":"00bdb0a26cf52bce1305f86a617662e7bb696655"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cb9533641497bd3d61a5be75f26f4ebb82be4cca","unresolved":true,"context_lines":[{"line_number":2569,"context_line":"                    version]"},{"line_number":2570,"context_line":"            self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2571,"context_line":"            with RingReader(\"%s.ring.gz\" % self.tmpfile) as reader:"},{"line_number":2572,"context_line":"                self.assertEqual(int(version), reader.version)"},{"line_number":2573,"context_line":""},{"line_number":2574,"context_line":"        exp_results \u003d {\u0027valid_exit_codes\u0027: [EXIT_ERROR]}"},{"line_number":2575,"context_line":"        out, err \u003d self.run_srb(\"write_ring\", \"--format-version\", \"3\","}],"source_content_type":"text/x-python","patch_set":9,"id":"7d9e4ec4_9dba6e74","line":2572,"updated":"2022-07-28 18:28:42.000000000","message":"+1 thanks.","commit_id":"4b2ba51a6d71ee7e3627b48dfe923d5c78e62133"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":2569,"context_line":"                    version]"},{"line_number":2570,"context_line":"            self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2571,"context_line":"            with RingReader(\"%s.ring.gz\" % self.tmpfile) as reader:"},{"line_number":2572,"context_line":"                self.assertEqual(int(version), reader.version)"},{"line_number":2573,"context_line":""},{"line_number":2574,"context_line":"        exp_results \u003d {\u0027valid_exit_codes\u0027: [EXIT_ERROR]}"},{"line_number":2575,"context_line":"        out, err \u003d self.run_srb(\"write_ring\", \"--format-version\", \"3\","}],"source_content_type":"text/x-python","patch_set":9,"id":"1048087e_2a5c34da","line":2572,"in_reply_to":"7d9e4ec4_9dba6e74","updated":"2022-10-21 20:02:17.000000000","message":"Ack","commit_id":"4b2ba51a6d71ee7e3627b48dfe923d5c78e62133"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":2408,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2409,"context_line":"        ring \u003d RingBuilder.load(self.tmpfile)"},{"line_number":2410,"context_line":"        self.assertTrue(ring.validate())"},{"line_number":2411,"context_line":"        self.assertIsNone(ring.devs[3])"},{"line_number":2412,"context_line":""},{"line_number":2413,"context_line":"    def test_rebalance_resets_time_remaining(self):"},{"line_number":2414,"context_line":"        self.create_sample_ring()"}],"source_content_type":"text/x-python","patch_set":17,"id":"a133c0cd_e2d44765","side":"PARENT","line":2411,"updated":"2023-03-29 00:40:00.000000000","message":"so this \"always truncate device list\" is a change in ringv1\n\nmight want to pull that out to a prefactor","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":2569,"context_line":"                    version]"},{"line_number":2570,"context_line":"            self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2571,"context_line":"            with RingReader(\"%s.ring.gz\" % self.tmpfile) as reader:"},{"line_number":2572,"context_line":"                self.assertEqual(int(version), reader.version)"},{"line_number":2573,"context_line":""},{"line_number":2574,"context_line":"        exp_results \u003d {\u0027valid_exit_codes\u0027: [EXIT_ERROR]}"},{"line_number":2575,"context_line":"        out, err \u003d self.run_srb(\"write_ring\", \"--format-version\", \"3\","}],"source_content_type":"text/x-python","patch_set":17,"id":"48348dc3_dfaace15","line":2572,"updated":"2023-03-29 00:40:00.000000000","message":"ok, i like that!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":2572,"context_line":"                self.assertEqual(int(version), reader.version)"},{"line_number":2573,"context_line":""},{"line_number":2574,"context_line":"        exp_results \u003d {\u0027valid_exit_codes\u0027: [EXIT_ERROR]}"},{"line_number":2575,"context_line":"        out, err \u003d self.run_srb(\"write_ring\", \"--format-version\", \"3\","},{"line_number":2576,"context_line":"                                exp_results\u003dexp_results)"},{"line_number":2577,"context_line":"        self.assertIn(\u0027invalid choice\u0027, err)"},{"line_number":2578,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"6042c738_82ccde85","line":2575,"updated":"2023-03-29 00:40:00.000000000","message":"maybe someday!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":2572,"context_line":"                self.assertEqual(int(version), reader.version)"},{"line_number":2573,"context_line":""},{"line_number":2574,"context_line":"        exp_results \u003d {\u0027valid_exit_codes\u0027: [EXIT_ERROR]}"},{"line_number":2575,"context_line":"        out, err \u003d self.run_srb(\"write_ring\", \"--format-version\", \"3\","},{"line_number":2576,"context_line":"                                exp_results\u003dexp_results)"},{"line_number":2577,"context_line":"        self.assertIn(\u0027invalid choice\u0027, err)"},{"line_number":2578,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"d740ded5_a54a072b","line":2575,"in_reply_to":"6042c738_82ccde85","updated":"2025-05-20 19:58:47.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":2583,"context_line":"        out, err \u003d self.run_srb(\"write_ring\", exp_results\u003dexp_results)"},{"line_number":2584,"context_line":"        exp_out \u003d \u0027Unable to write empty ring.\\n\u0027"},{"line_number":2585,"context_line":"        self.assertEqual(exp_out, out[-len(exp_out):])"},{"line_number":2586,"context_line":"        self.assertIn(\"Defaulting to --format-version\u003d1\", out)"},{"line_number":2587,"context_line":""},{"line_number":2588,"context_line":"        for version in 0, 1, 2:"},{"line_number":2589,"context_line":"            out, err \u003d self.run_srb(\"write_ring\","}],"source_content_type":"text/x-python","patch_set":17,"id":"601f85c7_b905d8be","line":2586,"updated":"2023-03-29 00:40:00.000000000","message":"nice","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":2583,"context_line":"        out, err \u003d self.run_srb(\"write_ring\", exp_results\u003dexp_results)"},{"line_number":2584,"context_line":"        exp_out \u003d \u0027Unable to write empty ring.\\n\u0027"},{"line_number":2585,"context_line":"        self.assertEqual(exp_out, out[-len(exp_out):])"},{"line_number":2586,"context_line":"        self.assertIn(\"Defaulting to --format-version\u003d1\", out)"},{"line_number":2587,"context_line":""},{"line_number":2588,"context_line":"        for version in 0, 1, 2:"},{"line_number":2589,"context_line":"            out, err \u003d self.run_srb(\"write_ring\","}],"source_content_type":"text/x-python","patch_set":17,"id":"3f01799e_4b8ed7c1","line":2586,"in_reply_to":"601f85c7_b905d8be","updated":"2025-05-20 19:58:47.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":2589,"context_line":"            out, err \u003d self.run_srb(\"write_ring\","},{"line_number":2590,"context_line":"                                    \"--format-version\u003d{}\".format(version),"},{"line_number":2591,"context_line":"                                    exp_results\u003dexp_results)"},{"line_number":2592,"context_line":"            self.assertEqual(exp_out, out)"},{"line_number":2593,"context_line":""},{"line_number":2594,"context_line":"    def test_write_builder(self):"},{"line_number":2595,"context_line":"        # Test builder file already exists"}],"source_content_type":"text/x-python","patch_set":17,"id":"b9595a37_9d8b5ffb","line":2592,"updated":"2023-03-29 00:40:00.000000000","message":"yeah, this looks like a a good pattern, or depending on how many tests we think we can extend in this way - we could try find calls to write_ring and rebalance add a --format-version\u003d%s % self.format_version then use the sublcass BaseTestCase trick","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":2564,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\"]"},{"line_number":2565,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2566,"context_line":""},{"line_number":2567,"context_line":"        for version in \"0\", \"1\", \"2\":"},{"line_number":2568,"context_line":"            argv \u003d [\"\", self.tmpfile, \"write_ring\", \"--format-version\","},{"line_number":2569,"context_line":"                    version]"},{"line_number":2570,"context_line":"            self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"}],"source_content_type":"text/x-python","patch_set":36,"id":"fbcb0c62_7a10a5da","line":2567,"updated":"2025-06-13 21:04:18.000000000","message":"are we sure we want to *add* the ability to write legacy-pickle-format-0 rings?  You can\u0027t do that on master, right?\n\nside-note: the for loop over the implicit tuple is quite terse!","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4b26e6feb579985c28c6f80724cbceab1814d0a4","unresolved":false,"context_lines":[{"line_number":2564,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\"]"},{"line_number":2565,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2566,"context_line":""},{"line_number":2567,"context_line":"        for version in \"0\", \"1\", \"2\":"},{"line_number":2568,"context_line":"            argv \u003d [\"\", self.tmpfile, \"write_ring\", \"--format-version\","},{"line_number":2569,"context_line":"                    version]"},{"line_number":2570,"context_line":"            self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"}],"source_content_type":"text/x-python","patch_set":36,"id":"1d4c5ba2_32aa8b28","line":2567,"in_reply_to":"8acd8569_14ce0057","updated":"2025-07-30 18:38:26.000000000","message":"Done","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bbdc2f10bae974649d85352eefa286b5e56ba677","unresolved":true,"context_lines":[{"line_number":2564,"context_line":"        argv \u003d [\"\", self.tmpfile, \"write_ring\"]"},{"line_number":2565,"context_line":"        self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"},{"line_number":2566,"context_line":""},{"line_number":2567,"context_line":"        for version in \"0\", \"1\", \"2\":"},{"line_number":2568,"context_line":"            argv \u003d [\"\", self.tmpfile, \"write_ring\", \"--format-version\","},{"line_number":2569,"context_line":"                    version]"},{"line_number":2570,"context_line":"            self.assertSystemExit(EXIT_SUCCESS, ringbuilder.main, argv)"}],"source_content_type":"text/x-python","patch_set":36,"id":"8acd8569_14ce0057","line":2567,"in_reply_to":"fbcb0c62_7a10a5da","updated":"2025-06-26 23:27:38.000000000","message":"Alternatively, merge https://review.opendev.org/c/openstack/swift/+/953497 and we can stop thinking about pickled rings ever again.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"}],"test/unit/common/ring/test_builder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":2983,"context_line":""},{"line_number":2984,"context_line":""},{"line_number":2985,"context_line":"class TestPartPowerIncreaseV2(TestPartPowerIncrease):"},{"line_number":2986,"context_line":"    FORMAT_VERSION \u003d 2"},{"line_number":2987,"context_line":""},{"line_number":2988,"context_line":""},{"line_number":2989,"context_line":"class TestGetRequiredOverload(unittest.TestCase):"}],"source_content_type":"text/x-python","patch_set":17,"id":"60833495_a8311a90","line":2986,"updated":"2023-03-29 00:40:00.000000000","message":"yeah, that\u0027s the subclass BaseTestCase trick!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":2983,"context_line":""},{"line_number":2984,"context_line":""},{"line_number":2985,"context_line":"class TestPartPowerIncreaseV2(TestPartPowerIncrease):"},{"line_number":2986,"context_line":"    FORMAT_VERSION \u003d 2"},{"line_number":2987,"context_line":""},{"line_number":2988,"context_line":""},{"line_number":2989,"context_line":"class TestGetRequiredOverload(unittest.TestCase):"}],"source_content_type":"text/x-python","patch_set":17,"id":"083d8427_2916c724","line":2986,"in_reply_to":"60833495_a8311a90","updated":"2025-06-13 21:04:18.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"}],"test/unit/common/ring/test_io.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"227d72b0c53ee36a1a6725586bcfeddd13995720","unresolved":true,"context_lines":[{"line_number":129,"context_line":"                self.assertNotIn(\u0027inner\u0027, writer.index)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"            writer.write(b\u0027can add arbitrary bytes\u0027)"},{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult..."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":135,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":7,"id":"f88b7982_f9552b20","line":132,"updated":"2022-07-20 10:24:42.000000000","message":"Hmm, it wouldn\u0027t be easy to jump here, you\u0027d need to jump to section \u0027baz\u0027 then seek its length to get here. And you wouldn\u0027t know the length, or maybe you would by looking for gabs in the index?\n\nSo we could either just leave it, and we know it\u0027s possible because of this test (which is great!) or we don\u0027t allow writing arbitary bytes if you have or once you have starting writing sections.. but that seems hard to police.\n\nQuestion, how does your follow up basic repair deal with this? Will it just get to arbitary bytes and try and read it as a length field. (I cant remember, I guess I could just go and check).","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"594aafdc29f45c6c6a4a7e35454872c1c1a740d2","unresolved":false,"context_lines":[{"line_number":129,"context_line":"                self.assertNotIn(\u0027inner\u0027, writer.index)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"            writer.write(b\u0027can add arbitrary bytes\u0027)"},{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult..."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":135,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":7,"id":"3c08ae89_1dbf1705","line":132,"in_reply_to":"5d7d3331_ba0c4770","updated":"2022-11-16 01:37:02.000000000","message":"Cool, well repair is in a different patch. I like how this file is commented the way it is.","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"de86ec7685e53f6895a571f5b3d18a80a3f9efc2","unresolved":true,"context_lines":[{"line_number":129,"context_line":"                self.assertNotIn(\u0027inner\u0027, writer.index)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"            writer.write(b\u0027can add arbitrary bytes\u0027)"},{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult..."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":135,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5d7d3331_ba0c4770","line":132,"in_reply_to":"f88b7982_f9552b20","updated":"2022-07-20 17:41:39.000000000","message":"\u003e Hmm, it wouldn\u0027t be easy to jump here, you\u0027d need to jump to section \u0027baz\u0027 then seek its length to get here. And you wouldn\u0027t know the length, or maybe you would by looking for gabs in the index?\n\nYeah, it\u0027s not well-supported (nor recommended). I think there\u0027s enough info in the various index offsets that you could probably piece it together, but it\u0027s definitely messy. I\u0027ll take a stab at getting this back during read.\n\n\u003e Question, how does your follow up basic repair deal with this? Will it just get to arbitary bytes and try and read it as a length field.\n\nYeah, it\u0027ll try to read it as though it\u0027s a section -- which will end badly. Arguably, repair should load up the whole unzipped stream and start walking backwards from the end and try to detect an index -- but that seems hairy. I might take a stab at it anyway...","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"423efd73dd66e6db4a97d67aa5a0b8b15d86326f","unresolved":true,"context_lines":[{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult; see below."},{"line_number":133,"context_line":"            # This *is not* a recommended pattern -- write proper length-value"},{"line_number":134,"context_line":"            # blobs instead (even if you don\u0027t include them as sections in the"},{"line_number":135,"context_line":"            # index)."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":138,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":8,"id":"c65d4a8b_1c1a7c91","line":135,"updated":"2022-07-25 07:34:28.000000000","message":"Yup definitely not recommended, but possible. Love the test based demonstraition on how it would work though. Test based doco :)","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b1ff6f8feb4d3f7ccb7327b083433573c836d7d6","unresolved":true,"context_lines":[{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult; see below."},{"line_number":133,"context_line":"            # This *is not* a recommended pattern -- write proper length-value"},{"line_number":134,"context_line":"            # blobs instead (even if you don\u0027t include them as sections in the"},{"line_number":135,"context_line":"            # index)."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":138,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":8,"id":"d8e84b3b_7a9414ef","line":135,"in_reply_to":"c65d4a8b_1c1a7c91","updated":"2022-07-25 19:25:25.000000000","message":"Yup -- there was a lot of that in this patch, not just this test but all of TestRingExtensibility 😊","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult; see below."},{"line_number":133,"context_line":"            # This *is not* a recommended pattern -- write proper length-value"},{"line_number":134,"context_line":"            # blobs instead (even if you don\u0027t include them as sections in the"},{"line_number":135,"context_line":"            # index)."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":138,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":8,"id":"a26a1e76_8ad428a3","line":135,"in_reply_to":"d8e84b3b_7a9414ef","updated":"2022-10-21 20:02:17.000000000","message":"Ack","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"423efd73dd66e6db4a97d67aa5a0b8b15d86326f","unresolved":true,"context_lines":[{"line_number":168,"context_line":"                # note the gap between baz and quux for the raw bytes"},{"line_number":169,"context_line":"                \u0027quux\u0027: (122933, 163901, \u0027sha256\u0027),"},{"line_number":170,"context_line":"                \u0027swift/index\u0027: (163901, None, None),"},{"line_number":171,"context_line":"            })"},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"            self.assertIn(\u0027foo\u0027, reader)"},{"line_number":174,"context_line":"            self.assertNotIn(\u0027inner\u0027, reader)"}],"source_content_type":"text/x-python","patch_set":8,"id":"faf42994_cde29c1e","line":171,"updated":"2022-07-25 07:34:28.000000000","message":"Awesome demo on how it\u0027ll look and how you can get the abitary bytes back by looking at the size via the index.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":168,"context_line":"                # note the gap between baz and quux for the raw bytes"},{"line_number":169,"context_line":"                \u0027quux\u0027: (122933, 163901, \u0027sha256\u0027),"},{"line_number":170,"context_line":"                \u0027swift/index\u0027: (163901, None, None),"},{"line_number":171,"context_line":"            })"},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"            self.assertIn(\u0027foo\u0027, reader)"},{"line_number":174,"context_line":"            self.assertNotIn(\u0027inner\u0027, reader)"}],"source_content_type":"text/x-python","patch_set":8,"id":"d6074c04_2ad599b8","line":171,"in_reply_to":"faf42994_cde29c1e","updated":"2022-10-21 20:02:17.000000000","message":"Ack","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"423efd73dd66e6db4a97d67aa5a0b8b15d86326f","unresolved":true,"context_lines":[{"line_number":198,"context_line":"            gap_length \u003d reader.index[\u0027quux\u0027][1] - reader.index[\u0027baz\u0027][3]"},{"line_number":199,"context_line":"            self.assertGreater(gap_length, 0)"},{"line_number":200,"context_line":"            self.assertEqual(b\u0027can add arbitrary bytes\u0027,"},{"line_number":201,"context_line":"                             reader.read(gap_length))"},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"    @with_tempdir"},{"line_number":204,"context_line":"    def test_sections_with_corruption(self, tempd):"}],"source_content_type":"text/x-python","patch_set":8,"id":"4a3414c2_d5d4fb0b","line":201,"updated":"2022-07-25 07:34:28.000000000","message":"Yup, love it! nice getting it back out. And showing how. Even though it isn\u0027t recommended.","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":198,"context_line":"            gap_length \u003d reader.index[\u0027quux\u0027][1] - reader.index[\u0027baz\u0027][3]"},{"line_number":199,"context_line":"            self.assertGreater(gap_length, 0)"},{"line_number":200,"context_line":"            self.assertEqual(b\u0027can add arbitrary bytes\u0027,"},{"line_number":201,"context_line":"                             reader.read(gap_length))"},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"    @with_tempdir"},{"line_number":204,"context_line":"    def test_sections_with_corruption(self, tempd):"}],"source_content_type":"text/x-python","patch_set":8,"id":"b490387e_2d46e6c0","line":201,"in_reply_to":"4a3414c2_d5d4fb0b","updated":"2022-10-21 20:02:17.000000000","message":"Ack","commit_id":"63f57a156273b05fa5c15ad813bf602139afc1d6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":36,"context_line":"            for x in range(n))"},{"line_number":37,"context_line":"        self.assertEqual(actual, {pattern: n})"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"    @with_tempdir"},{"line_number":40,"context_line":"    def test_write_failure(self, tempd):"},{"line_number":41,"context_line":"        tempf \u003d os.path.join(tempd, \u0027not-persisted\u0027)"},{"line_number":42,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":17,"id":"3cf3523a_c26d4315","line":39,"updated":"2023-03-29 00:40:00.000000000","message":"it looks like every test method in here uses @with_tempdir, maybe it would make sense to have it in setUp since this is test_io.TestRoundTrip and we should probably kind of expect to hit the filesystem in every test.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":36,"context_line":"            for x in range(n))"},{"line_number":37,"context_line":"        self.assertEqual(actual, {pattern: n})"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"    @with_tempdir"},{"line_number":40,"context_line":"    def test_write_failure(self, tempd):"},{"line_number":41,"context_line":"        tempf \u003d os.path.join(tempd, \u0027not-persisted\u0027)"},{"line_number":42,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":17,"id":"f7a8c626_2c4a81fb","line":39,"in_reply_to":"3cf3523a_c26d4315","updated":"2023-04-03 23:05:30.000000000","message":"\u003e we should probably kind of expect to hit the filesystem in every test.\n\nIDK -- I\u0027d consider some API changes to let us use `BytesIO` instead...","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"11b7880a1275f04ac271690978ed36008dde2021","unresolved":false,"context_lines":[{"line_number":36,"context_line":"            for x in range(n))"},{"line_number":37,"context_line":"        self.assertEqual(actual, {pattern: n})"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"    @with_tempdir"},{"line_number":40,"context_line":"    def test_write_failure(self, tempd):"},{"line_number":41,"context_line":"        tempf \u003d os.path.join(tempd, \u0027not-persisted\u0027)"},{"line_number":42,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":17,"id":"9e61b13c_fafa5c9f","line":39,"in_reply_to":"f7a8c626_2c4a81fb","updated":"2025-01-10 04:45:55.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":41,"context_line":"        tempf \u003d os.path.join(tempd, \u0027not-persisted\u0027)"},{"line_number":42,"context_line":"        try:"},{"line_number":43,"context_line":"            with RingWriter(tempf):"},{"line_number":44,"context_line":"                self.assertEqual(1, len(os.listdir(tempd)))"},{"line_number":45,"context_line":"                raise ValueError"},{"line_number":46,"context_line":"        except ValueError:"},{"line_number":47,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":17,"id":"a308eac9_699a9e97","line":44,"updated":"2023-03-29 00:40:00.000000000","message":"maybe better as \n\n    assertEqual(os.listdir(), [\u0027not-persisted\u0027])","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":true,"context_lines":[{"line_number":41,"context_line":"        tempf \u003d os.path.join(tempd, \u0027not-persisted\u0027)"},{"line_number":42,"context_line":"        try:"},{"line_number":43,"context_line":"            with RingWriter(tempf):"},{"line_number":44,"context_line":"                self.assertEqual(1, len(os.listdir(tempd)))"},{"line_number":45,"context_line":"                raise ValueError"},{"line_number":46,"context_line":"        except ValueError:"},{"line_number":47,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":17,"id":"c0af9e95_d6dad10a","line":44,"in_reply_to":"a308eac9_699a9e97","updated":"2025-01-28 17:30:08.000000000","message":"Won\u0027t pass! `open` uses a `NamedTemporaryFile`, where that `not-persisted` will just be a *prefix*. If everything goes well, we flush, fsync, and rename the tempfile to `not-persisted`, but of course the whole point of this test is that things **don\u0027t** go well. I could add a\n```\nself.assertTrue(os.listdir(tempd)[0].startswith(\u0027not-persisted\u0027))\n```\nif you really want, but I think the key assertions are covered.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":41,"context_line":"        tempf \u003d os.path.join(tempd, \u0027not-persisted\u0027)"},{"line_number":42,"context_line":"        try:"},{"line_number":43,"context_line":"            with RingWriter(tempf):"},{"line_number":44,"context_line":"                self.assertEqual(1, len(os.listdir(tempd)))"},{"line_number":45,"context_line":"                raise ValueError"},{"line_number":46,"context_line":"        except ValueError:"},{"line_number":47,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":17,"id":"394d84fb_07b682a9","line":44,"in_reply_to":"c0af9e95_d6dad10a","updated":"2025-06-13 21:04:18.000000000","message":"\u003e I think the key assertions are covered\n\nyes, the `startswith` assertion would have made it much more obvious what was going on!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":52,"context_line":"        tempf \u003d os.path.join(tempd, \u0027gzip-stream\u0027)"},{"line_number":53,"context_line":"        with RingWriter(tempf) as writer:"},{"line_number":54,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"},{"line_number":55,"context_line":"            writer.write(b\u0027\\xda\\x7a\\xda\\x7a\u0027 * 10240)"},{"line_number":56,"context_line":"            good_pos \u003d writer.tell()"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"            self.assertTrue(writer.flushed)"}],"source_content_type":"text/x-python","patch_set":17,"id":"d35da1d7_b6c24530","line":55,"updated":"2023-03-29 00:40:00.000000000","message":"I can see deadbeef \u003c3, but what is da7ada7a oh maybe datadata","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":52,"context_line":"        tempf \u003d os.path.join(tempd, \u0027gzip-stream\u0027)"},{"line_number":53,"context_line":"        with RingWriter(tempf) as writer:"},{"line_number":54,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"},{"line_number":55,"context_line":"            writer.write(b\u0027\\xda\\x7a\\xda\\x7a\u0027 * 10240)"},{"line_number":56,"context_line":"            good_pos \u003d writer.tell()"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"            self.assertTrue(writer.flushed)"}],"source_content_type":"text/x-python","patch_set":17,"id":"862662e8_7b90e693","line":55,"in_reply_to":"d35da1d7_b6c24530","updated":"2025-05-20 19:58:47.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":53,"context_line":"        with RingWriter(tempf) as writer:"},{"line_number":54,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"},{"line_number":55,"context_line":"            writer.write(b\u0027\\xda\\x7a\\xda\\x7a\u0027 * 10240)"},{"line_number":56,"context_line":"            good_pos \u003d writer.tell()"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"            self.assertTrue(writer.flushed)"},{"line_number":59,"context_line":"            pos \u003d writer.raw_fp.tell()"}],"source_content_type":"text/x-python","patch_set":17,"id":"2f7827d6_bdc734cd","line":56,"updated":"2023-03-29 00:40:00.000000000","message":"if i knew the RingWriter was a zlib stream, this would seem strange - but I remembered you say RingWriter will always flush before tell - and i guess we\u0027re fine with that because Gzip can flush at anytime and continue the stream, so we assume if a caller says \"tell\" they\u0027re asking because they want to seek back later and have valid data.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":53,"context_line":"        with RingWriter(tempf) as writer:"},{"line_number":54,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"},{"line_number":55,"context_line":"            writer.write(b\u0027\\xda\\x7a\\xda\\x7a\u0027 * 10240)"},{"line_number":56,"context_line":"            good_pos \u003d writer.tell()"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"            self.assertTrue(writer.flushed)"},{"line_number":59,"context_line":"            pos \u003d writer.raw_fp.tell()"}],"source_content_type":"text/x-python","patch_set":17,"id":"816aca91_2e365532","line":56,"in_reply_to":"2f7827d6_bdc734cd","updated":"2023-04-03 23:05:30.000000000","message":"\u003e we assume if a caller says \"tell\" they\u0027re asking because they want to seek back later and have valid data.\n\nBingo!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":69,"context_line":"            self.assertEqual(reader.raw_size, 12 * 10240)"},{"line_number":70,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027\\xde\\xad\\xbe\\xef\u0027, 10240)"},{"line_number":71,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027\\xda\\x7a\\xda\\x7a\u0027, 10240)"},{"line_number":72,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027more\u0027, 10240)"},{"line_number":73,"context_line":"            # Can seek backwards"},{"line_number":74,"context_line":"            reader.seek(good_pos)"},{"line_number":75,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027more\u0027, 10240)"}],"source_content_type":"text/x-python","patch_set":17,"id":"b30b5b4e_cd2b3cc8","line":72,"updated":"2023-03-29 00:40:00.000000000","message":"heh, assertRepeasts is kind of cute - i wonder if it fails better than assertEqual(read(4M), \u0027aaaa\u0027 * 1M)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":69,"context_line":"            self.assertEqual(reader.raw_size, 12 * 10240)"},{"line_number":70,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027\\xde\\xad\\xbe\\xef\u0027, 10240)"},{"line_number":71,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027\\xda\\x7a\\xda\\x7a\u0027, 10240)"},{"line_number":72,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027more\u0027, 10240)"},{"line_number":73,"context_line":"            # Can seek backwards"},{"line_number":74,"context_line":"            reader.seek(good_pos)"},{"line_number":75,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027more\u0027, 10240)"}],"source_content_type":"text/x-python","patch_set":17,"id":"9df91303_c8b19c5a","line":72,"in_reply_to":"b30b5b4e_cd2b3cc8","updated":"2023-04-03 23:05:30.000000000","message":"Definitely better (IMO) -- though I\u0027m a little torn about the length check on the front end. If the `read()` produces absolute garbage it\u0027ll still be pretty terrible, though.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":78,"context_line":"            self.assertRepeats(reader.read(40960), b\u0027\\xde\\xad\\xbe\\xef\u0027, 10240)"},{"line_number":79,"context_line":"            # but not arbitrarily"},{"line_number":80,"context_line":"            reader.seek(good_pos - 100)"},{"line_number":81,"context_line":"            with self.assertRaises(zlib.error):"},{"line_number":82,"context_line":"                reader.read(1)"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"    @with_tempdir"}],"source_content_type":"text/x-python","patch_set":17,"id":"b10c0e69_a06a6481","line":81,"updated":"2023-03-29 00:40:00.000000000","message":"oh nice!  this really feels more like a SeekableZlibReader/Writer than a RingReader/Writer","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":91,"context_line":"        }"},{"line_number":92,"context_line":"        tempf \u003d os.path.join(tempd, \u0027ring-v0\u0027)"},{"line_number":93,"context_line":"        with RingWriter(tempf) as writer:"},{"line_number":94,"context_line":"            writer.write(pickle.dumps(data, protocol\u003d2))"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"        with RingReader(tempf) as reader:"},{"line_number":97,"context_line":"            self.assertEqual(reader.version, 0)"}],"source_content_type":"text/x-python","patch_set":17,"id":"5b5ab513_43371eec","line":94,"updated":"2023-03-29 00:40:00.000000000","message":"can you use writer as an fp\n\n    pickle.dump(data, writer)","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":false,"context_lines":[{"line_number":91,"context_line":"        }"},{"line_number":92,"context_line":"        tempf \u003d os.path.join(tempd, \u0027ring-v0\u0027)"},{"line_number":93,"context_line":"        with RingWriter(tempf) as writer:"},{"line_number":94,"context_line":"            writer.write(pickle.dumps(data, protocol\u003d2))"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"        with RingReader(tempf) as reader:"},{"line_number":97,"context_line":"            self.assertEqual(reader.version, 0)"}],"source_content_type":"text/x-python","patch_set":17,"id":"ea708657_d0f3d24e","line":94,"in_reply_to":"5b5ab513_43371eec","updated":"2025-05-06 20:28:41.000000000","message":"Done in PS 19.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":95,"context_line":""},{"line_number":96,"context_line":"        with RingReader(tempf) as reader:"},{"line_number":97,"context_line":"            self.assertEqual(reader.version, 0)"},{"line_number":98,"context_line":"            self.assertEqual(data, pickle.load(reader))"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"            # section-based access only works for v2"},{"line_number":101,"context_line":"            self.assertFalse(\u0027foo\u0027 in reader)"}],"source_content_type":"text/x-python","patch_set":17,"id":"b34ba5ab_75536b36","line":98,"updated":"2023-03-29 00:40:00.000000000","message":"so here it looks like we\u0027re using read as a fp\n\nI assume we could also:\n\n    pickle.loads(reader.read())","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":95,"context_line":""},{"line_number":96,"context_line":"        with RingReader(tempf) as reader:"},{"line_number":97,"context_line":"            self.assertEqual(reader.version, 0)"},{"line_number":98,"context_line":"            self.assertEqual(data, pickle.load(reader))"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"            # section-based access only works for v2"},{"line_number":101,"context_line":"            self.assertFalse(\u0027foo\u0027 in reader)"}],"source_content_type":"text/x-python","patch_set":17,"id":"4f6dcf8c_989f0375","line":98,"in_reply_to":"b34ba5ab_75536b36","updated":"2023-04-03 23:05:30.000000000","message":"Yes -- but this also provides some coverage for pickle\u0027s requirement on a `readinto` for pythons with https://bugs.python.org/issue36785 addressed but not https://bugs.python.org/issue39681.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":101,"context_line":"            self.assertFalse(\u0027foo\u0027 in reader)"},{"line_number":102,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":103,"context_line":"                with reader.open_section(\u0027foo\u0027):"},{"line_number":104,"context_line":"                    pass"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"    @with_tempdir"},{"line_number":107,"context_line":"    def test_sections(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"1bf9d634_978e1782","line":104,"updated":"2023-03-29 00:40:00.000000000","message":"oh, i\u0027d like to see what this ValueError looks like!  you can capture:\n\n    with assertRaises as ctx:\n        ...\n    assertIn(\u0027some string\u0027, str(ctx.exception))\n    \n... probably something like \"no index loaded\"","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"11b7880a1275f04ac271690978ed36008dde2021","unresolved":false,"context_lines":[{"line_number":101,"context_line":"            self.assertFalse(\u0027foo\u0027 in reader)"},{"line_number":102,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":103,"context_line":"                with reader.open_section(\u0027foo\u0027):"},{"line_number":104,"context_line":"                    pass"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"    @with_tempdir"},{"line_number":107,"context_line":"    def test_sections(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"3c8a4173_444343f5","line":104,"in_reply_to":"1bf9d634_978e1782","updated":"2025-01-10 04:45:55.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":107,"context_line":"    def test_sections(self, tempd):"},{"line_number":108,"context_line":"        tempf \u003d os.path.join(tempd, \u0027ring-v2\u0027)"},{"line_number":109,"context_line":"        with RingWriter(tempf) as writer:"},{"line_number":110,"context_line":"            writer.write_magic(2)"},{"line_number":111,"context_line":"            with writer.section(\u0027foo\u0027):"},{"line_number":112,"context_line":"                writer.write_blob(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"},{"line_number":113,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"e37851f0_2cdf9b88","line":110,"updated":"2023-03-29 00:40:00.000000000","message":"ok and even tho this thing feels like a SeekableZlibWriter as soon as we get into the ringv2 index/section stuff it becomes more \"ring-like\" - in fact some of methods on this thing just bail early if self.version !\u003d 2","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":125,"context_line":"                # Can\u0027t nest sections"},{"line_number":126,"context_line":"                with self.assertRaises(ValueError):"},{"line_number":127,"context_line":"                    with writer.section(\u0027inner\u0027):"},{"line_number":128,"context_line":"                        pass"},{"line_number":129,"context_line":"                self.assertNotIn(\u0027inner\u0027, writer.index)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"            writer.write(b\u0027can add arbitrary bytes\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"81c0d9f9_857f6523","line":128,"updated":"2023-03-29 00:40:00.000000000","message":"ok, so with RingReader we have\n\n    with reader.open_section() as section:\n        section.read()\n        \nbut with RingWriter we just have:\n\n    with writer.section():\n        writer.write()\n        \nand the writer has to keep track of when it\u0027s in the context manager for you.\n\nmakes we wonder about:\n\n    with read.open_section() as s1;\n        with reader.open_section() as s2:\n            pass","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":125,"context_line":"                # Can\u0027t nest sections"},{"line_number":126,"context_line":"                with self.assertRaises(ValueError):"},{"line_number":127,"context_line":"                    with writer.section(\u0027inner\u0027):"},{"line_number":128,"context_line":"                        pass"},{"line_number":129,"context_line":"                self.assertNotIn(\u0027inner\u0027, writer.index)"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"            writer.write(b\u0027can add arbitrary bytes\u0027)"}],"source_content_type":"text/x-python","patch_set":17,"id":"6eb4717f_e3c9fb72","line":128,"in_reply_to":"81c0d9f9_857f6523","updated":"2023-04-03 23:05:30.000000000","message":"I mean, the return for `RingWriter.section` is the `RingWriter` -- there\u0027s nothing *stopping* you from saying\n\n with writer.section(\u0027foo\u0027) as section:\n     section.write(b\u0027data\u0027)\n\n... it\u0027s just a little misleading as to what\u0027s going on.\n\nNesting `open_section()` calls will generally end badly -- I can add a check for that if you like.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult; see below."},{"line_number":133,"context_line":"            # This *is not* a recommended pattern -- write proper length-value"},{"line_number":134,"context_line":"            # blobs instead (even if you don\u0027t include them as sections in the"},{"line_number":135,"context_line":"            # index)."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":138,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":17,"id":"25710465_62c30fa3","line":135,"updated":"2023-03-29 00:40:00.000000000","message":"ohhhh kay....\n\ni mean, i guess this *has* to be supported because we re-use this class to write v1 rings?\n\nmaybe we could use a _write internally and disable public write on v2?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":132,"context_line":"            # ...though accessing them on read may be difficult; see below."},{"line_number":133,"context_line":"            # This *is not* a recommended pattern -- write proper length-value"},{"line_number":134,"context_line":"            # blobs instead (even if you don\u0027t include them as sections in the"},{"line_number":135,"context_line":"            # index)."},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"            with writer.section(\u0027quux\u0027):"},{"line_number":138,"context_line":"                writer.write_blob(b\u0027data\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":17,"id":"d2b88418_03578c7b","line":135,"in_reply_to":"25710465_62c30fa3","updated":"2023-04-03 23:05:30.000000000","message":"This is python -- there is no public, no private -- you don\u0027t want to know the unholy things I\u0027ve done using other people\u0027s \"internal\" APIs. I\u0027d prefer we have some knowledge and understanding of what happens in unsupported cases and advise against getting into such a situation, rather than pretend it can\u0027t come up.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":146,"context_line":"                with writer.section(\u0027foo\u0027):"},{"line_number":147,"context_line":"                    pass"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"            # We\u0027re reserving globs, so we can later support something like"},{"line_number":150,"context_line":"            # reader.load_sections(\u0027swift/ring/*\u0027)"},{"line_number":151,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":152,"context_line":"                with writer.section(\u0027foo*\u0027):"}],"source_content_type":"text/x-python","patch_set":17,"id":"76d33af2_0e1d94a4","line":149,"updated":"2023-03-29 00:40:00.000000000","message":"ok, there you go - i was wondering why we didn\u0027t support names with \u0027*\u0027\n\nshould we go ahead and restrict them to \"alphanum + \u0027/\u0027\" too!?  always easier to open it up later.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":146,"context_line":"                with writer.section(\u0027foo\u0027):"},{"line_number":147,"context_line":"                    pass"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"            # We\u0027re reserving globs, so we can later support something like"},{"line_number":150,"context_line":"            # reader.load_sections(\u0027swift/ring/*\u0027)"},{"line_number":151,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":152,"context_line":"                with writer.section(\u0027foo*\u0027):"}],"source_content_type":"text/x-python","patch_set":17,"id":"ad93fe78_7a9eecc7","line":149,"in_reply_to":"76d33af2_0e1d94a4","updated":"2023-04-03 23:05:30.000000000","message":"*shrug* w/e","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5134e010fff3612f434ba2f4f0cd791350a507c9","unresolved":false,"context_lines":[{"line_number":146,"context_line":"                with writer.section(\u0027foo\u0027):"},{"line_number":147,"context_line":"                    pass"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"            # We\u0027re reserving globs, so we can later support something like"},{"line_number":150,"context_line":"            # reader.load_sections(\u0027swift/ring/*\u0027)"},{"line_number":151,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":152,"context_line":"                with writer.section(\u0027foo*\u0027):"}],"source_content_type":"text/x-python","patch_set":17,"id":"a94a655d_93a30abc","line":149,"in_reply_to":"ad93fe78_7a9eecc7","updated":"2025-01-28 17:30:08.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":156,"context_line":"            self.assertEqual(reader.version, 2)"},{"line_number":157,"context_line":"            # Order matters!"},{"line_number":158,"context_line":"            self.assertEqual(list(reader.index), ["},{"line_number":159,"context_line":"                \u0027foo\u0027, \u0027bar\u0027, \u0027baz\u0027, \u0027quux\u0027, \u0027swift/index\u0027])"},{"line_number":160,"context_line":"            self.assertEqual({"},{"line_number":161,"context_line":"                k: (uncomp_start, uncomp_end, algo)"},{"line_number":162,"context_line":"                for k, (_, uncomp_start, _, uncomp_end, algo, _)"}],"source_content_type":"text/x-python","patch_set":17,"id":"31a2e65e_f1eafe31","line":159,"updated":"2023-03-29 00:40:00.000000000","message":"the index is a map of names to IndexEntries, it\u0027s an ordered dict, but how does that *matter*?  just assuming we\u0027d want to read the file mostly linearly?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":156,"context_line":"            self.assertEqual(reader.version, 2)"},{"line_number":157,"context_line":"            # Order matters!"},{"line_number":158,"context_line":"            self.assertEqual(list(reader.index), ["},{"line_number":159,"context_line":"                \u0027foo\u0027, \u0027bar\u0027, \u0027baz\u0027, \u0027quux\u0027, \u0027swift/index\u0027])"},{"line_number":160,"context_line":"            self.assertEqual({"},{"line_number":161,"context_line":"                k: (uncomp_start, uncomp_end, algo)"},{"line_number":162,"context_line":"                for k, (_, uncomp_start, _, uncomp_end, algo, _)"}],"source_content_type":"text/x-python","patch_set":17,"id":"c9bec933_4ca4a9cf","line":159,"in_reply_to":"31a2e65e_f1eafe31","updated":"2023-04-03 23:05:30.000000000","message":"\u003e just assuming we\u0027d want to read the file mostly linearly?\n\nYep. We\u0027ve gone to some effort to ensure that you can read sections back in an arbitrary order, but the fact remains that the file on disk has sections in a particular order and we may as well present that information to the caller.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":159,"context_line":"                \u0027foo\u0027, \u0027bar\u0027, \u0027baz\u0027, \u0027quux\u0027, \u0027swift/index\u0027])"},{"line_number":160,"context_line":"            self.assertEqual({"},{"line_number":161,"context_line":"                k: (uncomp_start, uncomp_end, algo)"},{"line_number":162,"context_line":"                for k, (_, uncomp_start, _, uncomp_end, algo, _)"},{"line_number":163,"context_line":"                in reader.index.items()"},{"line_number":164,"context_line":"            }, {"},{"line_number":165,"context_line":"                \u0027foo\u0027: (6, 40974, \u0027sha256\u0027),"}],"source_content_type":"text/x-python","patch_set":17,"id":"43ff60fd_80c13e9a","line":162,"updated":"2023-03-29 00:40:00.000000000","message":"i forget all the elements of this named tuple - some of them have to do with the compressed offsets?  we have some \"length\" bits in there in between sections too, right?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":159,"context_line":"                \u0027foo\u0027, \u0027bar\u0027, \u0027baz\u0027, \u0027quux\u0027, \u0027swift/index\u0027])"},{"line_number":160,"context_line":"            self.assertEqual({"},{"line_number":161,"context_line":"                k: (uncomp_start, uncomp_end, algo)"},{"line_number":162,"context_line":"                for k, (_, uncomp_start, _, uncomp_end, algo, _)"},{"line_number":163,"context_line":"                in reader.index.items()"},{"line_number":164,"context_line":"            }, {"},{"line_number":165,"context_line":"                \u0027foo\u0027: (6, 40974, \u0027sha256\u0027),"}],"source_content_type":"text/x-python","patch_set":17,"id":"53183e46_2217ab73","line":162,"in_reply_to":"43ff60fd_80c13e9a","updated":"2023-04-03 23:05:30.000000000","message":"\u003e i forget all the elements of this named tuple - some of them have to do with the compressed offsets?\n\nYeah -- compressed start, uncompressed start, compressed end, uncompressed end, checksum algo, checksum value.\n\n\u003e we have some \"length\" bits in there in between sections too, right?\n\nLengths get computed as `end - start`.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":181,"context_line":"                self.assertRepeats(s.read(), b\u0027\\xda\\x7a\\xda\\x7a\u0027, 10230)"},{"line_number":182,"context_line":"            # If you know that one section follows another, you don\u0027t *have*"},{"line_number":183,"context_line":"            # to \"open\" the next one"},{"line_number":184,"context_line":"            self.assertRepeats(reader.read_blob(), b\u0027more\u0027, 10240)"},{"line_number":185,"context_line":"            self.assertRepeats(reader.read_section(\u0027quux\u0027),"},{"line_number":186,"context_line":"                               b\u0027data\u0027, 10240)"},{"line_number":187,"context_line":"            index_dict \u003d json.loads(reader.read_section(\u0027swift/index\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"923344eb_bc7191c3","line":184,"updated":"2023-03-29 00:40:00.000000000","message":"oh, ok this is the baz section - and read_blob() knows it\u0027s length so all that name/index non-sense is just for seeks and giggles\n\nI can get down on that, but it seems to me again like this interface is doing two differnt things over two different domains.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":181,"context_line":"                self.assertRepeats(s.read(), b\u0027\\xda\\x7a\\xda\\x7a\u0027, 10230)"},{"line_number":182,"context_line":"            # If you know that one section follows another, you don\u0027t *have*"},{"line_number":183,"context_line":"            # to \"open\" the next one"},{"line_number":184,"context_line":"            self.assertRepeats(reader.read_blob(), b\u0027more\u0027, 10240)"},{"line_number":185,"context_line":"            self.assertRepeats(reader.read_section(\u0027quux\u0027),"},{"line_number":186,"context_line":"                               b\u0027data\u0027, 10240)"},{"line_number":187,"context_line":"            index_dict \u003d json.loads(reader.read_section(\u0027swift/index\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"e5e32513_9f763ad5","line":184,"in_reply_to":"923344eb_bc7191c3","updated":"2023-04-03 23:05:30.000000000","message":"Comes in handy if you ever accidentally naively gunzip/gzip the whole thing: https://review.opendev.org/c/openstack/swift/+/849441","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"11b7880a1275f04ac271690978ed36008dde2021","unresolved":true,"context_lines":[{"line_number":181,"context_line":"                self.assertRepeats(s.read(), b\u0027\\xda\\x7a\\xda\\x7a\u0027, 10230)"},{"line_number":182,"context_line":"            # If you know that one section follows another, you don\u0027t *have*"},{"line_number":183,"context_line":"            # to \"open\" the next one"},{"line_number":184,"context_line":"            self.assertRepeats(reader.read_blob(), b\u0027more\u0027, 10240)"},{"line_number":185,"context_line":"            self.assertRepeats(reader.read_section(\u0027quux\u0027),"},{"line_number":186,"context_line":"                               b\u0027data\u0027, 10240)"},{"line_number":187,"context_line":"            index_dict \u003d json.loads(reader.read_section(\u0027swift/index\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"4e25e1ac_6cf7e34b","line":184,"in_reply_to":"b450a1b9_8605f36b","updated":"2025-01-10 04:45:55.000000000","message":"`tar.gz` requires a linear read of the whole thing to do much of anything interesting (such as just finding file names)\n\n`zip` imposes a decent bit of operational complexity to deal with the name change ([though I\u0027ve previously spiked it out for you](https://review.opendev.org/c/openstack/swift/+/865066))\n\nI threw that repair patch together not because I expect anyone to actually do such a thing, but more to demonstrate that the format is robust enough that **you can recover** from stupid shit like that.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"caaf9e65b8497cec6328d8c81a464d43c230fd56","unresolved":true,"context_lines":[{"line_number":181,"context_line":"                self.assertRepeats(s.read(), b\u0027\\xda\\x7a\\xda\\x7a\u0027, 10230)"},{"line_number":182,"context_line":"            # If you know that one section follows another, you don\u0027t *have*"},{"line_number":183,"context_line":"            # to \"open\" the next one"},{"line_number":184,"context_line":"            self.assertRepeats(reader.read_blob(), b\u0027more\u0027, 10240)"},{"line_number":185,"context_line":"            self.assertRepeats(reader.read_section(\u0027quux\u0027),"},{"line_number":186,"context_line":"                               b\u0027data\u0027, 10240)"},{"line_number":187,"context_line":"            index_dict \u003d json.loads(reader.read_section(\u0027swift/index\u0027))"}],"source_content_type":"text/x-python","patch_set":17,"id":"b450a1b9_8605f36b","line":184,"in_reply_to":"e5e32513_9f763ad5","updated":"2025-01-08 19:52:26.000000000","message":"how terrifying!  makes we forget again why not use `tar.gz` interface instead.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":187,"context_line":"            index_dict \u003d json.loads(reader.read_section(\u0027swift/index\u0027))"},{"line_number":188,"context_line":"            self.assertEqual(reader.index, {"},{"line_number":189,"context_line":"                section: IndexEntry(*entry)"},{"line_number":190,"context_line":"                for section, entry in index_dict.items()})"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"            # Missing section"},{"line_number":193,"context_line":"            with self.assertRaises(KeyError):"}],"source_content_type":"text/x-python","patch_set":17,"id":"51c5db20_fad54a70","line":190,"updated":"2023-03-29 00:40:00.000000000","message":"I see, maybe the named tuple makes serialization/deserialization through json easier somehow.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":187,"context_line":"            index_dict \u003d json.loads(reader.read_section(\u0027swift/index\u0027))"},{"line_number":188,"context_line":"            self.assertEqual(reader.index, {"},{"line_number":189,"context_line":"                section: IndexEntry(*entry)"},{"line_number":190,"context_line":"                for section, entry in index_dict.items()})"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"            # Missing section"},{"line_number":193,"context_line":"            with self.assertRaises(KeyError):"}],"source_content_type":"text/x-python","patch_set":17,"id":"6660924d_5fcf85a3","line":190,"in_reply_to":"51c5db20_fad54a70","updated":"2023-04-03 23:05:30.000000000","message":"A little, anyway -- it\u0027d be cleaner if `json.loads` had an `array_hook` like it has an `object_hook` but w/e","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":201,"context_line":"                          reader.index[\u0027baz\u0027].uncompressed_end)"},{"line_number":202,"context_line":"            self.assertGreater(gap_length, 0)"},{"line_number":203,"context_line":"            self.assertEqual(b\u0027can add arbitrary bytes\u0027,"},{"line_number":204,"context_line":"                             reader.read(gap_length))"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"    @with_tempdir"},{"line_number":207,"context_line":"    def test_sections_with_corruption(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"fa1e71c9_8b2e59d1","line":204,"updated":"2023-03-29 00:40:00.000000000","message":"and so this somehow *isn\u0027t* a length offset section, it\u0027s like a ringv1 blob in the middle of a v2 format archive - that\u0027s ... terrible.  Thank you for being so honest in the test!","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":201,"context_line":"                          reader.index[\u0027baz\u0027].uncompressed_end)"},{"line_number":202,"context_line":"            self.assertGreater(gap_length, 0)"},{"line_number":203,"context_line":"            self.assertEqual(b\u0027can add arbitrary bytes\u0027,"},{"line_number":204,"context_line":"                             reader.read(gap_length))"},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"    @with_tempdir"},{"line_number":207,"context_line":"    def test_sections_with_corruption(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"377fa230_fe5d3170","line":204,"in_reply_to":"fa1e71c9_8b2e59d1","updated":"2025-05-20 19:58:47.000000000","message":"Acknowledged","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":214,"context_line":"        with RingReader(tempf) as reader:"},{"line_number":215,"context_line":"            # if you open a section, you better read it all!"},{"line_number":216,"context_line":"            read_bytes \u003d b\u0027\u0027"},{"line_number":217,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":218,"context_line":"                with reader.open_section(\u0027foo\u0027) as s:"},{"line_number":219,"context_line":"                    read_bytes \u003d s.read(4)"},{"line_number":220,"context_line":"            self.assertEqual(b\u0027\\xde\\xad\\xbe\\xef\u0027, read_bytes)"}],"source_content_type":"text/x-python","patch_set":17,"id":"8f67a06f_3c01b882","line":217,"updated":"2023-03-29 00:40:00.000000000","message":"again i think `as ctx` would and assert on ctx.exception would make it more obvious which error is popping.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"11b7880a1275f04ac271690978ed36008dde2021","unresolved":false,"context_lines":[{"line_number":214,"context_line":"        with RingReader(tempf) as reader:"},{"line_number":215,"context_line":"            # if you open a section, you better read it all!"},{"line_number":216,"context_line":"            read_bytes \u003d b\u0027\u0027"},{"line_number":217,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":218,"context_line":"                with reader.open_section(\u0027foo\u0027) as s:"},{"line_number":219,"context_line":"                    read_bytes \u003d s.read(4)"},{"line_number":220,"context_line":"            self.assertEqual(b\u0027\\xde\\xad\\xbe\\xef\u0027, read_bytes)"}],"source_content_type":"text/x-python","patch_set":17,"id":"184d2286_9e77a6e4","line":217,"in_reply_to":"8f67a06f_3c01b882","updated":"2025-01-10 04:45:55.000000000","message":"Done","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":221,"context_line":""},{"line_number":222,"context_line":"            # if there\u0027s a digest mismatch, you can read data, but it\u0027ll"},{"line_number":223,"context_line":"            # throw an error on close"},{"line_number":224,"context_line":"            self.assertEqual(\u0027sha256\u0027, reader.index[\u0027foo\u0027].checksum_method)"},{"line_number":225,"context_line":"            reader.index[\u0027foo\u0027] \u003d IndexEntry(*("},{"line_number":226,"context_line":"                reader.index[\u0027foo\u0027][:-1] + (\u0027not-the-sha\u0027,)))"},{"line_number":227,"context_line":"            read_bytes \u003d b\u0027\u0027"}],"source_content_type":"text/x-python","patch_set":17,"id":"4f573005_f7270727","line":224,"updated":"2023-03-29 00:40:00.000000000","message":"i\u0027m really excited we have checksuming around sections - that\u0027s super cool, I didn\u0027t know ringv1 had that?\n\nhere in this test we learn that index objects have a checksum_method attribute but I had to go lookup the name of the checksum_vaule property?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":221,"context_line":""},{"line_number":222,"context_line":"            # if there\u0027s a digest mismatch, you can read data, but it\u0027ll"},{"line_number":223,"context_line":"            # throw an error on close"},{"line_number":224,"context_line":"            self.assertEqual(\u0027sha256\u0027, reader.index[\u0027foo\u0027].checksum_method)"},{"line_number":225,"context_line":"            reader.index[\u0027foo\u0027] \u003d IndexEntry(*("},{"line_number":226,"context_line":"                reader.index[\u0027foo\u0027][:-1] + (\u0027not-the-sha\u0027,)))"},{"line_number":227,"context_line":"            read_bytes \u003d b\u0027\u0027"}],"source_content_type":"text/x-python","patch_set":17,"id":"ffa23ac6_f0467dd2","line":224,"in_reply_to":"4f573005_f7270727","updated":"2023-04-03 23:05:30.000000000","message":"\u003e I didn\u0027t know ringv1 had that?\n\nIt doesn\u0027t -- I wanted to add some amount of ring-integrity-checking as part of v2. I seem to recall a class of bug when you\u0027ve got some ring-deployment process that doesn\u0027t do atomic overwrites, but can\u0027t find a launchpad bug for it right now... something where the reload bombs out in `json.loads()` or (worse) reads only a truncated replica2part2dev_id table.\n\n\u003e here in this test we learn that index objects have a checksum_method attribute but I had to go lookup the name of the checksum_value property?\n\nThe idea is that callers shouldn\u0027t really need to think about the checksumming -- as long as\n\n with reader.open_section(name) as section:\n     section.read()\n\ndoesn\u0027t raise any error, they can just trust the content.","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":223,"context_line":"            # throw an error on close"},{"line_number":224,"context_line":"            self.assertEqual(\u0027sha256\u0027, reader.index[\u0027foo\u0027].checksum_method)"},{"line_number":225,"context_line":"            reader.index[\u0027foo\u0027] \u003d IndexEntry(*("},{"line_number":226,"context_line":"                reader.index[\u0027foo\u0027][:-1] + (\u0027not-the-sha\u0027,)))"},{"line_number":227,"context_line":"            read_bytes \u003d b\u0027\u0027"},{"line_number":228,"context_line":"            with self.assertRaises(ValueError):"},{"line_number":229,"context_line":"                with reader.open_section(\u0027foo\u0027) as s:"}],"source_content_type":"text/x-python","patch_set":17,"id":"09447ef2_ac1286d3","line":226,"updated":"2023-03-29 00:40:00.000000000","message":"maybe as a # sanity before we monkey-patch the index we could do a happy path open and read and assert on the expected checksum_method value","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":250,"context_line":"            mock.call(\u0027swift.ring\u0027).warning("},{"line_number":251,"context_line":"                \u0027Ignoring unsupported checksum %s:%s for section %s\u0027,"},{"line_number":252,"context_line":"                \u0027not_a_digest\u0027, mock.ANY, \u0027foo\u0027),"},{"line_number":253,"context_line":"        ])"},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"    @with_tempdir"},{"line_number":256,"context_line":"    def test_recompressed(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"d37c400e_c922bf57","line":253,"updated":"2023-03-29 00:40:00.000000000","message":"So I guess we could add a new checksum method in new swift, and old swift could still open the ring files (w/o checksumming) - I think that\u0027s called \"forward compatibility\" and it\u0027s pretty are because it\u0027s hard to get right.\n\nDo you have some reason to predict we should invest now on needing different kinds of checksums in the future?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":250,"context_line":"            mock.call(\u0027swift.ring\u0027).warning("},{"line_number":251,"context_line":"                \u0027Ignoring unsupported checksum %s:%s for section %s\u0027,"},{"line_number":252,"context_line":"                \u0027not_a_digest\u0027, mock.ANY, \u0027foo\u0027),"},{"line_number":253,"context_line":"        ])"},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"    @with_tempdir"},{"line_number":256,"context_line":"    def test_recompressed(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"5c5443bc_85fd0af4","line":253,"in_reply_to":"d37c400e_c922bf57","updated":"2023-04-03 23:05:30.000000000","message":"\u003e Do you have some reason to predict we should invest now on needing different kinds of checksums in the future?\n\nOur copious use of MD5? How great would it have been if rings (or etags) could be configured to use sha256 or blake2 or something?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":270,"context_line":""},{"line_number":271,"context_line":"        with self.assertRaises(IOError):"},{"line_number":272,"context_line":"            # ...but we can\u0027t read it"},{"line_number":273,"context_line":"            RingReader(tempf)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"    @with_tempdir"},{"line_number":276,"context_line":"    def test_version_too_high(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"c0ff6517_114fec10","line":273,"updated":"2023-03-29 00:40:00.000000000","message":"it\u0027s not obvious to me what\u0027s going on here - is it because we forgot to write_magic in L269?","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":270,"context_line":""},{"line_number":271,"context_line":"        with self.assertRaises(IOError):"},{"line_number":272,"context_line":"            # ...but we can\u0027t read it"},{"line_number":273,"context_line":"            RingReader(tempf)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"    @with_tempdir"},{"line_number":276,"context_line":"    def test_version_too_high(self, tempd):"}],"source_content_type":"text/x-python","patch_set":17,"id":"d206cc86_28abcfcd","line":273,"in_reply_to":"c0ff6517_114fec10","updated":"2023-04-03 23:05:30.000000000","message":"Nah -- when we read the whole thing on L266, it includes the magic. The trouble is that after determining that it\u0027s a v2 ring, we can\u0027t read the index (or rather, we can\u0027t read the *index offset*, since it\u0027s neither uncompressed nor preceded by a flush).","commit_id":"59b5bf4da4ada7c28bbc08d8af0cc417727c5cfd"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":179,"context_line":"            \u0027baz\u0027: (81942, 122910, \u0027sha256\u0027),"},{"line_number":180,"context_line":"            # note the gap between baz and quux for the raw bytes"},{"line_number":181,"context_line":"            \u0027quux\u0027: (122933, 163901, \u0027sha256\u0027),"},{"line_number":182,"context_line":"            \u0027swift/index\u0027: (163901, None, None),"},{"line_number":183,"context_line":"        })"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        self.assertIn(\u0027foo\u0027, reader)"}],"source_content_type":"text/x-python","patch_set":36,"id":"7c361b42_47340e14","line":182,"updated":"2025-06-13 21:04:18.000000000","message":"I\u0027m not sure I understand the justification for including the anemic IndexEntry for the index itself in the index\n\nmaybe we should write the end/checksum fields for index at the same time we write the compressed_start as some kind of uncompressed fixed width struct - or maybe keep the schemeless/json design for the index and just don\u0027t include half of the index in itself?\n\nOTOH, maybe the only harm in having it half-filled-in is inconsistency with the other sections, of which the index is obviously not just your \"normal\" section?  Of course if the index itself not just a \"normal\" section - maybe it should NOT be in the index?\n\n\u003e Special cases aren\u0027t special enough to break the rules.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":179,"context_line":"            \u0027baz\u0027: (81942, 122910, \u0027sha256\u0027),"},{"line_number":180,"context_line":"            # note the gap between baz and quux for the raw bytes"},{"line_number":181,"context_line":"            \u0027quux\u0027: (122933, 163901, \u0027sha256\u0027),"},{"line_number":182,"context_line":"            \u0027swift/index\u0027: (163901, None, None),"},{"line_number":183,"context_line":"        })"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        self.assertIn(\u0027foo\u0027, reader)"}],"source_content_type":"text/x-python","patch_set":36,"id":"de6ccabd_8d9699d7","line":182,"in_reply_to":"13f4b7cd_1fc65001","updated":"2025-06-18 22:21:44.000000000","message":"\u003e I like having the index tell you as much as possible about what\u0027s written down.\n\nbut I think it\u0027s already not telling you \"as much as possible\" it\u0027s just telling you \"as much as is convenient\" - we *could* calculate a checksum and end bytes for the index and write them down - we just can\u0027t write them down *in the index*\n\nIf we wanted we could instead write down the IndexEntry for the index in it\u0027s own blob after we write the index; and point to *that* from the index pointer and read the index\u0027s IndexEntry to load the section index (with checksumming!).\n\nMaybe ultimately maybe it\u0027s too late to have a checksum on the index; but I can\u0027t shake that it *is* different that every other section.\n\n * it\u0027s the one and only blob we don\u0027t read by looking up it\u0027s start location in the index\n * it\u0027s the one and only blob that doesn\u0027t get checksumming\n * it\u0027s the one and only blob that doesn\u0027t record it\u0027s end locations\n\nI think the index is ultimately just not a *section* - it\u0027s a magical blob at the end of the file we find with the index pointer(s).  If we want the index\u0027s IndexEntry we should write it down completely - and therefore not in the index.\n\nAlternatively, if we don\u0027t want (or need) the index\u0027s IndexEntry then we shouldn\u0027t write it down - and therefore remove the anemic/why-not confusingly self-referential IndexEntry from the index.  Which makes all the entries in the index clearly *sections* - which all have equivalent properties.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":179,"context_line":"            \u0027baz\u0027: (81942, 122910, \u0027sha256\u0027),"},{"line_number":180,"context_line":"            # note the gap between baz and quux for the raw bytes"},{"line_number":181,"context_line":"            \u0027quux\u0027: (122933, 163901, \u0027sha256\u0027),"},{"line_number":182,"context_line":"            \u0027swift/index\u0027: (163901, None, None),"},{"line_number":183,"context_line":"        })"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        self.assertIn(\u0027foo\u0027, reader)"}],"source_content_type":"text/x-python","patch_set":36,"id":"13f4b7cd_1fc65001","line":182,"in_reply_to":"7c361b42_47340e14","updated":"2025-06-16 23:28:49.000000000","message":"\u003e Special cases aren\u0027t special enough to break the rules.\n\nMy read is that this doesn\u0027t help push `swift/index` towards inclusion **or** away from it.\n\nIs the rule \"all blobs should be in the index\"? Then it should, too.\n\nIs the rule \"all checksum-enabled sections should go in the index\"? Then no.\n\nMy gut is to go with that first rule -- I like having the index tell you as much as possible about what\u0027s written down.\n\nThe long and short of it is that you *can\u0027t* include length and checksum info for the index *in the index itself* because those values impact what the checksum will be -- trying to *find* values that would actually satisfy the contract starts to smell like building a blockchain.\n\n\u003e Although practicality beats purity.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"edcd41bf5a1671e45dbaecbf1adfe84a2a5de49a","unresolved":true,"context_lines":[{"line_number":50,"context_line":"    def test_arbitrary_bytes(self):"},{"line_number":51,"context_line":"        buf \u003d io.BytesIO()"},{"line_number":52,"context_line":"        with RingWriter(buf) as writer:"},{"line_number":53,"context_line":"            # Still need to write good magic, or we won\u0027t be able to read"},{"line_number":54,"context_line":"            writer.write_magic(1)"},{"line_number":55,"context_line":"            # but after that, we can kinda do whatever"},{"line_number":56,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":42,"id":"bfc95693_a9198cbc","line":53,"updated":"2025-07-25 16:56:07.000000000","message":"This is probably pointing to the fact that you really *don\u0027t* want to shove arbitrary bytes in here, now that we don\u0027t need to worry about pickled rings... it really *is* new-rings-specific now. Might want to just drop this test.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[{"line_number":50,"context_line":"    def test_arbitrary_bytes(self):"},{"line_number":51,"context_line":"        buf \u003d io.BytesIO()"},{"line_number":52,"context_line":"        with RingWriter(buf) as writer:"},{"line_number":53,"context_line":"            # Still need to write good magic, or we won\u0027t be able to read"},{"line_number":54,"context_line":"            writer.write_magic(1)"},{"line_number":55,"context_line":"            # but after that, we can kinda do whatever"},{"line_number":56,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":42,"id":"b417e794_758f01b1","line":53,"in_reply_to":"110abf24_4134b781","updated":"2025-08-05 07:23:12.000000000","message":"Done","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a05f263a47724d0f661eb153527114722dfe42a8","unresolved":true,"context_lines":[{"line_number":50,"context_line":"    def test_arbitrary_bytes(self):"},{"line_number":51,"context_line":"        buf \u003d io.BytesIO()"},{"line_number":52,"context_line":"        with RingWriter(buf) as writer:"},{"line_number":53,"context_line":"            # Still need to write good magic, or we won\u0027t be able to read"},{"line_number":54,"context_line":"            writer.write_magic(1)"},{"line_number":55,"context_line":"            # but after that, we can kinda do whatever"},{"line_number":56,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":42,"id":"110abf24_4134b781","line":53,"in_reply_to":"77ea51c9_916519f2","updated":"2025-07-30 07:49:51.000000000","message":"I guess is doesn\u0027t hurt to stay for now.. but also happy for it to be removed. If you do spin one more version then remove it. Otherwise another follow up.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"acd97a2ec90ff7e1c972b8a0f9def93a0d2b223c","unresolved":true,"context_lines":[{"line_number":50,"context_line":"    def test_arbitrary_bytes(self):"},{"line_number":51,"context_line":"        buf \u003d io.BytesIO()"},{"line_number":52,"context_line":"        with RingWriter(buf) as writer:"},{"line_number":53,"context_line":"            # Still need to write good magic, or we won\u0027t be able to read"},{"line_number":54,"context_line":"            writer.write_magic(1)"},{"line_number":55,"context_line":"            # but after that, we can kinda do whatever"},{"line_number":56,"context_line":"            writer.write(b\u0027\\xde\\xad\\xbe\\xef\u0027 * 10240)"}],"source_content_type":"text/x-python","patch_set":42,"id":"77ea51c9_916519f2","line":53,"in_reply_to":"bfc95693_a9198cbc","updated":"2025-07-28 09:02:29.000000000","message":"Yeah, not really sure what this test is doing except that it can open up something with the right magic and then can read data from the file","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"}],"test/unit/common/ring/test_ring.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"357c90f3903834ee1797f4742e6ff026de707982","unresolved":true,"context_lines":[{"line_number":1359,"context_line":"    def deserialize_v2(cls, reader, *args, **kwargs):"},{"line_number":1360,"context_line":"        ring_data \u003d super(ExtendedRing, cls).deserialize_v2("},{"line_number":1361,"context_line":"            reader, *args, **kwargs)"},{"line_number":1362,"context_line":"        # This step kinda sucks :-("},{"line_number":1363,"context_line":"        obj \u003d cls(ring_data[\u0027replica2part2dev_id\u0027],"},{"line_number":1364,"context_line":"                  ring_data[\u0027devs\u0027], ring_data[\u0027part_shift\u0027],"},{"line_number":1365,"context_line":"                  ring_data.get(\u0027next_part_power\u0027),"}],"source_content_type":"text/x-python","patch_set":5,"id":"64d03118_490b5c52","line":1362,"updated":"2022-06-30 02:05:07.000000000","message":"maybe my serialization classes will make it look cleaner :shrug:","commit_id":"00bdb0a26cf52bce1305f86a617662e7bb696655"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b2a9052883ff39e133e18ca1fb842e0d4aea873d","unresolved":true,"context_lines":[{"line_number":1359,"context_line":"    def deserialize_v2(cls, reader, *args, **kwargs):"},{"line_number":1360,"context_line":"        ring_data \u003d super(ExtendedRing, cls).deserialize_v2("},{"line_number":1361,"context_line":"            reader, *args, **kwargs)"},{"line_number":1362,"context_line":"        # This step kinda sucks :-("},{"line_number":1363,"context_line":"        obj \u003d cls(ring_data[\u0027replica2part2dev_id\u0027],"},{"line_number":1364,"context_line":"                  ring_data[\u0027devs\u0027], ring_data[\u0027part_shift\u0027],"},{"line_number":1365,"context_line":"                  ring_data.get(\u0027next_part_power\u0027),"}],"source_content_type":"text/x-python","patch_set":5,"id":"b839c8c3_4e739861","line":1362,"in_reply_to":"64d03118_490b5c52","updated":"2022-07-08 22:48:31.000000000","message":"The more I think about this, the less I like it. The subclass has to break the deserialize_v2 API, with its\n\n        :returns: A dict containing `devs`, `part_shift`, and\n                  `replica2part2dev_id`\n\nbut there wasn\u0027t any other way to hang the new property off the final RingData. Then there\u0027s the fact that the next time we add a new arg to our RingData constructor, all extenders need to update, too.\n\nMaybe I could get somewhere better with a new from_dict() class method? It feels a little weird that we\u0027ve got a to_dict() with no from_dict() anyway...","commit_id":"00bdb0a26cf52bce1305f86a617662e7bb696655"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":1359,"context_line":"    def deserialize_v2(cls, reader, *args, **kwargs):"},{"line_number":1360,"context_line":"        ring_data \u003d super(ExtendedRing, cls).deserialize_v2("},{"line_number":1361,"context_line":"            reader, *args, **kwargs)"},{"line_number":1362,"context_line":"        # This step kinda sucks :-("},{"line_number":1363,"context_line":"        obj \u003d cls(ring_data[\u0027replica2part2dev_id\u0027],"},{"line_number":1364,"context_line":"                  ring_data[\u0027devs\u0027], ring_data[\u0027part_shift\u0027],"},{"line_number":1365,"context_line":"                  ring_data.get(\u0027next_part_power\u0027),"}],"source_content_type":"text/x-python","patch_set":5,"id":"e6b0f8b0_25394907","line":1362,"in_reply_to":"b839c8c3_4e739861","updated":"2022-10-21 20:02:17.000000000","message":"Done","commit_id":"00bdb0a26cf52bce1305f86a617662e7bb696655"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"227d72b0c53ee36a1a6725586bcfeddd13995720","unresolved":true,"context_lines":[{"line_number":1369,"context_line":"    def to_dict(self):"},{"line_number":1370,"context_line":"        ring_data \u003d super(ExtendedRing, self).to_dict()"},{"line_number":1371,"context_line":"        ring_data.setdefault(\u0027extra\u0027, self.extra)"},{"line_number":1372,"context_line":"        return ring_data"},{"line_number":1373,"context_line":""},{"line_number":1374,"context_line":"    def serialize_v2(self, writer):"},{"line_number":1375,"context_line":"        super(ExtendedRing, self).serialize_v2(writer)"}],"source_content_type":"text/x-python","patch_set":7,"id":"84e8e275_3f573c42","line":1372,"updated":"2022-07-20 10:24:42.000000000","message":"This is much cleaner! kudos","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"028f7e4e077d9c34b506cc0b80638446762d8a32","unresolved":false,"context_lines":[{"line_number":1369,"context_line":"    def to_dict(self):"},{"line_number":1370,"context_line":"        ring_data \u003d super(ExtendedRing, self).to_dict()"},{"line_number":1371,"context_line":"        ring_data.setdefault(\u0027extra\u0027, self.extra)"},{"line_number":1372,"context_line":"        return ring_data"},{"line_number":1373,"context_line":""},{"line_number":1374,"context_line":"    def serialize_v2(self, writer):"},{"line_number":1375,"context_line":"        super(ExtendedRing, self).serialize_v2(writer)"}],"source_content_type":"text/x-python","patch_set":7,"id":"782aaade_e522f7ec","line":1372,"in_reply_to":"1cf12ea6_5cf6d99e","updated":"2022-10-21 20:02:17.000000000","message":"Ack","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"de86ec7685e53f6895a571f5b3d18a80a3f9efc2","unresolved":true,"context_lines":[{"line_number":1369,"context_line":"    def to_dict(self):"},{"line_number":1370,"context_line":"        ring_data \u003d super(ExtendedRing, self).to_dict()"},{"line_number":1371,"context_line":"        ring_data.setdefault(\u0027extra\u0027, self.extra)"},{"line_number":1372,"context_line":"        return ring_data"},{"line_number":1373,"context_line":""},{"line_number":1374,"context_line":"    def serialize_v2(self, writer):"},{"line_number":1375,"context_line":"        super(ExtendedRing, self).serialize_v2(writer)"}],"source_content_type":"text/x-python","patch_set":7,"id":"1cf12ea6_5cf6d99e","line":1372,"in_reply_to":"84e8e275_3f573c42","updated":"2022-07-20 17:41:39.000000000","message":"Yeah, that from_dict() hook made this all *way* better.","commit_id":"352eb9d83d8365159136c49c8ee5aa88f7f03d12"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db5465794bce74c7cab1ea0ea4340193a9cc7f53","unresolved":true,"context_lines":[{"line_number":109,"context_line":"            {\u0027id\u0027: 0, \u0027zone\u0027: 0, \u0027region\u0027: 1},"},{"line_number":110,"context_line":"            {\u0027id\u0027: 1, \u0027zone\u0027: 1, \u0027region\u0027: 1},"},{"line_number":111,"context_line":"        ], meta_only.devs)"},{"line_number":112,"context_line":"        self.assertEqual([], meta_only._replica2part2dev_id)"},{"line_number":113,"context_line":"        rd2 \u003d ring.RingData.load(ring_fname)"},{"line_number":114,"context_line":"        self.assert_ring_data_equal(rd, rd2)"},{"line_number":115,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"afb2e742_2d7f4b8b","side":"PARENT","line":112,"updated":"2023-03-29 00:40:00.000000000","message":"it seems like maybe v1 would return devs - it would be nice to test roundtrip serialization for both ring serialization format versions.","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2d5abb1ca0e779758b150197f1d393c8e3ca51e0","unresolved":true,"context_lines":[{"line_number":109,"context_line":"            {\u0027id\u0027: 0, \u0027zone\u0027: 0, \u0027region\u0027: 1},"},{"line_number":110,"context_line":"            {\u0027id\u0027: 1, \u0027zone\u0027: 1, \u0027region\u0027: 1},"},{"line_number":111,"context_line":"        ], meta_only.devs)"},{"line_number":112,"context_line":"        self.assertEqual([], meta_only._replica2part2dev_id)"},{"line_number":113,"context_line":"        rd2 \u003d ring.RingData.load(ring_fname)"},{"line_number":114,"context_line":"        self.assert_ring_data_equal(rd, rd2)"},{"line_number":115,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"e7d9807c_fe8bce38","side":"PARENT","line":112,"in_reply_to":"afb2e742_2d7f4b8b","updated":"2023-04-03 23:05:30.000000000","message":"We have that below in `test_load` -- maybe I should just ax this test?","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":109,"context_line":"            {\u0027id\u0027: 0, \u0027zone\u0027: 0, \u0027region\u0027: 1},"},{"line_number":110,"context_line":"            {\u0027id\u0027: 1, \u0027zone\u0027: 1, \u0027region\u0027: 1},"},{"line_number":111,"context_line":"        ], meta_only.devs)"},{"line_number":112,"context_line":"        self.assertEqual([], meta_only._replica2part2dev_id)"},{"line_number":113,"context_line":"        rd2 \u003d ring.RingData.load(ring_fname)"},{"line_number":114,"context_line":"        self.assert_ring_data_equal(rd, rd2)"},{"line_number":115,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"a49fc81c_925aaaaf","side":"PARENT","line":112,"in_reply_to":"e7d9807c_fe8bce38","updated":"2025-06-13 21:04:18.000000000","message":"\u003e maybe I should just ax this test\n\nI think maybe better do the same `TestRingData.format_version` trick you did for `TestRing` - at least for the tests that use `save`; but yeah maybe there\u0027s pre-existing overlap.","commit_id":"03d8f823ef0b24d40376e6f8db995eece9215e47"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":true,"context_lines":[{"line_number":395,"context_line":"        self.assertEqual(loaded_rd.to_dict(), expected_rd_dict)"},{"line_number":396,"context_line":"        loaded_rd \u003d ring.RingData.load(ring_fname_1, metadata_only\u003dTrue)"},{"line_number":397,"context_line":"        expected_rd_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":398,"context_line":"        self.assertEqual(loaded_rd.to_dict(), expected_rd_dict)"},{"line_number":399,"context_line":""},{"line_number":400,"context_line":"    def test_save(self):"},{"line_number":401,"context_line":"        ring_fname \u003d os.path.join(self.testdir, \u0027foo.ring.gz\u0027)"}],"source_content_type":"text/x-python","patch_set":28,"id":"7136c079_b5ac7678","line":398,"updated":"2025-03-26 19:12:40.000000000","message":"test_load do... what?  how many different things are we testing?  happy path, error path?  version 1 and 2?  metadata_only True/False?","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":395,"context_line":"        self.assertEqual(loaded_rd.to_dict(), expected_rd_dict)"},{"line_number":396,"context_line":"        loaded_rd \u003d ring.RingData.load(ring_fname_1, metadata_only\u003dTrue)"},{"line_number":397,"context_line":"        expected_rd_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":398,"context_line":"        self.assertEqual(loaded_rd.to_dict(), expected_rd_dict)"},{"line_number":399,"context_line":""},{"line_number":400,"context_line":"    def test_save(self):"},{"line_number":401,"context_line":"        ring_fname \u003d os.path.join(self.testdir, \u0027foo.ring.gz\u0027)"}],"source_content_type":"text/x-python","patch_set":28,"id":"0eabe0ea_d2841f1e","line":398,"in_reply_to":"1f157d93_2ea28fbf","updated":"2025-06-13 21:04:18.000000000","message":"i.e. `test_consistent_ring_data_load` - might be more clear/concise if the `Unknown ring format version` errors were split out.\n\nProbably also `test_consistent_ring_data_metadata` could say \"ring v1 \u0026 v2 are the same; but look ring v0 always loads devices\"","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":395,"context_line":"        self.assertEqual(loaded_rd.to_dict(), expected_rd_dict)"},{"line_number":396,"context_line":"        loaded_rd \u003d ring.RingData.load(ring_fname_1, metadata_only\u003dTrue)"},{"line_number":397,"context_line":"        expected_rd_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":398,"context_line":"        self.assertEqual(loaded_rd.to_dict(), expected_rd_dict)"},{"line_number":399,"context_line":""},{"line_number":400,"context_line":"    def test_save(self):"},{"line_number":401,"context_line":"        ring_fname \u003d os.path.join(self.testdir, \u0027foo.ring.gz\u0027)"}],"source_content_type":"text/x-python","patch_set":28,"id":"1f157d93_2ea28fbf","line":398,"in_reply_to":"7136c079_b5ac7678","updated":"2025-05-06 20:28:41.000000000","message":"I can break out the `metadata_only\u003dTrue` cases, but I think there\u0027s value in seeing that the same Python data structure can be serialized in any of these three ways and when loaded you get the exact same data.","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e787aa9151c3aee1df49e5e1bbd8d55cdb8a6964","unresolved":true,"context_lines":[{"line_number":413,"context_line":"        with self.assertRaises(ValueError) as err:"},{"line_number":414,"context_line":"            for version in (3, None, \"some version\"):"},{"line_number":415,"context_line":"                rd.save(ring_fname, format_version\u003dversion)"},{"line_number":416,"context_line":"        self.assertEqual(\"format_version mus\""},{"line_number":417,"context_line":"                         \"t be one of (0, 1, 2)\","},{"line_number":418,"context_line":"                         str(err.exception))"},{"line_number":419,"context_line":"        # re-serialisation is already handled in test_load."},{"line_number":420,"context_line":""}],"source_content_type":"text/x-python","patch_set":28,"id":"22dae9a3_585c5250","line":417,"range":{"start_line":416,"start_character":25,"end_line":417,"end_character":48},"updated":"2025-05-06 20:28:41.000000000","message":"Ick -- what happened here?","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"528bf7afce51ea8a4f80d013ccc79d3bcea0d110","unresolved":false,"context_lines":[{"line_number":413,"context_line":"        with self.assertRaises(ValueError) as err:"},{"line_number":414,"context_line":"            for version in (3, None, \"some version\"):"},{"line_number":415,"context_line":"                rd.save(ring_fname, format_version\u003dversion)"},{"line_number":416,"context_line":"        self.assertEqual(\"format_version mus\""},{"line_number":417,"context_line":"                         \"t be one of (0, 1, 2)\","},{"line_number":418,"context_line":"                         str(err.exception))"},{"line_number":419,"context_line":"        # re-serialisation is already handled in test_load."},{"line_number":420,"context_line":""}],"source_content_type":"text/x-python","patch_set":28,"id":"6f8dc3d9_d462a539","line":417,"range":{"start_line":416,"start_character":25,"end_line":417,"end_character":48},"in_reply_to":"22dae9a3_585c5250","updated":"2025-05-20 19:58:47.000000000","message":"Done","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"db4f03b1a3a0eeb5a00025d89a89b05ed2e02d86","unresolved":true,"context_lines":[{"line_number":1298,"context_line":"            with self.assertRaises(ValueError) as err:"},{"line_number":1299,"context_line":"                ring_data.set_dev_id_bytes(4)"},{"line_number":1300,"context_line":"            self.assertEqual(str(err.exception),"},{"line_number":1301,"context_line":"                             \u0027Too many devices for 4-byte device ids\u0027)"},{"line_number":1302,"context_line":""},{"line_number":1303,"context_line":""},{"line_number":1304,"context_line":"class TestRingV0(TestRing):"}],"source_content_type":"text/x-python","patch_set":28,"id":"dcfa9c2b_ec18d449","line":1301,"updated":"2025-03-26 19:12:40.000000000","message":"as best I can tell this is the only place we ever try to call set_dev_id_bytes on a RingData object (!!??)","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":false,"context_lines":[{"line_number":1298,"context_line":"            with self.assertRaises(ValueError) as err:"},{"line_number":1299,"context_line":"                ring_data.set_dev_id_bytes(4)"},{"line_number":1300,"context_line":"            self.assertEqual(str(err.exception),"},{"line_number":1301,"context_line":"                             \u0027Too many devices for 4-byte device ids\u0027)"},{"line_number":1302,"context_line":""},{"line_number":1303,"context_line":""},{"line_number":1304,"context_line":"class TestRingV0(TestRing):"}],"source_content_type":"text/x-python","patch_set":28,"id":"1a90b0db_af21ad3a","line":1301,"in_reply_to":"dcfa9c2b_ec18d449","updated":"2025-06-13 21:04:18.000000000","message":"Done","commit_id":"3a2af1b4dbedfec296b0ece0d50c63f3ccc0dd2c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":444,"context_line":"        loaded_dict \u003d loaded_rd.to_dict()"},{"line_number":445,"context_line":"        # v0 can\u0027t *not* load the assignments"},{"line_number":446,"context_line":"        self.assertEqual(len(loaded_dict[\u0027replica2part2dev_id\u0027]), 2)"},{"line_number":447,"context_line":"        # clear it so it can meet expectations"},{"line_number":448,"context_line":"        loaded_dict[\u0027replica2part2dev_id\u0027] \u003d []"},{"line_number":449,"context_line":"        self.assertEqual(expected_rd_dict, loaded_dict)"},{"line_number":450,"context_line":""}],"source_content_type":"text/x-python","patch_set":33,"id":"2b26dd63_76b45527","line":447,"updated":"2025-06-13 21:04:18.000000000","message":"I think there\u0027s still some indications that this test _could_ be structured differently to look more like a:\n\n```\ndef test_vX_specific_behavior():\n    # setup\n    rd \u003d _common_setup()\n    # crank turn\n    rd.save(version_to_test)\n    loaded_rd \u003d RingData.load()\n    # assert specific behavior\n    ....\n```","commit_id":"dd36c8a41eb8d39b90b31eb4caa396e8b5c5b0e7"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":73,"context_line":"            self.assertEqual([], rd_got._replica2part2dev_id)"},{"line_number":74,"context_line":"        else:"},{"line_number":75,"context_line":"            self.assertEqual(rd_expected._replica2part2dev_id,"},{"line_number":76,"context_line":"                             rd_got._replica2part2dev_id)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"    def test_attrs(self):"},{"line_number":79,"context_line":"        r2p2d \u003d [[0, 1, 0, 1], [0, 1, 0, 1]]"}],"source_content_type":"text/x-python","patch_set":34,"id":"4acfa190_d4b4f8f7","line":76,"updated":"2025-06-13 21:04:18.000000000","message":"ok, so metdata_only is a new kwarg that changes that the assert help actually verifies.","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":143,"context_line":"        rd.save(ring_fname)"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"        meta_only \u003d ring.RingData.load(ring_fname, metadata_only\u003dTrue)"},{"line_number":146,"context_line":"        self.assert_ring_data_equal(rd, meta_only, metadata_only\u003dTrue)"},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"        rd2 \u003d ring.RingData.load(ring_fname)"},{"line_number":149,"context_line":"        self.assert_ring_data_equal(rd, rd2)"}],"source_content_type":"text/x-python","patch_set":34,"id":"f4fc213a_c40c1f9a","line":146,"updated":"2025-06-13 21:04:18.000000000","message":"what I don\u0027t love about this phrasing is that we pass in a `rd` that *has* a replica2part2dev and a device list... then we assert it\u0027s equal to a `meta_only`\n\nI *think* what\u0027s going on is the `meta_only` *does* have the devices, but not a `replica2part2dev` - so it\u0027s not so much `rd ~\u003d meta_only`, but rather:\n\n```\nself.assert_similar_and_missing_replica2part2dev(rd, meta_only)\n```\n\n????","commit_id":"7ff680a1de8f0780b16e36b1c5b46122a3c256fe"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":265,"context_line":"                expected_md5.update(chunk)"},{"line_number":266,"context_line":"                expected_size +\u003d len(chunk)"},{"line_number":267,"context_line":"        self.assertEqual(self.ring.md5, expected_md5.hexdigest())"},{"line_number":268,"context_line":"        self.assertEqual(self.ring.size, expected_size)"},{"line_number":269,"context_line":""},{"line_number":270,"context_line":"        # test invalid endcap"},{"line_number":271,"context_line":"        with mock.patch.object(utils, \u0027HASH_PATH_SUFFIX\u0027, b\u0027\u0027), \\"}],"source_content_type":"text/x-python","patch_set":36,"id":"22fe799d_1acdf717","side":"PARENT","line":268,"updated":"2025-06-13 21:04:18.000000000","message":"so, `size` is the the... \"size of the .gz file\"","commit_id":"afacfb6cea88d1edd71c7a70a5532e3e12bd8304"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":327,"context_line":""},{"line_number":328,"context_line":"    def test_load(self):"},{"line_number":329,"context_line":"        rd \u003d ring.RingData("},{"line_number":330,"context_line":"            [[0, 1, 0, 1], [0, 1, 0, 1]],"},{"line_number":331,"context_line":"            [{\u0027id\u0027: 0, \u0027zone\u0027: 0, \u0027ip\u0027: \u002710.1.1.0\u0027, \u0027port\u0027: 7000},"},{"line_number":332,"context_line":"             {\u0027id\u0027: 1, \u0027zone\u0027: 1, \u0027ip\u0027: \u002710.1.1.1\u0027, \u0027port\u0027: 7000}],"},{"line_number":333,"context_line":"            30)"}],"source_content_type":"text/x-python","patch_set":36,"id":"5e5e7243_ec7ae17f","line":330,"updated":"2025-06-13 21:04:18.000000000","message":"so here we\u0027re passing in normal arrays of ints","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":347,"context_line":"        # Loading the bad ring will fail because it\u0027s an unknown version"},{"line_number":348,"context_line":"        with self.assertRaises(Exception) as ex:"},{"line_number":349,"context_line":"            ring.RingData.load(ring_fname_bad_version)"},{"line_number":350,"context_line":"        self.assertEqual(\u0027Unsupported ring version: 5\u0027, str(ex.exception))"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"        orig_load_index \u003d io.RingReader.load_index"},{"line_number":353,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"d1dd6a33_f5c39718","line":350,"updated":"2025-06-13 21:04:18.000000000","message":"this is pretty clearly testing loading future ring versions - we throw an error - that\u0027s smart; forward compact is near impossible to get right and every so rarely useful.","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":359,"context_line":"                        mock_load_index):"},{"line_number":360,"context_line":"            with self.assertRaises(Exception) as ex:"},{"line_number":361,"context_line":"                ring.RingData.load(ring_fname_1)"},{"line_number":362,"context_line":"        self.assertEqual(\u0027Unknown ring format version 5\u0027, str(ex.exception))"},{"line_number":363,"context_line":""},{"line_number":364,"context_line":"        expected_r2p2d \u003d ["},{"line_number":365,"context_line":"            array.array(\u0027H\u0027, [0, 1, 0, 1]),"}],"source_content_type":"text/x-python","patch_set":36,"id":"8fd79d09_3e1674b6","line":362,"updated":"2025-06-13 21:04:18.000000000","message":"I don\u0027t understand what\u0027s going on here, the ring_fname_1 was serialized with format_version\u003d1  what is the mock_load_index trying to show us?  Can we drop this block or move it to it\u0027s own test with more comments/explanation?","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1357,"context_line":"    FORMAT_VERSION \u003d 0"},{"line_number":1358,"context_line":""},{"line_number":1359,"context_line":""},{"line_number":1360,"context_line":"class ExtendedRing(ring.RingData):"},{"line_number":1361,"context_line":"    extra \u003d b\u0027some super-specific data\u0027"},{"line_number":1362,"context_line":""},{"line_number":1363,"context_line":"    def to_dict(self):"}],"source_content_type":"text/x-python","patch_set":36,"id":"1d19f2ee_105f0b4e","line":1360,"updated":"2025-06-13 21:04:18.000000000","message":"This isn\u0027t really an \"ExtendedRing\" - it\u0027s \"just\" and \"ExtendedRingData\" - if you wanted to *use* the new \"extra\" data you\u0027d have to subclass `Ring` and override `_reload` as well I think...","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1374,"context_line":"    def deserialize_v2(cls, reader, *args, **kwargs):"},{"line_number":1375,"context_line":"        ring_data \u003d super(ExtendedRing, cls).deserialize_v2("},{"line_number":1376,"context_line":"            reader, *args, **kwargs)"},{"line_number":1377,"context_line":"        with reader.open_section(\u0027my-custom-section\u0027) as s:"},{"line_number":1378,"context_line":"            ring_data[\u0027extra\u0027] \u003d s.read()"},{"line_number":1379,"context_line":"        return ring_data"},{"line_number":1380,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"d2230194_02659bc4","line":1377,"updated":"2025-06-13 21:04:18.000000000","message":"I\u0027m struggling to imagine if a \"real\" out-of-tree `ExtendedRingData` would want to check for `\u0027my-custom-section\u0027 in reader.index` or handle the KeyError on `open_section` - I guess it depends on if the new tool/script trying to take advantage of \"my-custom-section\" can do it\u0027s job without \"extra\"","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1381,"context_line":"    @classmethod"},{"line_number":1382,"context_line":"    def from_dict(cls, ring_data):"},{"line_number":1383,"context_line":"        obj \u003d super(ExtendedRing, cls).from_dict(ring_data)"},{"line_number":1384,"context_line":"        obj.extra \u003d ring_data.get(\u0027extra\u0027)"},{"line_number":1385,"context_line":"        return obj"},{"line_number":1386,"context_line":""},{"line_number":1387,"context_line":""}],"source_content_type":"text/x-python","patch_set":36,"id":"d2c713f4_c8ce078c","line":1384,"updated":"2025-06-13 21:04:18.000000000","message":"... here it\u0027s attempting to be robust to the `ring_dict` not having an extra","commit_id":"ed3556da3b22361874b3da31112831be75ec1fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1302,"context_line":""},{"line_number":1303,"context_line":"            index[\u0027swift/ring/metadata\u0027] \u003d ["},{"line_number":1304,"context_line":"                os.fstat(fp.fileno()).st_size, fp.tell(),"},{"line_number":1305,"context_line":"                None, None, None, None]"},{"line_number":1306,"context_line":"            meta \u003d json.dumps({"},{"line_number":1307,"context_line":"                \"dev_id_bytes\": 4,"},{"line_number":1308,"context_line":"                \"part_shift\": 29,"}],"source_content_type":"text/x-python","patch_set":38,"id":"76267abc_d81906db","line":1305,"updated":"2025-06-13 21:04:18.000000000","message":"I don\u0027t know if it\u0027s only a good thing that our deserializer is robust to an \"invalid\" index; or rather that we\u0027ve defined \"didn\u0027t write down the end of the section nor a checksum\" as \"valid\"\n\n... maybe Postel\u0027s Law applies, but it feels like baking in Protocol Decay on purpose from the start is only hurting us.","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":1302,"context_line":""},{"line_number":1303,"context_line":"            index[\u0027swift/ring/metadata\u0027] \u003d ["},{"line_number":1304,"context_line":"                os.fstat(fp.fileno()).st_size, fp.tell(),"},{"line_number":1305,"context_line":"                None, None, None, None]"},{"line_number":1306,"context_line":"            meta \u003d json.dumps({"},{"line_number":1307,"context_line":"                \"dev_id_bytes\": 4,"},{"line_number":1308,"context_line":"                \"part_shift\": 29,"}],"source_content_type":"text/x-python","patch_set":38,"id":"a5b8ccf9_78559acb","line":1305,"in_reply_to":"69939998_2e2b98f5","updated":"2025-06-18 22:21:44.000000000","message":"\u003e Part of this is demonstrating that there\u0027s nothing special about swift/index\n\nI don\u0027t think that\u0027s the right way to think about it, the index is very unique as it\u0027s the only section you can find w/o the index b/c of the uncompressed index pointer at the end of the file; but I think I hear you saying \"look, checksumming is totally optional - so it\u0027s not weird that the index isn\u0027t checksummed\"\n\nI\u0027m saying: let\u0027s make checksumming expected/required for all v2 format sections - and let\u0027s acknowledge the index isn\u0027t a section.\n\n\u003e can omit length and checksum info\n\nright, I think that\u0027s what I meant by \"protocol decay\" - it\u0027s not obviously a good thing.  Do you have a use-case in mind for a section that can\u0027t discover it\u0027s length or we don\u0027t want to checksum?  Or is the main reason for wanting to support that theoretical future so that we don\u0027t have to special case the index?\n\n\u003e it\u0027ll continue to be a snapshot of what was valid now.\n\nconversely if we define \"valid\" as very liberal to start - it will be harder to make stronger expectations and requirements later.","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":1302,"context_line":""},{"line_number":1303,"context_line":"            index[\u0027swift/ring/metadata\u0027] \u003d ["},{"line_number":1304,"context_line":"                os.fstat(fp.fileno()).st_size, fp.tell(),"},{"line_number":1305,"context_line":"                None, None, None, None]"},{"line_number":1306,"context_line":"            meta \u003d json.dumps({"},{"line_number":1307,"context_line":"                \"dev_id_bytes\": 4,"},{"line_number":1308,"context_line":"                \"part_shift\": 29,"}],"source_content_type":"text/x-python","patch_set":38,"id":"69939998_2e2b98f5","line":1305,"in_reply_to":"76267abc_d81906db","updated":"2025-06-16 23:28:49.000000000","message":"Part of this is demonstrating that there\u0027s nothing special about `swift/index` -- other sections (including, potentially, ones added in the future) can omit length and checksum info.\n\nThe other part is showing how you don\u0027t need a `RingWriter` to build these things -- `gzip.GzipFile` and `zlib` are all you need. Which means this test can stay the same as we continue to evolve the format over time, and (minus the checksumming...) it\u0027ll continue to be a snapshot of what was valid now.","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1329,"context_line":"            index[\u0027swift/ring/assignments\u0027] \u003d ["},{"line_number":1330,"context_line":"                os.fstat(fp.fileno()).st_size, fp.tell(),"},{"line_number":1331,"context_line":"                None, None, None, None]"},{"line_number":1332,"context_line":"            fp.write(struct.pack(\u0027!Q\u0027, 48) + 4 * ("},{"line_number":1333,"context_line":"                b\u0027\\x00\\x00\\x00\\x03\u0027"},{"line_number":1334,"context_line":"                b\u0027\\x00\\x00\\x00\\x02\u0027"},{"line_number":1335,"context_line":"                b\u0027\\x00\\x00\\x00\\x00\u0027))"}],"source_content_type":"text/x-python","patch_set":38,"id":"8c721ed7_c5c641c8","line":1332,"updated":"2025-06-13 21:04:18.000000000","message":"why is 48 the correct size for this table?  48 \u003d\u003d 4 * 12... but I\u0027m counting 3 replicas with 8 parts - shouldn\u0027t that be 24 device assignments; each of which is 4 bytes; for a total of 96 bytes?\n\nI think this is like a 1.5 replica ring or something...","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[{"line_number":1329,"context_line":"            index[\u0027swift/ring/assignments\u0027] \u003d ["},{"line_number":1330,"context_line":"                os.fstat(fp.fileno()).st_size, fp.tell(),"},{"line_number":1331,"context_line":"                None, None, None, None]"},{"line_number":1332,"context_line":"            fp.write(struct.pack(\u0027!Q\u0027, 48) + 4 * ("},{"line_number":1333,"context_line":"                b\u0027\\x00\\x00\\x00\\x03\u0027"},{"line_number":1334,"context_line":"                b\u0027\\x00\\x00\\x00\\x02\u0027"},{"line_number":1335,"context_line":"                b\u0027\\x00\\x00\\x00\\x00\u0027))"}],"source_content_type":"text/x-python","patch_set":38,"id":"6dc915eb_24c68d1c","line":1332,"in_reply_to":"87abb5e6_3a07e245","updated":"2025-08-05 07:23:12.000000000","message":"Acknowledged","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":1329,"context_line":"            index[\u0027swift/ring/assignments\u0027] \u003d ["},{"line_number":1330,"context_line":"                os.fstat(fp.fileno()).st_size, fp.tell(),"},{"line_number":1331,"context_line":"                None, None, None, None]"},{"line_number":1332,"context_line":"            fp.write(struct.pack(\u0027!Q\u0027, 48) + 4 * ("},{"line_number":1333,"context_line":"                b\u0027\\x00\\x00\\x00\\x03\u0027"},{"line_number":1334,"context_line":"                b\u0027\\x00\\x00\\x00\\x02\u0027"},{"line_number":1335,"context_line":"                b\u0027\\x00\\x00\\x00\\x00\u0027))"}],"source_content_type":"text/x-python","patch_set":38,"id":"aca6c306_7258c13d","line":1332,"in_reply_to":"8c721ed7_c5c641c8","updated":"2025-06-16 23:28:49.000000000","message":"\u003e I think this is like a 1.5 replica ring or something...\n\nYeah, pretty clearly so given that last assertion.\n\nLooks like I added that test... three years ago? No idea whether I\u0027d intended it to be a 3 or 1.5 replica ring.\n\nStill probably not the stupidest test in our tree, though 😜","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":1329,"context_line":"            index[\u0027swift/ring/assignments\u0027] \u003d ["},{"line_number":1330,"context_line":"                os.fstat(fp.fileno()).st_size, fp.tell(),"},{"line_number":1331,"context_line":"                None, None, None, None]"},{"line_number":1332,"context_line":"            fp.write(struct.pack(\u0027!Q\u0027, 48) + 4 * ("},{"line_number":1333,"context_line":"                b\u0027\\x00\\x00\\x00\\x03\u0027"},{"line_number":1334,"context_line":"                b\u0027\\x00\\x00\\x00\\x02\u0027"},{"line_number":1335,"context_line":"                b\u0027\\x00\\x00\\x00\\x00\u0027))"}],"source_content_type":"text/x-python","patch_set":38,"id":"87abb5e6_3a07e245","line":1332,"in_reply_to":"aca6c306_7258c13d","updated":"2025-06-18 22:21:44.000000000","message":"\u003e Still probably not the stupidest test in our tree, though 😜\n\nmaybe not!","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1347,"context_line":"            fp.write(struct.pack(\u0027!Q\u0027, index[\u0027swift/index\u0027][0]))"},{"line_number":1348,"context_line":"            fp.flush(zlib.Z_FULL_FLUSH)"},{"line_number":1349,"context_line":""},{"line_number":1350,"context_line":"        r \u003d ring.Ring(ring_file)"},{"line_number":1351,"context_line":"        self.assertEqual("},{"line_number":1352,"context_line":"            [[d[\u0027id\u0027] for d in r.get_part_nodes(p)] for p in range(8)],"},{"line_number":1353,"context_line":"            [[3, 0], [2, 3], [0, 2], [3, 0], [2], [0], [3], [2]])"}],"source_content_type":"text/x-python","patch_set":38,"id":"28c13b61_adb877b9","line":1350,"updated":"2025-06-13 21:04:18.000000000","message":"why should a 3-replica ring produce a 2 row long replica2part2dev_id without error; is that reasonable?","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1350,"context_line":"        r \u003d ring.Ring(ring_file)"},{"line_number":1351,"context_line":"        self.assertEqual("},{"line_number":1352,"context_line":"            [[d[\u0027id\u0027] for d in r.get_part_nodes(p)] for p in range(8)],"},{"line_number":1353,"context_line":"            [[3, 0], [2, 3], [0, 2], [3, 0], [2], [0], [3], [2]])"},{"line_number":1354,"context_line":""},{"line_number":1355,"context_line":""},{"line_number":1356,"context_line":"class TestRingV0(TestRing):"}],"source_content_type":"text/x-python","patch_set":38,"id":"cefd36f0_51dabc74","line":1353,"updated":"2025-06-13 21:04:18.000000000","message":"why would a 3-replica ring ever return less than 3 devices for a given part?","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1357,"context_line":"    FORMAT_VERSION \u003d 0"},{"line_number":1358,"context_line":""},{"line_number":1359,"context_line":""},{"line_number":1360,"context_line":"class ExtendedRing(ring.RingData):"},{"line_number":1361,"context_line":"    extra \u003d b\u0027some super-specific data\u0027"},{"line_number":1362,"context_line":""},{"line_number":1363,"context_line":"    def to_dict(self):"}],"source_content_type":"text/x-python","patch_set":38,"id":"87f968c9_ecc4d480","line":1360,"updated":"2025-06-13 21:04:18.000000000","message":"this is \"just\" an \"ExtendedRingData\" - if you want to have a ring actually *use* \"extra\" you have to override `Ring._reload` as well","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":false,"context_lines":[{"line_number":1357,"context_line":"    FORMAT_VERSION \u003d 0"},{"line_number":1358,"context_line":""},{"line_number":1359,"context_line":""},{"line_number":1360,"context_line":"class ExtendedRing(ring.RingData):"},{"line_number":1361,"context_line":"    extra \u003d b\u0027some super-specific data\u0027"},{"line_number":1362,"context_line":""},{"line_number":1363,"context_line":"    def to_dict(self):"}],"source_content_type":"text/x-python","patch_set":38,"id":"6499e80b_a26f60b5","line":1360,"in_reply_to":"7afce17f_96a31e1e","updated":"2025-08-05 07:23:12.000000000","message":"Acknowledged","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":1357,"context_line":"    FORMAT_VERSION \u003d 0"},{"line_number":1358,"context_line":""},{"line_number":1359,"context_line":""},{"line_number":1360,"context_line":"class ExtendedRing(ring.RingData):"},{"line_number":1361,"context_line":"    extra \u003d b\u0027some super-specific data\u0027"},{"line_number":1362,"context_line":""},{"line_number":1363,"context_line":"    def to_dict(self):"}],"source_content_type":"text/x-python","patch_set":38,"id":"f6e09e23_31a6f604","line":1360,"in_reply_to":"87f968c9_ecc4d480","updated":"2025-06-16 23:28:49.000000000","message":"\u003e this is \"just\" an \"ExtendedRingData\"\n\nSure -- can rename.\n\n\u003e if you want to have a ring actually use \"extra\" you have to override `Ring._reload` as well\n\nNot so sure about that -- if you want to *also use `Ring` methods*, sure, but our experience with operators wanting to open up rings in their tooling suggests they don\u0027t typically *need* that, they likely just want to be able to look at the data. *Maybe* if it was written in Python they would\u0027ve liked to be able to use some `get_my_ports(my_ips)` method, but we don\u0027t even have that, it\u0027s all buried down in `all_bind_ports_for_node` and has to look across *all* rings.\n\nMy current assumption (unless and until someone comes up with some \"use this `Ring` class\" entry point patch) is that any extra data they write will be separated (or at least separable) from Swift\u0027s `Ring` functions.","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0f4050d22c2b5882077bd0ae4cd445199e4d1936","unresolved":true,"context_lines":[{"line_number":1357,"context_line":"    FORMAT_VERSION \u003d 0"},{"line_number":1358,"context_line":""},{"line_number":1359,"context_line":""},{"line_number":1360,"context_line":"class ExtendedRing(ring.RingData):"},{"line_number":1361,"context_line":"    extra \u003d b\u0027some super-specific data\u0027"},{"line_number":1362,"context_line":""},{"line_number":1363,"context_line":"    def to_dict(self):"}],"source_content_type":"text/x-python","patch_set":38,"id":"7afce17f_96a31e1e","line":1360,"in_reply_to":"f6e09e23_31a6f604","updated":"2025-06-18 22:21:44.000000000","message":"\u003e any extra data they write will be separated (or at least separable) from Swift\u0027s Ring functions.\n\nthat may actually be reasonable - I\u0027ll have to look at Matt\u0027s last_rebalance patches and think more on that!","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"fd5215c3ea5c584aea2fe59a785b6f2954284cc7","unresolved":true,"context_lines":[{"line_number":1374,"context_line":"    def deserialize_v2(cls, reader, *args, **kwargs):"},{"line_number":1375,"context_line":"        ring_data \u003d super(ExtendedRing, cls).deserialize_v2("},{"line_number":1376,"context_line":"            reader, *args, **kwargs)"},{"line_number":1377,"context_line":"        with reader.open_section(\u0027my-custom-section\u0027) as s:"},{"line_number":1378,"context_line":"            ring_data[\u0027extra\u0027] \u003d s.read()"},{"line_number":1379,"context_line":"        return ring_data"},{"line_number":1380,"context_line":""}],"source_content_type":"text/x-python","patch_set":38,"id":"b38ffa6f_399511ab","line":1377,"updated":"2025-06-13 21:04:18.000000000","message":"I\u0027m not sure I can imagine if a \"real\" attempt at an out-of-tree ExtendedRing would want to be robust to this section not existing?  Some kind of `if \u0027my-custom-section\u0027 in reader` check, or catch the `KeyError`.\n\nMaybe it depends on how much your new tool/script *depends* on the new section/data to do it\u0027s job correctly...","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca776ab21df538fe9b6c2422d5899e544d0f3a2e","unresolved":true,"context_lines":[{"line_number":1412,"context_line":"        self.assertEqual(rd2.devs, d)"},{"line_number":1413,"context_line":"        self.assertEqual(rd2._part_shift, s)"},{"line_number":1414,"context_line":"        self.assertEqual(rd2.extra, b\u0027some other value\u0027)"},{"line_number":1415,"context_line":"        self.assertEqual(rd2.size, bytes_written)"},{"line_number":1416,"context_line":""},{"line_number":1417,"context_line":""},{"line_number":1418,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":38,"id":"dc1ca31e_a8a3b301","line":1415,"updated":"2025-06-16 23:28:49.000000000","message":"Hmm... this should maybe *also* open it up as a normal `Ring`...","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ef4242d4b8cf95ca87cb59725093fd81a19dbfc5","unresolved":false,"context_lines":[{"line_number":1412,"context_line":"        self.assertEqual(rd2.devs, d)"},{"line_number":1413,"context_line":"        self.assertEqual(rd2._part_shift, s)"},{"line_number":1414,"context_line":"        self.assertEqual(rd2.extra, b\u0027some other value\u0027)"},{"line_number":1415,"context_line":"        self.assertEqual(rd2.size, bytes_written)"},{"line_number":1416,"context_line":""},{"line_number":1417,"context_line":""},{"line_number":1418,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":38,"id":"f59c853e_750f14d1","line":1415,"in_reply_to":"dc1ca31e_a8a3b301","updated":"2025-06-17 02:56:47.000000000","message":"Done","commit_id":"b89ee3acd988c29df1ba1ce1542f2843388bb168"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b059cbbc81f92672002003cd3b4c8341774627ab","unresolved":true,"context_lines":[{"line_number":232,"context_line":"        self.assertTrue(loaded_rd[\u0027replica2part2dev_id\u0027])"},{"line_number":233,"context_line":""},{"line_number":234,"context_line":"    def test_deserialize_v2(self):"},{"line_number":235,"context_line":"        # First save it as a ring v1 and then try and load it using"},{"line_number":236,"context_line":"        # deserialize_v2"},{"line_number":237,"context_line":"        ring_fname \u003d os.path.join(self.testdir, \u0027foo.ring.gz\u0027)"},{"line_number":238,"context_line":"        rd \u003d ring.RingData("},{"line_number":239,"context_line":"            [[0, 1, 0, 1], [0, 1, 0, 1]],"}],"source_content_type":"text/x-python","patch_set":42,"id":"37b77bb6_af33883d","line":236,"range":{"start_line":235,"start_character":8,"end_line":236,"end_character":24},"updated":"2025-08-05 07:23:12.000000000","message":"I think this was supposed to be similar to the start of `test_deserialize_v1`, but I didn\u0027t actually do this.","commit_id":"ae062f8b09aed2f7bad9581396607045ea217fa8"}]}