)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f615a9f05a41c0912b8564cb8667cf72d38ba83e","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"utils: Add crc32c function"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"No callers yet, but this will be used by s3api in a later patch."},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Change-Id: Ic0c55e307ce10b56b569c9fee728c445a2300cbd"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"3518c78d_e71154f0","line":9,"updated":"2024-04-02 21:59:35.000000000","message":"ok, good to know.","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"787ada5a0c071bba4383641f49eb4c147340eef9","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"utils: Add crc32c function"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"No callers yet, but this will be used by s3api in a later patch."},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"Change-Id: Ic0c55e307ce10b56b569c9fee728c445a2300cbd"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"c2b39bc7_fbebc95b","line":9,"in_reply_to":"3518c78d_e71154f0","updated":"2024-09-20 23:28:39.000000000","message":"Acknowledged","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":10,"context_line":"in pyeclib\u0027s binary wheels."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"If ISA-L is not available, use kernel sockets for crc32c and disable"},{"line_number":13,"context_line":"crc64nvme. (Though we still provide a reference implementation, it\u0027s"},{"line_number":14,"context_line":"too slow to put in the client path.)"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"No callers yet, but these will be used by s3api in a later patch."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Change-Id: Ic0c55e307ce10b56b569c9fee728c445a2300cbd"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":29,"id":"789a6973_bf91f588","line":15,"range":{"start_line":13,"start_character":11,"end_line":15,"end_character":1},"updated":"2025-01-22 19:00:46.000000000","message":"IDK -- maybe we\u0027ll enable the slow version -- especially if we\u0027re enabling the slow version of crc32c on BSDs.","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ed64e310d051be6b133a118f3aafb0f7ca82e066","unresolved":true,"context_lines":[{"line_number":10,"context_line":"in pyeclib\u0027s binary wheels."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"If ISA-L is not available, use kernel sockets for crc32c and disable"},{"line_number":13,"context_line":"crc64nvme. (Though we still provide a reference implementation, it\u0027s"},{"line_number":14,"context_line":"too slow to put in the client path.)"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"No callers yet, but these will be used by s3api in a later patch."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Change-Id: Ic0c55e307ce10b56b569c9fee728c445a2300cbd"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":29,"id":"9469a5e5_d2d3f413","line":15,"range":{"start_line":13,"start_character":11,"end_line":15,"end_character":1},"in_reply_to":"789a6973_bf91f588","updated":"2025-03-14 01:43:30.000000000","message":"Or -- ditch the reference versions for [`anycrc`](https://pypi.org/project/anycrc/) instead.","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"269e5ab975d275ab8b72cbd0b146918f33ed9dd9","unresolved":false,"context_lines":[{"line_number":10,"context_line":"in pyeclib\u0027s binary wheels."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"If ISA-L is not available, use kernel sockets for crc32c and disable"},{"line_number":13,"context_line":"crc64nvme. (Though we still provide a reference implementation, it\u0027s"},{"line_number":14,"context_line":"too slow to put in the client path.)"},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"No callers yet, but these will be used by s3api in a later patch."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Change-Id: Ic0c55e307ce10b56b569c9fee728c445a2300cbd"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":29,"id":"1585cab5_28a8cee7","line":15,"range":{"start_line":13,"start_character":11,"end_line":15,"end_character":1},"in_reply_to":"9469a5e5_d2d3f413","updated":"2025-04-14 03:56:24.000000000","message":"Done","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1fcc99d490c636276462a2ad34d9d0ccfa4cc847","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     Alistair Coles \u003calistairncoles@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2025-03-11 17:30:51 +0000"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"utils: Add crc32c and crc64nvme functions"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Use ISA-L if we can find it; on py38+ this includes the ISA-L bundled"},{"line_number":10,"context_line":"in pyeclib\u0027s binary wheels."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":34,"id":"3fabae54_f312aa46","line":7,"updated":"2025-03-12 21:02:36.000000000","message":"I suggest we pull the CRCHasher class into this patch from https://review.opendev.org/c/openstack/swift/+/944073 and then provide functions to construct each hasher so the interface is similar to hashlib","commit_id":"f110ffb10f74b0689a3fb3219b44d20055e63890"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eaa896e5e203b5640b52e67b12016d72559c7cdd","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Alistair Coles \u003calistairncoles@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2025-03-11 17:30:51 +0000"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"utils: Add crc32c and crc64nvme functions"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Use ISA-L if we can find it; on py38+ this includes the ISA-L bundled"},{"line_number":10,"context_line":"in pyeclib\u0027s binary wheels."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":34,"id":"00346e92_89100128","line":7,"in_reply_to":"3fabae54_f312aa46","updated":"2025-04-02 14:21:29.000000000","message":"Done","commit_id":"f110ffb10f74b0689a3fb3219b44d20055e63890"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fc4576a904d161596e059e13c3c0e3ed3a7a724f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"4e76a5ed_d4acf701","updated":"2024-03-01 21:27:37.000000000","message":"recheck\n\npy2 failures should be fixed by https://review.opendev.org/c/openstack/swift/+/910753\n\npy3 probe test failure was just another instance of https://bugs.launchpad.net/swift/+bug/2028175","commit_id":"7d882320bbbe2c921d0b5590fe93ad1f1f5871c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4e6accceac5851334d8f5cac9da9339f183dd06b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"f535306a_1ec7a5d6","updated":"2024-04-24 19:18:30.000000000","message":"This shouldn\u0027t merge until we\u0027ve got consensus that https://review.opendev.org/c/openstack/swift/+/909801 / https://review.opendev.org/c/openstack/swift/+/909802 are ready to merge.\n\nReview/feedback is welcome, though!","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f615a9f05a41c0912b8564cb8667cf72d38ba83e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"b707dd73_ba3288f9","updated":"2024-04-02 21:59:35.000000000","message":"on my vsaio I was able to clone https://github.com/intel/isa-l and run `reec` - afterwards I got the isa-l tests:\n\nBefore:\n\n\tvagrant@saio:~$ pytest swift/test/unit/common/utils/test_checksum.py \n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d test session starts \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\tplatform linux -- Python 3.10.12, pytest-7.4.3, pluggy-1.3.0 -- /usr/bin/python\n\t...\n\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_equivalence PASSED                                                                                                     [ 25%]\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_isal SKIPPED (No ISA-L CRC32C)                                                                                         [ 50%]\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_kern PASSED                                                                                                            [ 75%]\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_ref PASSED                                                                                                             [100%]\n\n\t...\n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d 3 passed, 1 skipped, 1 warning in 1.04s \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n\n\tvagrant@saio:~$ ./.tox-swift/py27/bin/python -m pytest swift/test/unit/common/utils/test_checksum.py\n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d test session starts \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\tplatform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.11.0, pluggy-0.13.1 -- /home/vagrant/.tox-swift/py27/bin/python\n\t...\n\n\tswift/::TestCRC32C::test_equivalence PASSED                                                                                                                                            [ 25%]\n\tswift/::TestCRC32C::test_isal SKIPPED                                                                                                                                                  [ 50%]\n\tswift/::TestCRC32C::test_kern SKIPPED                                                                                                                                                  [ 75%]\n\tswift/::TestCRC32C::test_ref PASSED                                                                                                                                                    [100%]\n\n\t...\n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d 2 passed, 2 skipped, 1 warnings in 1.58 seconds \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nAfter:\n\n\tvagrant@saio:~$ pytest swift/test/unit/common/utils/test_checksum.py \n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d test session starts \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\tplatform linux -- Python 3.10.12, pytest-7.4.3, pluggy-1.3.0 -- /usr/bin/python\n\t...\n\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_equivalence PASSED                                                                                                     [ 25%]\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_isal PASSED                                                                                                            [ 50%]\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_kern PASSED                                                                                                            [ 75%]\n\tswift/test/unit/common/utils/test_checksum.py::TestCRC32C::test_ref PASSED                                                                                                             [100%]\n\n\t...\n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d 4 passed, 1 warning in 0.86s \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\n\n\tvagrant@saio:~$ ./.tox-swift/py27/bin/python -m pytest swift/test/unit/common/utils/test_checksum.py\n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d test session starts \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\tplatform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.11.0, pluggy-0.13.1 -- /home/vagrant/.tox-swift/py27/bin/python\n\t...\n\n\tswift/::TestCRC32C::test_equivalence PASSED                                                                                                                                            [ 25%]\n\tswift/::TestCRC32C::test_isal PASSED                                                                                                                                                   [ 50%]\n\tswift/::TestCRC32C::test_kern SKIPPED                                                                                                                                                  [ 75%]\n\tswift/::TestCRC32C::test_ref PASSED                                                                                                                                                    [100%]\n\n\t...\n\t\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d 3 passed, 1 skipped, 1 warnings in 1.23 seconds \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nN.B. I had to update my reec script to get the latest isa-l code to make correctly:\n\nhttps://github.com/NVIDIA/vagrant-swift-all-in-one/pull/159","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f8000c341131ffa5ef35cb96be3b67540effd9f1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"558a6818_2ac4a4c9","updated":"2024-09-18 06:11:03.000000000","message":"Might be interesting (not in this patch) to extend the checksum namespace to include xxhash support that I think drop down and use CPU instructions and the like.. then maybe we can one day extend our etags to have a prefix (\u003calgorithm\u003e:hash) and slowly move away from md5, esp if they\u0027re causing us grief elsewhere.\n\nBasically liking the idea of having swift.common.utils.checksum","commit_id":"d402e93058c9bcdaec41343899f58db5764f38ed"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"138da2a34af297c11b395532ee5b1dd6661572a9","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":19,"id":"865d7f25_eb789009","updated":"2024-09-23 16:20:38.000000000","message":"Looks like i should be reviewing this one first since it seems to be the prelim req for https://review.opendev.org/c/openstack/swift/+/909801/25","commit_id":"e9b65fdd12f8cda00f483571302085b65d335259"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"20e5392a9e8a3005132b40f37759fabd76a4cba8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":19,"id":"0e5c8ea3_26c1a740","in_reply_to":"865d7f25_eb789009","updated":"2024-10-10 15:34:44.000000000","message":"Done","commit_id":"e9b65fdd12f8cda00f483571302085b65d335259"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"a20f19887658acb8fec6ca58e9c8dca97fcbea38","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"618ed1cb_28d8cd77","updated":"2024-10-11 12:28:35.000000000","message":"recheck","commit_id":"d85889d88387df6a2af4595917842c556dbe2e26"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"6e9adfcf6a1c0a90d226a4b1dd93d813094c0425","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"73ea6a23_24421885","updated":"2024-10-10 15:16:14.000000000","message":"recheck","commit_id":"d85889d88387df6a2af4595917842c556dbe2e26"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"83364fa6bde564333eb02d34cf4569d2e0c94d75","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"b7c9d513_615bf3ed","updated":"2024-12-17 00:52:18.000000000","message":"Loving this, like the use of the kernel sockets and even a fall back I hope we never have to every really use... but nice to have something.\n\nAnd man that isa-l flies!","commit_id":"84e9b6160a0fe04759fb91c358de057c75d3fbf9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b36fefcf7e6c2b006a9c35e85f1903453d0023f3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":26,"id":"cba3bd3e_d0931f1a","updated":"2024-12-16 23:22:01.000000000","message":"Next two patches have some positive review; dropping the -2.","commit_id":"84e9b6160a0fe04759fb91c358de057c75d3fbf9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"5c13b41613673d0896448c369c35384818494ded","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"7aa2f855_d347576d","updated":"2025-01-14 18:09:39.000000000","message":"Good module and performance test results as well. It should be able to drop the usages of six, since upstream code only support py3 now. I also wonder whether this module still needs the python native implementation ``crc32c_ref``, since it\u0027s available on system with python \u003e\u003d 3.6.","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":29,"id":"3ef2b6e8_806114ae","updated":"2025-01-20 17:23:55.000000000","message":"Can we do better for isa-l and py37 re manylinux wheel installation?\n\nThe assertions re order of preference rely on test envs (not) having isa-l, but I suspect all out CI jobs do have isa-l. Certainly one of the assertions is wrong (re falling back to the ref impl) but is never executed in our CI.\n\n\nI wonder if we could fallback to a ref impl for macos devs, or at least allow crc32c_kern to be None, just so the tests pass? I can install isa-l on macos but I can\u0027t modify the kernel ;-)","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"79d48d47a0497ac4a329a9a4741f4dec3e79a134","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":37,"id":"af14ab94_5a20236b","updated":"2025-04-02 09:52:06.000000000","message":"Here is a rough attempt at a kernel crc64nvme implementation, but because it doesn\u0027t support the setopt it\u0027s done a little different.","commit_id":"ef29bf933a898ed6e012c35c42794b77de42e1cf"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7730e5acf4a222ac46e2d33ea191ebd0e565294b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":37,"id":"7dc68fdf_8f49f473","updated":"2025-04-02 06:36:04.000000000","message":"Looking great to me. Tested all 3 implementations of crc32c on my linux box. Working well.\n\nI guess there not being an NVME kernel fallback is a pain. Although there was a socket ATG one wasn\u0027t there, just the ability to send the previous output when setting up a new socket didn\u0027t work, so couldn\u0027t use the same interface like the others. Instead it\u0027ll only work if you used a socket and kept sending data through it and got the crc at the end (so not like the hashlib functions work). But getting that working could be an interesting alternative to falling back to the slow ref implementation?\n\nIe the NVME socket hasher class would need to create a socket, keep it and keep reusing it, not create a new socket with a start value everytime.","commit_id":"ef29bf933a898ed6e012c35c42794b77de42e1cf"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"97c7ba74ce6bbb36aa165988f9331802b773fb47","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":37,"id":"b2c9f7f9_b49f6a5f","in_reply_to":"af14ab94_5a20236b","updated":"2025-04-02 22:07:32.000000000","message":"https://review.opendev.org/c/openstack/swift/+/946125 (opps forgot the link)","commit_id":"ef29bf933a898ed6e012c35c42794b77de42e1cf"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f0607f1587d7f833ef7e7a7e5419de95712389d2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":53,"id":"169bd7c3_aaccc9fa","updated":"2025-05-12 09:21:40.000000000","message":"Working great, nice work!","commit_id":"6fb9fc71b26b2d0ad146f48bdca240533345d371"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0bd7667e400ce010a0b45b56518503f83b5b4774","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":56,"id":"a7e44ad8_d030e271","updated":"2025-05-14 04:16:37.000000000","message":"Now that the parent is +Aed, let\u0027s land this one too!","commit_id":"b65855d7153cde393752c28c2516eeeb7d1a10e1"}],"swift/common/utils/checksum.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fc4576a904d161596e059e13c3c0e3ed3a7a724f","unresolved":true,"context_lines":[{"line_number":37,"context_line":"                rem \u003d (rem \u003e\u003e 1) ^ p"},{"line_number":38,"context_line":"            else:"},{"line_number":39,"context_line":"                rem \u003d (rem \u003e\u003e 1)"},{"line_number":40,"context_line":"    return rem ^ 0xffffffff"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"# If isal is available system-wide, great!"}],"source_content_type":"text/x-python","patch_set":3,"id":"b26a0cf8_6f470d11","line":40,"updated":"2024-03-01 21:27:37.000000000","message":"I\u0027ve got another pure-python version kicking around that operates per-byte instead of per-bit; it uses arrays and looks a lot more like https://opendev.org/openstack/liberasurecode/src/branch/master/src/utils/chksum/crc32.c -- but it only got me ~6x improvement (not even the full 8x you might hope for) and so was still orders of magnitude worse than the kernel socket approach.\n\nKeeping this one for the simplicity and so we have a fallback for py2. Longer term, I see two ways forward:\n\n1. We get rid of it when we rip out py2, and maybe push harder toward having ISA-L installed.\n2. We look at optimizing it with cython or something, and see if we can get closer to the kernel-implementation speeds.","commit_id":"b6a772dc47f1fb9151f7322a3878b4fcbc917034"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d96953e4cc0bb3dd848d2745a9ede6dc31040012","unresolved":true,"context_lines":[{"line_number":37,"context_line":"                rem \u003d (rem \u003e\u003e 1) ^ p"},{"line_number":38,"context_line":"            else:"},{"line_number":39,"context_line":"                rem \u003d (rem \u003e\u003e 1)"},{"line_number":40,"context_line":"    return rem ^ 0xffffffff"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"# If isal is available system-wide, great!"}],"source_content_type":"text/x-python","patch_set":3,"id":"cf5a8553_04557f16","line":40,"in_reply_to":"5637064f_b30cd511","updated":"2025-03-13 04:31:57.000000000","message":"Looks like we\u0027re not getting rid of it (in case the kernel implementation isn\u0027t available, for example on OS X), but will probably push harder to have ISA-L installed.","commit_id":"b6a772dc47f1fb9151f7322a3878b4fcbc917034"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7b73473b9a5da64bfc72d9b34892931e532588af","unresolved":true,"context_lines":[{"line_number":37,"context_line":"                rem \u003d (rem \u003e\u003e 1) ^ p"},{"line_number":38,"context_line":"            else:"},{"line_number":39,"context_line":"                rem \u003d (rem \u003e\u003e 1)"},{"line_number":40,"context_line":"    return rem ^ 0xffffffff"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"# If isal is available system-wide, great!"}],"source_content_type":"text/x-python","patch_set":3,"id":"5637064f_b30cd511","line":40,"in_reply_to":"b26a0cf8_6f470d11","updated":"2024-10-03 21:47:50.000000000","message":"Nice, this is very helpful, thanks","commit_id":"b6a772dc47f1fb9151f7322a3878b4fcbc917034"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4cdd79fb2d5a7c7be20b20238842621bec62a078","unresolved":false,"context_lines":[{"line_number":37,"context_line":"                rem \u003d (rem \u003e\u003e 1) ^ p"},{"line_number":38,"context_line":"            else:"},{"line_number":39,"context_line":"                rem \u003d (rem \u003e\u003e 1)"},{"line_number":40,"context_line":"    return rem ^ 0xffffffff"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"# If isal is available system-wide, great!"}],"source_content_type":"text/x-python","patch_set":3,"id":"573c7a7d_eb1eacc3","line":40,"in_reply_to":"cf5a8553_04557f16","updated":"2025-03-14 11:38:16.000000000","message":"Done","commit_id":"b6a772dc47f1fb9151f7322a3878b4fcbc917034"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"fc4576a904d161596e059e13c3c0e3ed3a7a724f","unresolved":true,"context_lines":[{"line_number":91,"context_line":"# ---------------+----------+----------+---------"},{"line_number":92,"context_line":"# Intel N100     | ~9GB/s   | ~3.5GB/s | ~1.1MB/s"},{"line_number":93,"context_line":"# ARM Cortex-A55 | ~2.5GB/s | ~0.4GB/s | ~0.2MB/s"},{"line_number":94,"context_line":"# Intel 11850H   | ~7GB/s   | ~2.6GB/s | ~1.5MB/s"},{"line_number":95,"context_line":"#"},{"line_number":96,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":97,"context_line":"# which is still \u003e1000x faster than a naive python implementation."}],"source_content_type":"text/x-python","patch_set":3,"id":"4c51f79e_ce2788c2","line":94,"updated":"2024-03-01 21:27:37.000000000","message":"As I recall, these were given by timing a single CRC32C calculation for the same random 10MiB block with the different implementations.\n\nIt\u0027ll be interesting to see how these speeds compare to actual end-to-end tests. In particular, I seem to recall Jian expressing some concern about crossing that kernel barrier too often; it might be the sort of thing that\u0027s tolerable in a single process/single thread with a large chunk size, but terrible when you have many workers trying to handle hundreds of requests per second.\n\nI think as long as we\u0027re within an order of magnitude of these kernel speeds, though, this is unlikely to be a significant bottleneck.","commit_id":"b6a772dc47f1fb9151f7322a3878b4fcbc917034"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"c61fc102d7adfb94506a78b86fefb44d1d783ae8","unresolved":false,"context_lines":[{"line_number":91,"context_line":"# ---------------+----------+----------+---------"},{"line_number":92,"context_line":"# Intel N100     | ~9GB/s   | ~3.5GB/s | ~1.1MB/s"},{"line_number":93,"context_line":"# ARM Cortex-A55 | ~2.5GB/s | ~0.4GB/s | ~0.2MB/s"},{"line_number":94,"context_line":"# Intel 11850H   | ~7GB/s   | ~2.6GB/s | ~1.5MB/s"},{"line_number":95,"context_line":"#"},{"line_number":96,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":97,"context_line":"# which is still \u003e1000x faster than a naive python implementation."}],"source_content_type":"text/x-python","patch_set":3,"id":"1f183bbc_2f06aacf","line":94,"in_reply_to":"22da0171_5e3e8b26","updated":"2024-10-10 15:33:52.000000000","message":"Awesome!","commit_id":"b6a772dc47f1fb9151f7322a3878b4fcbc917034"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a6f877ae2d32340fa56f7a4cf83fec5b8a9cb3c1","unresolved":true,"context_lines":[{"line_number":91,"context_line":"# ---------------+----------+----------+---------"},{"line_number":92,"context_line":"# Intel N100     | ~9GB/s   | ~3.5GB/s | ~1.1MB/s"},{"line_number":93,"context_line":"# ARM Cortex-A55 | ~2.5GB/s | ~0.4GB/s | ~0.2MB/s"},{"line_number":94,"context_line":"# Intel 11850H   | ~7GB/s   | ~2.6GB/s | ~1.5MB/s"},{"line_number":95,"context_line":"#"},{"line_number":96,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":97,"context_line":"# which is still \u003e1000x faster than a naive python implementation."}],"source_content_type":"text/x-python","patch_set":3,"id":"22da0171_5e3e8b26","line":94,"in_reply_to":"4c51f79e_ce2788c2","updated":"2024-10-04 19:54:08.000000000","message":"I think I found the script I was using! https://paste.opendev.org/show/bQOcoiCQUvYDwfxpMN5C/\n\nIt\u0027s even got the lookup table for the optimized pure-python version!","commit_id":"b6a772dc47f1fb9151f7322a3878b4fcbc917034"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f615a9f05a41c0912b8564cb8667cf72d38ba83e","unresolved":true,"context_lines":[{"line_number":100,"context_line":"if crc32c_isal:"},{"line_number":101,"context_line":"    crc32c \u003d crc32c_isal"},{"line_number":102,"context_line":"elif crc32c_kern:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":104,"context_line":"else:"},{"line_number":105,"context_line":"    # Well, something\u0027s better than nothing, I suppose..."},{"line_number":106,"context_line":"    crc32c \u003d crc32c_ref"}],"source_content_type":"text/x-python","patch_set":7,"id":"c26d7246_1b53caca","line":103,"updated":"2024-04-02 21:59:35.000000000","message":"i guess my vsaio doesn\u0027t have isal installed\n\n\t\u003e\u003e\u003e from swift.common.utils import checksum\n\t\u003e\u003e\u003e checksum.crc32c\n\t\u003cfunction crc32c_kern at 0x7f86164ce8c0\u003e","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7e544dce082d616c1eba497e3b76ef30563cbf01","unresolved":false,"context_lines":[{"line_number":100,"context_line":"if crc32c_isal:"},{"line_number":101,"context_line":"    crc32c \u003d crc32c_isal"},{"line_number":102,"context_line":"elif crc32c_kern:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":104,"context_line":"else:"},{"line_number":105,"context_line":"    # Well, something\u0027s better than nothing, I suppose..."},{"line_number":106,"context_line":"    crc32c \u003d crc32c_ref"}],"source_content_type":"text/x-python","patch_set":7,"id":"2c2c8b43_8786ad2a","line":103,"in_reply_to":"2fb6c945_03672c8e","updated":"2024-10-25 20:27:24.000000000","message":"Done","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f8000c341131ffa5ef35cb96be3b67540effd9f1","unresolved":true,"context_lines":[{"line_number":100,"context_line":"if crc32c_isal:"},{"line_number":101,"context_line":"    crc32c \u003d crc32c_isal"},{"line_number":102,"context_line":"elif crc32c_kern:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":104,"context_line":"else:"},{"line_number":105,"context_line":"    # Well, something\u0027s better than nothing, I suppose..."},{"line_number":106,"context_line":"    crc32c \u003d crc32c_ref"}],"source_content_type":"text/x-python","patch_set":7,"id":"bbc0ba63_89e4a7d1","line":103,"in_reply_to":"59e18a72_88cedce0","updated":"2024-09-18 06:11:03.000000000","message":"yay, just want I need another rabbit hole :P\nMan really need to remember about pyeclib","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"c61fc102d7adfb94506a78b86fefb44d1d783ae8","unresolved":true,"context_lines":[{"line_number":100,"context_line":"if crc32c_isal:"},{"line_number":101,"context_line":"    crc32c \u003d crc32c_isal"},{"line_number":102,"context_line":"elif crc32c_kern:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":104,"context_line":"else:"},{"line_number":105,"context_line":"    # Well, something\u0027s better than nothing, I suppose..."},{"line_number":106,"context_line":"    crc32c \u003d crc32c_ref"}],"source_content_type":"text/x-python","patch_set":7,"id":"2fb6c945_03672c8e","line":103,"in_reply_to":"8bafb81f_19574d43","updated":"2024-10-10 15:33:52.000000000","message":"I was able to verify the installation using `apt install` might also be nice to try downloading the `aarch64` wheels and evaluate them against the `x86\n_64` wheels on a perf. basis","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1cadf608b099e647f7786642f0433c613aa2ad7a","unresolved":true,"context_lines":[{"line_number":100,"context_line":"if crc32c_isal:"},{"line_number":101,"context_line":"    crc32c \u003d crc32c_isal"},{"line_number":102,"context_line":"elif crc32c_kern:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":104,"context_line":"else:"},{"line_number":105,"context_line":"    # Well, something\u0027s better than nothing, I suppose..."},{"line_number":106,"context_line":"    crc32c \u003d crc32c_ref"}],"source_content_type":"text/x-python","patch_set":7,"id":"8bafb81f_19574d43","line":103,"in_reply_to":"bbc0ba63_89e4a7d1","updated":"2024-09-27 22:45:06.000000000","message":"Alternatively, just install ISA-L. On an ubuntu vsaio, it\u0027s as simple as `sudo apt install isal`","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f08d5a635fbd45bdc19b71b37172c6ed882d5111","unresolved":true,"context_lines":[{"line_number":100,"context_line":"if crc32c_isal:"},{"line_number":101,"context_line":"    crc32c \u003d crc32c_isal"},{"line_number":102,"context_line":"elif crc32c_kern:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":104,"context_line":"else:"},{"line_number":105,"context_line":"    # Well, something\u0027s better than nothing, I suppose..."},{"line_number":106,"context_line":"    crc32c \u003d crc32c_ref"}],"source_content_type":"text/x-python","patch_set":7,"id":"59e18a72_88cedce0","line":103,"in_reply_to":"c26d7246_1b53caca","updated":"2024-09-12 01:22:42.000000000","message":"FWIW, this should play well with the wheels we\u0027d start publishing with https://review.opendev.org/c/openstack/pyeclib/+/927654\n\nCan even preview them:\n- [x86_64](https://zuul.opendev.org/t/openstack/build/8bcf07673775404b89bfb8c357e9ac38/artifacts)\n- [aarch64](https://zuul.opendev.org/t/openstack/build/bee28774299c4de8a7c4f47dbca502a7/artifacts)","commit_id":"7ed859bdfcb0809e323ddef281f572eec5aa6d5d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f08d5a635fbd45bdc19b71b37172c6ed882d5111","unresolved":true,"context_lines":[{"line_number":46,"context_line":"if isal_lib is None and pkg_files is not None:"},{"line_number":47,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":48,"context_line":"    # with isal baked in?"},{"line_number":49,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027) if \u0027isal\u0027 in str(f)]"},{"line_number":50,"context_line":"    if len(isal_libs) \u003d\u003d 1:"},{"line_number":51,"context_line":"        isal_lib \u003d isal_libs[0].locate()"},{"line_number":52,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"77df6401_135dea65","line":49,"range":{"start_line":49,"start_character":52,"end_line":49,"end_character":68},"updated":"2024-09-12 01:22:42.000000000","message":"Probably better as `f.name.startswith(\"libisal\")`","commit_id":"d402e93058c9bcdaec41343899f58db5764f38ed"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"787ada5a0c071bba4383641f49eb4c147340eef9","unresolved":false,"context_lines":[{"line_number":46,"context_line":"if isal_lib is None and pkg_files is not None:"},{"line_number":47,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":48,"context_line":"    # with isal baked in?"},{"line_number":49,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027) if \u0027isal\u0027 in str(f)]"},{"line_number":50,"context_line":"    if len(isal_libs) \u003d\u003d 1:"},{"line_number":51,"context_line":"        isal_lib \u003d isal_libs[0].locate()"},{"line_number":52,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"a67bc3e9_6d7862df","line":49,"range":{"start_line":49,"start_character":52,"end_line":49,"end_character":68},"in_reply_to":"1b86f8d0_00c4b5cc","updated":"2024-09-20 23:28:39.000000000","message":"Done","commit_id":"d402e93058c9bcdaec41343899f58db5764f38ed"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f8000c341131ffa5ef35cb96be3b67540effd9f1","unresolved":true,"context_lines":[{"line_number":46,"context_line":"if isal_lib is None and pkg_files is not None:"},{"line_number":47,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":48,"context_line":"    # with isal baked in?"},{"line_number":49,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027) if \u0027isal\u0027 in str(f)]"},{"line_number":50,"context_line":"    if len(isal_libs) \u003d\u003d 1:"},{"line_number":51,"context_line":"        isal_lib \u003d isal_libs[0].locate()"},{"line_number":52,"context_line":""}],"source_content_type":"text/x-python","patch_set":17,"id":"1b86f8d0_00c4b5cc","line":49,"range":{"start_line":49,"start_character":52,"end_line":49,"end_character":68},"in_reply_to":"77df6401_135dea65","updated":"2024-09-18 06:11:03.000000000","message":"startswith does sound better","commit_id":"d402e93058c9bcdaec41343899f58db5764f38ed"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7b73473b9a5da64bfc72d9b34892931e532588af","unresolved":true,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"import ctypes"},{"line_number":17,"context_line":"import ctypes.util"},{"line_number":18,"context_line":"import six"},{"line_number":19,"context_line":"import socket"},{"line_number":20,"context_line":"import struct"},{"line_number":21,"context_line":""}],"source_content_type":"text/x-python","patch_set":21,"id":"ca2920bf_475fa7b4","line":18,"updated":"2024-10-03 21:47:50.000000000","message":"ahh py2 strikes again.","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"c61fc102d7adfb94506a78b86fefb44d1d783ae8","unresolved":false,"context_lines":[{"line_number":15,"context_line":""},{"line_number":16,"context_line":"import ctypes"},{"line_number":17,"context_line":"import ctypes.util"},{"line_number":18,"context_line":"import six"},{"line_number":19,"context_line":"import socket"},{"line_number":20,"context_line":"import struct"},{"line_number":21,"context_line":""}],"source_content_type":"text/x-python","patch_set":21,"id":"b8761d80_04ba1b92","line":18,"in_reply_to":"ca2920bf_475fa7b4","updated":"2024-10-10 15:33:52.000000000","message":"Acknowledged","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7b73473b9a5da64bfc72d9b34892931e532588af","unresolved":true,"context_lines":[{"line_number":20,"context_line":"import struct"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"try:"},{"line_number":23,"context_line":"    import pyeclib  # noqa"},{"line_number":24,"context_line":"    from importlib.metadata import files as pkg_files  # py38+"},{"line_number":25,"context_line":"except ImportError:"},{"line_number":26,"context_line":"    pkg_files \u003d None"}],"source_content_type":"text/x-python","patch_set":21,"id":"10d9c56e_0aa0fcaa","line":23,"updated":"2024-10-03 21:47:50.000000000","message":"The new linuxwheels getting built ensure forward compatibility with rocky8, ref: https://review.opendev.org/c/openstack/pyeclib/+/817498/19","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7e544dce082d616c1eba497e3b76ef30563cbf01","unresolved":false,"context_lines":[{"line_number":20,"context_line":"import struct"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"try:"},{"line_number":23,"context_line":"    import pyeclib  # noqa"},{"line_number":24,"context_line":"    from importlib.metadata import files as pkg_files  # py38+"},{"line_number":25,"context_line":"except ImportError:"},{"line_number":26,"context_line":"    pkg_files \u003d None"}],"source_content_type":"text/x-python","patch_set":21,"id":"0482cf43_5cd34a5f","line":23,"in_reply_to":"10d9c56e_0aa0fcaa","updated":"2024-10-25 20:27:24.000000000","message":"Acknowledged","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7b73473b9a5da64bfc72d9b34892931e532588af","unresolved":true,"context_lines":[{"line_number":26,"context_line":"    pkg_files \u003d None"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":30,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":31,"context_line":"    # reference implementation"},{"line_number":32,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":21,"id":"4b0f5d48_1a65cdf0","line":29,"updated":"2024-10-03 21:47:50.000000000","message":"It seems we have three implementations of the CRC32C checksum algorithm\n\n- crc32c_ref:  which is a naive python implementation heavily influenced by isa_l. \n\n- crc32c_isal: which is an implementation we have using the isa-L library\n\n- crc32c_kern: which is an implementation using kernel sockets.\n\nIt might be nice to see to see if i can come up with a small benchmarking script to evaluate the different times we get for the above functions for a prod-like environment. My understanding is that we use `isa_l_rs_cauchy` in our large clusters and since isa-l is optimized for performance","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"788f60320aa9004471d87d3bdf1a3c8adcb494d5","unresolved":false,"context_lines":[{"line_number":26,"context_line":"    pkg_files \u003d None"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":30,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":31,"context_line":"    # reference implementation"},{"line_number":32,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":21,"id":"2001b7b8_bfb36927","line":29,"in_reply_to":"46015bd2_56fc407f","updated":"2024-10-10 21:14:53.000000000","message":"I was able to run your [original script](https://paste.opendev.org/show/bQOcoiCQUvYDwfxpMN5C/) and obtain results:\n\n```\npython checksum_eval.py\nisal: 2513.1MB/s\nref:  1.6MB/s\nmemo: 11.8MB/s\nkern: 3015.3MB/s\nzlib: 1120.9MB/s\nkern: 123.8MB/s\n[\u0027D75B6858\u0027, \u0027D75B6858\u0027, \u0027D75B6858\u0027, \u0027D75B6858\u0027, \u00270560D5DE\u0027, \u00270560D5DE\u0027]\n```","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a6f877ae2d32340fa56f7a4cf83fec5b8a9cb3c1","unresolved":true,"context_lines":[{"line_number":26,"context_line":"    pkg_files \u003d None"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":30,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":31,"context_line":"    # reference implementation"},{"line_number":32,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":21,"id":"878d0e0a_f84060ef","line":29,"in_reply_to":"4b0f5d48_1a65cdf0","updated":"2024-10-04 19:54:08.000000000","message":"FWIW, I did some amount of benchmarking (see table at L92) -- but yeah, we should definitely do something similar on some of our proper hardware instead of my dinky personal dev lab.\n\nAnd yeah, since we (and presumably others) have ISA-L around already for erasure-coding, I figured we could leverage it for CRCs, too.","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"c61fc102d7adfb94506a78b86fefb44d1d783ae8","unresolved":false,"context_lines":[{"line_number":26,"context_line":"    pkg_files \u003d None"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":30,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":31,"context_line":"    # reference implementation"},{"line_number":32,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":21,"id":"46015bd2_56fc407f","line":29,"in_reply_to":"878d0e0a_f84060ef","updated":"2024-10-10 15:33:52.000000000","message":"Nice, i was curious to see which benchmarking tool was used? I would love to learn the tools out there that are capable of evaluating these permutations of crc32c checksums\n\nedit: i think you answered this question in another thread.","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7b73473b9a5da64bfc72d9b34892931e532588af","unresolved":true,"context_lines":[{"line_number":47,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":48,"context_line":"    # with isal baked in?"},{"line_number":49,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027)"},{"line_number":50,"context_line":"                 if f.name.startswith(\"libisal\")]"},{"line_number":51,"context_line":"    if len(isal_libs) \u003d\u003d 1:"},{"line_number":52,"context_line":"        isal_lib \u003d isal_libs[0].locate()"},{"line_number":53,"context_line":""}],"source_content_type":"text/x-python","patch_set":21,"id":"1efd3f9a_3211ed3d","line":50,"updated":"2024-10-03 21:47:50.000000000","message":"Do we need to be robust and functional and have a fallback, preferably the kernel method even if `libisal` is not available since we already error handle for missing libraries in L46 ?","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a6f877ae2d32340fa56f7a4cf83fec5b8a9cb3c1","unresolved":true,"context_lines":[{"line_number":47,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":48,"context_line":"    # with isal baked in?"},{"line_number":49,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027)"},{"line_number":50,"context_line":"                 if f.name.startswith(\"libisal\")]"},{"line_number":51,"context_line":"    if len(isal_libs) \u003d\u003d 1:"},{"line_number":52,"context_line":"        isal_lib \u003d isal_libs[0].locate()"},{"line_number":53,"context_line":""}],"source_content_type":"text/x-python","patch_set":21,"id":"498879ee_df3fcaa1","line":50,"in_reply_to":"1efd3f9a_3211ed3d","updated":"2024-10-04 19:54:08.000000000","message":"Yes -- all the \"select the best available implementation\" stuff happens at the end:\n```\nif crc32c_isal:\n    crc32c \u003d crc32c_isal\nelif crc32c_kern:\n    crc32c \u003d crc32c_kern\nelse:\n    # Well, something\u0027s better than nothing, I suppose...\n    crc32c \u003d crc32c_ref\n```","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"c61fc102d7adfb94506a78b86fefb44d1d783ae8","unresolved":false,"context_lines":[{"line_number":47,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":48,"context_line":"    # with isal baked in?"},{"line_number":49,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027)"},{"line_number":50,"context_line":"                 if f.name.startswith(\"libisal\")]"},{"line_number":51,"context_line":"    if len(isal_libs) \u003d\u003d 1:"},{"line_number":52,"context_line":"        isal_lib \u003d isal_libs[0].locate()"},{"line_number":53,"context_line":""}],"source_content_type":"text/x-python","patch_set":21,"id":"cab3a6e6_eef848b8","line":50,"in_reply_to":"498879ee_df3fcaa1","updated":"2024-10-10 15:33:52.000000000","message":"Done, we fallback to the naive checksum calc but what other choice do we have after all and its at the bottom of the hierarchy, i like that!","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7b73473b9a5da64bfc72d9b34892931e532588af","unresolved":true,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"if isal_lib:"},{"line_number":55,"context_line":"    isal \u003d ctypes.CDLL(isal_lib)"},{"line_number":56,"context_line":"    isal.crc32_iscsi.argtypes \u003d [ctypes.c_char_p, ctypes.c_int, ctypes.c_uint]"},{"line_number":57,"context_line":"    isal.crc32_iscsi.restype \u003d ctypes.c_uint"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    def crc32c_isal(data, seed\u003d0):"}],"source_content_type":"text/x-python","patch_set":21,"id":"5aee2099_201716f2","line":56,"updated":"2024-10-03 21:47:50.000000000","message":"Should we have an extra var here for isa_l_rsa_cauchy? Not very familiar with pyeclib but its clear to me that the EC algorithm will be available to us when we install those new wheels using `reec`, maybe all we need is an additional cross-compat test ?","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a6f877ae2d32340fa56f7a4cf83fec5b8a9cb3c1","unresolved":true,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"if isal_lib:"},{"line_number":55,"context_line":"    isal \u003d ctypes.CDLL(isal_lib)"},{"line_number":56,"context_line":"    isal.crc32_iscsi.argtypes \u003d [ctypes.c_char_p, ctypes.c_int, ctypes.c_uint]"},{"line_number":57,"context_line":"    isal.crc32_iscsi.restype \u003d ctypes.c_uint"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    def crc32c_isal(data, seed\u003d0):"}],"source_content_type":"text/x-python","patch_set":21,"id":"98741b82_6c873f21","line":56,"in_reply_to":"5aee2099_201716f2","updated":"2024-10-04 19:54:08.000000000","message":"So `isa_l_rs_cauchy` and `isa_l_rs_vand` are different erasure coding algorithms (well, matrices, IIRC?) -- they\u0027re entirely separate from the crc32 bits I want to use here.\n\nFWIW, ISA-L *also* includes some zlib/gzip optimizations, too, which we\u0027ve not used. While [there are some python bindings](https://pypi.org/project/isal/) for the zlib/gzip stuff, and pyeclib provides bindings for the erasure-coding stuff, I\u0027m not aware of any other python bindings for just the crc32 stuff. Even if there were, though, I\u0027m not sure I\u0027d use it -- this is remarkably tame as far as `ctypes` usage goes. Much better than [the last time](https://review.opendev.org/c/openstack/swift/+/873222/8/swift/common/utils/ipaddrs.py) I dipped down there.","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"c61fc102d7adfb94506a78b86fefb44d1d783ae8","unresolved":false,"context_lines":[{"line_number":53,"context_line":""},{"line_number":54,"context_line":"if isal_lib:"},{"line_number":55,"context_line":"    isal \u003d ctypes.CDLL(isal_lib)"},{"line_number":56,"context_line":"    isal.crc32_iscsi.argtypes \u003d [ctypes.c_char_p, ctypes.c_int, ctypes.c_uint]"},{"line_number":57,"context_line":"    isal.crc32_iscsi.restype \u003d ctypes.c_uint"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"    def crc32c_isal(data, seed\u003d0):"}],"source_content_type":"text/x-python","patch_set":21,"id":"db179c3c_38ff6e2a","line":56,"in_reply_to":"98741b82_6c873f21","updated":"2024-10-10 15:33:52.000000000","message":"Ack, thanks for **educating** me on this. I was already under the impression that they are separate EC algos, i guess i was emphasizing the use of cauchy over vand but you just answered that question for me since it all falls under the same isa_l library (which is just another implementation of Reed-Solomon)","commit_id":"e191d3fad730c278d3d514c24fad03ff890527cd"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e82155644f75ae6196f364d4eb54022a1e03c23e","unresolved":true,"context_lines":[{"line_number":93,"context_line":"# ---------------+----------+----------+---------"},{"line_number":94,"context_line":"# Intel N100     | ~9GB/s   | ~3.5GB/s | ~1.1MB/s"},{"line_number":95,"context_line":"# ARM Cortex-A55 | ~2.5GB/s | ~0.4GB/s | ~0.2MB/s"},{"line_number":96,"context_line":"# Intel 11850H   | ~7GB/s   | ~2.6GB/s | ~1.5MB/s"},{"line_number":97,"context_line":"#"},{"line_number":98,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":99,"context_line":"# which is still \u003e1000x faster than a naive python implementation."}],"source_content_type":"text/x-python","patch_set":23,"id":"494cc33b_c5679a6f","line":96,"updated":"2024-10-22 04:47:13.000000000","message":"Another row for the table:\n\nAMD 3900 XT\n- ISA-L: ~20GB/s\n- Kernel: ~5GB/s\n- Naive: ~1.1MB/s\n\nCuriously (though perhaps not surprisingly), the kernel implementation was cut down to ~1.7GB/s on WSL. The other two remained about the same.","commit_id":"d85889d88387df6a2af4595917842c556dbe2e26"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"440b3f5ebd96b3f443637259f8c76f8b176d1d1d","unresolved":false,"context_lines":[{"line_number":93,"context_line":"# ---------------+----------+----------+---------"},{"line_number":94,"context_line":"# Intel N100     | ~9GB/s   | ~3.5GB/s | ~1.1MB/s"},{"line_number":95,"context_line":"# ARM Cortex-A55 | ~2.5GB/s | ~0.4GB/s | ~0.2MB/s"},{"line_number":96,"context_line":"# Intel 11850H   | ~7GB/s   | ~2.6GB/s | ~1.5MB/s"},{"line_number":97,"context_line":"#"},{"line_number":98,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":99,"context_line":"# which is still \u003e1000x faster than a naive python implementation."}],"source_content_type":"text/x-python","patch_set":23,"id":"23afd5ae_b62eb6d3","line":96,"in_reply_to":"02efef4e_929949c4","updated":"2024-11-06 20:54:06.000000000","message":"I don\u0027t think we need to seriously consider \"supporting\" WSL -- I just found it an interesting data point.\n\nTable\u0027s updated.","commit_id":"d85889d88387df6a2af4595917842c556dbe2e26"},{"author":{"_account_id":34892,"name":"ASHWIN A NAIR","display_name":"indianwhocodes","email":"nairashwin952013@gmail.com","username":"indianwhocodes","status":"Nvidia"},"change_message_id":"7e544dce082d616c1eba497e3b76ef30563cbf01","unresolved":true,"context_lines":[{"line_number":93,"context_line":"# ---------------+----------+----------+---------"},{"line_number":94,"context_line":"# Intel N100     | ~9GB/s   | ~3.5GB/s | ~1.1MB/s"},{"line_number":95,"context_line":"# ARM Cortex-A55 | ~2.5GB/s | ~0.4GB/s | ~0.2MB/s"},{"line_number":96,"context_line":"# Intel 11850H   | ~7GB/s   | ~2.6GB/s | ~1.5MB/s"},{"line_number":97,"context_line":"#"},{"line_number":98,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":99,"context_line":"# which is still \u003e1000x faster than a naive python implementation."}],"source_content_type":"text/x-python","patch_set":23,"id":"02efef4e_929949c4","line":96,"in_reply_to":"494cc33b_c5679a6f","updated":"2024-10-25 20:27:24.000000000","message":"It might take me a week at the very least to setup a vsaio WSL VM, maybe @shreeyad@nvidia.com has one lying around.\n\nFWIW, my vote on this patch is still a `+1`","commit_id":"d85889d88387df6a2af4595917842c556dbe2e26"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2cf00ae45f17796a4b079ed90dc9b02f6e47b155","unresolved":true,"context_lines":[{"line_number":69,"context_line":"    def crc32c_kern(data, seed\u003d0):"},{"line_number":70,"context_line":"        crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)"},{"line_number":71,"context_line":"        try:"},{"line_number":72,"context_line":"            crc32c_sock.bind((\"hash\", \"crc32c\"))"},{"line_number":73,"context_line":"            crc32c_sock.setsockopt("},{"line_number":74,"context_line":"                socket.SOL_ALG,"},{"line_number":75,"context_line":"                socket.ALG_SET_KEY,"}],"source_content_type":"text/x-python","patch_set":25,"id":"a6bb3155_34baaf74","line":72,"updated":"2024-11-14 00:17:32.000000000","message":"There\u0027s a chance this could raise `ENOENT`, if the kernel doesn\u0027t provide crc32c... but I\u0027ve not encountered any without it. Certainly, the algorithm has been present in the kernel throughout all of [its git history](https://github.com/torvalds/linux/blob/v2.6.12-rc2/lib/libcrc32c.c), but I don\u0027t know enough about the details of kernel sockets to tell when it got exposed via this API.","commit_id":"bb611fd6957f4407b912a480681abfccac89e454"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d96953e4cc0bb3dd848d2745a9ede6dc31040012","unresolved":false,"context_lines":[{"line_number":69,"context_line":"    def crc32c_kern(data, seed\u003d0):"},{"line_number":70,"context_line":"        crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)"},{"line_number":71,"context_line":"        try:"},{"line_number":72,"context_line":"            crc32c_sock.bind((\"hash\", \"crc32c\"))"},{"line_number":73,"context_line":"            crc32c_sock.setsockopt("},{"line_number":74,"context_line":"                socket.SOL_ALG,"},{"line_number":75,"context_line":"                socket.ALG_SET_KEY,"}],"source_content_type":"text/x-python","patch_set":25,"id":"834756fd_3bf20d1a","line":72,"in_reply_to":"8ba16a85_cca48512","updated":"2025-03-13 04:31:57.000000000","message":"OK, I think this probably [came in around 3.9](https://github.com/torvalds/linux/commit/78c37d191dd6899d8c219fee597a17d6e3c5d288#diff-75a9800db46631c5961073f214dac7b542a5c8d3ee097543376c34f2eb3b1364) 12 years ago. Regardless, I\u0027ve now got it trying to instantiate one of these at import-time to check whether it\u0027s available.","commit_id":"bb611fd6957f4407b912a480681abfccac89e454"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"83364fa6bde564333eb02d34cf4569d2e0c94d75","unresolved":true,"context_lines":[{"line_number":69,"context_line":"    def crc32c_kern(data, seed\u003d0):"},{"line_number":70,"context_line":"        crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)"},{"line_number":71,"context_line":"        try:"},{"line_number":72,"context_line":"            crc32c_sock.bind((\"hash\", \"crc32c\"))"},{"line_number":73,"context_line":"            crc32c_sock.setsockopt("},{"line_number":74,"context_line":"                socket.SOL_ALG,"},{"line_number":75,"context_line":"                socket.ALG_SET_KEY,"}],"source_content_type":"text/x-python","patch_set":25,"id":"8ba16a85_cca48512","line":72,"in_reply_to":"a6bb3155_34baaf74","updated":"2024-12-17 00:52:18.000000000","message":"I guess we could throw in an\n\n```\nexcept OSError as err:\n  if err.errno \u003d\u003d errno.ENOENT:\n    crc32c_kern \u003d None \n```\n\nOr something. But also happy to no worry about that until if it ever happens seeing as this code only ever runs if `hasatttr(socket, \u0027AL_ALG\u0027)` which is probably already enough of a guard.\nSo I think I\u0027m ok with this how it is!","commit_id":"bb611fd6957f4407b912a480681abfccac89e454"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"5c13b41613673d0896448c369c35384818494ded","unresolved":true,"context_lines":[{"line_number":32,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"},{"line_number":33,"context_line":"    rem \u003d seed ^ 0xffffffff"},{"line_number":34,"context_line":"    for x in data:"},{"line_number":35,"context_line":"        rem ^\u003d ord(x) if six.PY2 else x"},{"line_number":36,"context_line":"        for _ in range(8):"},{"line_number":37,"context_line":"            if rem \u0026 1:"},{"line_number":38,"context_line":"                rem \u003d (rem \u003e\u003e 1) ^ p"}],"source_content_type":"text/x-python","patch_set":27,"id":"bcb1ecfc_e5a5041d","line":35,"updated":"2025-01-14 18:09:39.000000000","message":"we can get rid of py2 now","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d96953e4cc0bb3dd848d2745a9ede6dc31040012","unresolved":false,"context_lines":[{"line_number":32,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"},{"line_number":33,"context_line":"    rem \u003d seed ^ 0xffffffff"},{"line_number":34,"context_line":"    for x in data:"},{"line_number":35,"context_line":"        rem ^\u003d ord(x) if six.PY2 else x"},{"line_number":36,"context_line":"        for _ in range(8):"},{"line_number":37,"context_line":"            if rem \u0026 1:"},{"line_number":38,"context_line":"                rem \u003d (rem \u003e\u003e 1) ^ p"}],"source_content_type":"text/x-python","patch_set":27,"id":"e601e517_00bb6145","line":35,"in_reply_to":"bcb1ecfc_e5a5041d","updated":"2025-03-13 04:31:57.000000000","message":"Done","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"5c13b41613673d0896448c369c35384818494ded","unresolved":true,"context_lines":[{"line_number":83,"context_line":"        finally:"},{"line_number":84,"context_line":"            crc32c_sock.close()"},{"line_number":85,"context_line":"else:"},{"line_number":86,"context_line":"    # py2"},{"line_number":87,"context_line":"    crc32c_kern \u003d None"},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"# Use the best implementation available."}],"source_content_type":"text/x-python","patch_set":27,"id":"a80110e4_45a703b7","line":86,"updated":"2025-01-14 18:09:39.000000000","message":"here as well","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d96953e4cc0bb3dd848d2745a9ede6dc31040012","unresolved":false,"context_lines":[{"line_number":83,"context_line":"        finally:"},{"line_number":84,"context_line":"            crc32c_sock.close()"},{"line_number":85,"context_line":"else:"},{"line_number":86,"context_line":"    # py2"},{"line_number":87,"context_line":"    crc32c_kern \u003d None"},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"# Use the best implementation available."}],"source_content_type":"text/x-python","patch_set":27,"id":"6a8ea498_cfbb7609","line":86,"in_reply_to":"a80110e4_45a703b7","updated":"2025-03-13 04:31:57.000000000","message":"Done (sort of)","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"5c13b41613673d0896448c369c35384818494ded","unresolved":true,"context_lines":[{"line_number":97,"context_line":"# AMD 3900XT     | ~20GB/s   | ~5GB/s   | ~1.1MB/s"},{"line_number":98,"context_line":"#"},{"line_number":99,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":100,"context_line":"# which is still \u003e1000x faster than a naive python implementation."},{"line_number":101,"context_line":"# Get on py3 so you can use the kernel sockets!"},{"line_number":102,"context_line":"if crc32c_isal:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_isal"}],"source_content_type":"text/x-python","patch_set":27,"id":"0e6fc7a9_7d5db7cb","line":100,"updated":"2025-01-14 18:09:39.000000000","message":"wow, ISA-L and kernel sockets are so much faster...\n\nI wonder if we still need the naive python implementation anymore: 1. swift doesn\u0027t support py2 anymore, so py3 at least can use kernel sockets (from python 3.6); 2. if a user enabled ISA-L or kernel sockets, and had it working for a while; but then some system/environment issue is causing ``crc32c`` to fall back to ``crc32c_ref`` silently, and then one critical component within swift becomes extremely slow.","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1fcc99d490c636276462a2ad34d9d0ccfa4cc847","unresolved":true,"context_lines":[{"line_number":97,"context_line":"# AMD 3900XT     | ~20GB/s   | ~5GB/s   | ~1.1MB/s"},{"line_number":98,"context_line":"#"},{"line_number":99,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":100,"context_line":"# which is still \u003e1000x faster than a naive python implementation."},{"line_number":101,"context_line":"# Get on py3 so you can use the kernel sockets!"},{"line_number":102,"context_line":"if crc32c_isal:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_isal"}],"source_content_type":"text/x-python","patch_set":27,"id":"5e54d6dc_f8216dda","line":100,"in_reply_to":"0e6fc7a9_7d5db7cb","updated":"2025-03-12 21:02:36.000000000","message":"we need the ref impl for platforms that don\u0027t have the kernel support (e.g. macos)","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4cdd79fb2d5a7c7be20b20238842621bec62a078","unresolved":false,"context_lines":[{"line_number":97,"context_line":"# AMD 3900XT     | ~20GB/s   | ~5GB/s   | ~1.1MB/s"},{"line_number":98,"context_line":"#"},{"line_number":99,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":100,"context_line":"# which is still \u003e1000x faster than a naive python implementation."},{"line_number":101,"context_line":"# Get on py3 so you can use the kernel sockets!"},{"line_number":102,"context_line":"if crc32c_isal:"},{"line_number":103,"context_line":"    crc32c \u003d crc32c_isal"}],"source_content_type":"text/x-python","patch_set":27,"id":"1523f340_6d6972e7","line":100,"in_reply_to":"5e54d6dc_f8216dda","updated":"2025-03-14 11:38:16.000000000","message":"Done","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":62,"context_line":"            seed ^ 0xffff_ffff,"},{"line_number":63,"context_line":"        )"},{"line_number":64,"context_line":"        # for some reason, despite us specifying that restype is uint,"},{"line_number":65,"context_line":"        # it can come back signed??"},{"line_number":66,"context_line":"        return (result \u0026 0xffff_ffff) ^ 0xffff_ffff"},{"line_number":67,"context_line":"else:"},{"line_number":68,"context_line":"    crc32c_isal \u003d None"}],"source_content_type":"text/x-python","patch_set":28,"id":"1068d864_b3c975e0","line":65,"updated":"2025-01-22 19:00:46.000000000","message":"IDK why I didn\u0027t see this fail before, but","commit_id":"577c810879f166bf76fcb88ac23f7c5b46ec9a8a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ed64e310d051be6b133a118f3aafb0f7ca82e066","unresolved":false,"context_lines":[{"line_number":62,"context_line":"            seed ^ 0xffff_ffff,"},{"line_number":63,"context_line":"        )"},{"line_number":64,"context_line":"        # for some reason, despite us specifying that restype is uint,"},{"line_number":65,"context_line":"        # it can come back signed??"},{"line_number":66,"context_line":"        return (result \u0026 0xffff_ffff) ^ 0xffff_ffff"},{"line_number":67,"context_line":"else:"},{"line_number":68,"context_line":"    crc32c_isal \u003d None"}],"source_content_type":"text/x-python","patch_set":28,"id":"e51cb8e7_1235ec81","line":65,"in_reply_to":"1068d864_b3c975e0","updated":"2025-03-14 01:43:30.000000000","message":"I don\u0027t entirely remember the circumstances, but I saw some negative values coming out here before adding the `\u0026 0xffff_ffff`","commit_id":"577c810879f166bf76fcb88ac23f7c5b46ec9a8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":true,"context_lines":[{"line_number":25,"context_line":"    pkg_files \u003d None"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":29,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":30,"context_line":"    # reference implementation"},{"line_number":31,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":29,"id":"894c5d7d_a8845a0e","line":28,"range":{"start_line":28,"start_character":4,"end_line":28,"end_character":28},"updated":"2025-01-20 17:23:55.000000000","message":"so IIUC we can\u0027t stream data to a crc32c? Just buffer the whole lot and then calculate the crc?","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a5adbc29d4dd8429394348f2a1d371c9393c64b3","unresolved":true,"context_lines":[{"line_number":25,"context_line":"    pkg_files \u003d None"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":29,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":30,"context_line":"    # reference implementation"},{"line_number":31,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":29,"id":"97ddad9d_835cb3b4","line":28,"range":{"start_line":28,"start_character":4,"end_line":28,"end_character":28},"in_reply_to":"5a3df661_d819f85e","updated":"2025-01-23 09:23:24.000000000","message":"yeah, I was being dumb, or at least I hadn\u0027t interpreted ``seed`` as being the vehicle for iterating. Perhaps a rename might help and/or a doc-string","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":25,"context_line":"    pkg_files \u003d None"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":29,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":30,"context_line":"    # reference implementation"},{"line_number":31,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a3df661_d819f85e","line":28,"range":{"start_line":28,"start_character":4,"end_line":28,"end_character":28},"in_reply_to":"894c5d7d_a8845a0e","updated":"2025-01-22 19:00:46.000000000","message":"You can stream data through it; it works just like [crc32](https://docs.python.org/3/library/zlib.html#zlib.crc32) -- you just pass the previous result as the `seed`. (Maybe I should rename it `value` like in stdlib? I don\u0027t remember why I named it that.) It\u0027s even tested:\n```\n    def check_crc_func(self, impl):\n        self.assertEqual(impl(b\"123456789\"), 0xe3069283)\n        # Check that we can save/continue\n        partial \u003d impl(b\"12345\")\n        self.assertEqual(impl(b\"6789\", partial), 0xe3069283)\n```","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3213d076f29a7aec66c07f72469c8c3c602059be","unresolved":false,"context_lines":[{"line_number":25,"context_line":"    pkg_files \u003d None"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"def crc32c_ref(data, seed\u003d0):"},{"line_number":29,"context_line":"    # Dumb-as-dirt CRC32C implementation, heavily influenced by ISA-L\u0027s"},{"line_number":30,"context_line":"    # reference implementation"},{"line_number":31,"context_line":"    p \u003d 0x82F63B78  # reversed polynomial"}],"source_content_type":"text/x-python","patch_set":29,"id":"55acfc00_80b00140","line":28,"range":{"start_line":28,"start_character":4,"end_line":28,"end_character":28},"in_reply_to":"97ddad9d_835cb3b4","updated":"2025-03-11 18:43:23.000000000","message":"Done","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":true,"context_lines":[{"line_number":43,"context_line":"# If isal is available system-wide, great!"},{"line_number":44,"context_line":"isal_lib \u003d ctypes.util.find_library(\u0027isal\u0027)"},{"line_number":45,"context_line":"if isal_lib is None and pkg_files is not None:"},{"line_number":46,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":47,"context_line":"    # with isal baked in?"},{"line_number":48,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027)"},{"line_number":49,"context_line":"                 if f.name.startswith(\"libisal\")]"},{"line_number":50,"context_line":"    if len(isal_libs) \u003d\u003d 1:"}],"source_content_type":"text/x-python","patch_set":29,"id":"3dfda0ea_34a556c8","line":47,"range":{"start_line":46,"start_character":4,"end_line":47,"end_character":25},"updated":"2025-01-20 17:23:55.000000000","message":"Is this only for py38+ because importlib.metadata is py38+, or is it because the pyeclib manylinux wheels is only built for py38+?\n\nIf it\u0027s due to importlib.metadata, then can\u0027t we use importlib_metadata? I\u0027m thinking of an op who has installed manylinux wheel with py37 - seems like they\u0027d really like to use the isa-l crc implementation.","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":43,"context_line":"# If isal is available system-wide, great!"},{"line_number":44,"context_line":"isal_lib \u003d ctypes.util.find_library(\u0027isal\u0027)"},{"line_number":45,"context_line":"if isal_lib is None and pkg_files is not None:"},{"line_number":46,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":47,"context_line":"    # with isal baked in?"},{"line_number":48,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027)"},{"line_number":49,"context_line":"                 if f.name.startswith(\"libisal\")]"},{"line_number":50,"context_line":"    if len(isal_libs) \u003d\u003d 1:"}],"source_content_type":"text/x-python","patch_set":29,"id":"758b7332_4b0f04e1","line":47,"range":{"start_line":46,"start_character":4,"end_line":47,"end_character":25},"in_reply_to":"3dfda0ea_34a556c8","updated":"2025-01-22 19:00:46.000000000","message":"\u003e Is this only for py38+ because importlib.metadata is py38+\n\nThat\u0027s the one -- and I prefer leaving a carrot for upgrading python over investing in figuring out how to locate arbitrary extra files on old python.\n\n\u003e or is it because the pyeclib manylinux wheels is only built for py38+?\n\nNo, we publish manylinux wheels for 2.7 and 3.5+ (but not 3.13t): https://pypi.org/project/pyeclib/#files -- though I\u0027d like to [trim that](https://review.opendev.org/c/openstack/pyeclib/+/938098) for our next release.\n\n\u003e If it\u0027s due to importlib.metadata, then can\u0027t we use importlib_metadata? I\u0027m thinking of an op who has installed manylinux wheel with py37 - seems like they\u0027d really like to use the isa-l crc implementation.\n\nIDK, 3-5x speed-up is nice and all, but it\u0027s probably dwarfed by all the MD5\u0027ing we do. And I\u0027m not sure I want to add the dependency (even if we can do it with a `;python_version\u003c\u00273.8\u0027`)\n\nIf you\u0027re worried about a *specific* operator that only just got to py37 😉:\n- we don\u0027t use the binary wheels that are published; we install pyeclib, liberasurecode, and isa-l via separate RPMs\n- we install isa-l in a vendor-specific location, not as a system-wide library\n- as a result, we still won\u0027t find isa-l.\n\nI see it all as another reason to get out of the business of packaging these things ourselves; if we can upgrade OS and start using system packages for isa-l (and maybe even liberasurecode?) it all just works and is simpler (both to implement and maintain).","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a5adbc29d4dd8429394348f2a1d371c9393c64b3","unresolved":false,"context_lines":[{"line_number":43,"context_line":"# If isal is available system-wide, great!"},{"line_number":44,"context_line":"isal_lib \u003d ctypes.util.find_library(\u0027isal\u0027)"},{"line_number":45,"context_line":"if isal_lib is None and pkg_files is not None:"},{"line_number":46,"context_line":"    # py38+: Hopefully pyeclib was installed from a manylinux wheel"},{"line_number":47,"context_line":"    # with isal baked in?"},{"line_number":48,"context_line":"    isal_libs \u003d [f for f in pkg_files(\u0027pyeclib\u0027)"},{"line_number":49,"context_line":"                 if f.name.startswith(\"libisal\")]"},{"line_number":50,"context_line":"    if len(isal_libs) \u003d\u003d 1:"}],"source_content_type":"text/x-python","patch_set":29,"id":"bef3a342_01904f89","line":47,"range":{"start_line":46,"start_character":4,"end_line":47,"end_character":25},"in_reply_to":"758b7332_4b0f04e1","updated":"2025-01-23 09:23:24.000000000","message":"Acknowledged","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":true,"context_lines":[{"line_number":69,"context_line":""},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"def crc32c_kern(data, seed\u003d0):"},{"line_number":72,"context_line":"    crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)"},{"line_number":73,"context_line":"    try:"},{"line_number":74,"context_line":"        crc32c_sock.bind((\"hash\", \"crc32c\"))"},{"line_number":75,"context_line":"        crc32c_sock.setsockopt("}],"source_content_type":"text/x-python","patch_set":29,"id":"4c56d979_111715bf","line":72,"updated":"2025-01-20 17:23:55.000000000","message":"blows up on macos\n\nhow about\n\n```\ndiff --git a/swift/common/utils/checksum.py b/swift/common/utils/checksum.py\nindex 133edf4fc..cea6fc508 100644\n--- a/swift/common/utils/checksum.py\n+++ b/swift/common/utils/checksum.py\n@@ -67,24 +67,25 @@ if hasattr(isal, \u0027crc32_iscsi\u0027):  # isa-l \u003e\u003d 2.16\n else:\n     crc32c_isal \u003d None\n\n-\n-def crc32c_kern(data, seed\u003d0):\n-    crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)\n-    try:\n-        crc32c_sock.bind((\"hash\", \"crc32c\"))\n-        crc32c_sock.setsockopt(\n-            socket.SOL_ALG,\n-            socket.ALG_SET_KEY,\n-            struct.pack(\"I\", seed ^ 0xffff_ffff))\n-        sock, _ \u003d crc32c_sock.accept()\n+if hasattr(socket, \u0027AF_ALG\u0027):\n+    def crc32c_kern(data, seed\u003d0):\n+        crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)\n         try:\n-            sock.sendall(data)\n-            return struct.unpack(\"I\", sock.recv(4))[0]\n+            crc32c_sock.bind((\"hash\", \"crc32c\"))\n+            crc32c_sock.setsockopt(\n+                socket.SOL_ALG,\n+                socket.ALG_SET_KEY,\n+                struct.pack(\"I\", seed ^ 0xffff_ffff))\n+            sock, _ \u003d crc32c_sock.accept()\n+            try:\n+                sock.sendall(data)\n+                return struct.unpack(\"I\", sock.recv(4))[0]\n+            finally:\n+                sock.close()\n         finally:\n-            sock.close()\n-    finally:\n-        crc32c_sock.close()\n-\n+            crc32c_sock.close()\n+else:\n+    crc32c_kern \u003d None\n\n # Use the best implementation available.\n # On various hardware we\u0027ve seen\n@@ -98,7 +99,9 @@ def crc32c_kern(data, seed\u003d0):\n #\n # i.e., ISA-L is consistently 3-5x faster than kernel sockets,\n # which is still \u003e1000x faster than a naive python implementation.\n-crc32c \u003d crc32c_isal if crc32c_isal else crc32c_kern\n+crc32c \u003d crc32c_isal if crc32c_isal \\\n+    else crc32c_kern if crc32c_kern \\\n+    else crc32c_ref\n\n\n def crc64nvme_ref(data, seed\u003d0):\n```","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1fcc99d490c636276462a2ad34d9d0ccfa4cc847","unresolved":false,"context_lines":[{"line_number":69,"context_line":""},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"def crc32c_kern(data, seed\u003d0):"},{"line_number":72,"context_line":"    crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)"},{"line_number":73,"context_line":"    try:"},{"line_number":74,"context_line":"        crc32c_sock.bind((\"hash\", \"crc32c\"))"},{"line_number":75,"context_line":"        crc32c_sock.setsockopt("}],"source_content_type":"text/x-python","patch_set":29,"id":"b613a15b_7ad8956b","line":72,"in_reply_to":"30372d08_14bdf1be","updated":"2025-03-12 21:02:36.000000000","message":"Done","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":69,"context_line":""},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"def crc32c_kern(data, seed\u003d0):"},{"line_number":72,"context_line":"    crc32c_sock \u003d socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET)"},{"line_number":73,"context_line":"    try:"},{"line_number":74,"context_line":"        crc32c_sock.bind((\"hash\", \"crc32c\"))"},{"line_number":75,"context_line":"        crc32c_sock.setsockopt("}],"source_content_type":"text/x-python","patch_set":29,"id":"30372d08_14bdf1be","line":72,"in_reply_to":"4c56d979_111715bf","updated":"2025-01-22 19:00:46.000000000","message":"Yeah, we can do that. Stupid BSDs :P","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":true,"context_lines":[{"line_number":98,"context_line":"#"},{"line_number":99,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":100,"context_line":"# which is still \u003e1000x faster than a naive python implementation."},{"line_number":101,"context_line":"crc32c \u003d crc32c_isal if crc32c_isal else crc32c_kern"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"def crc64nvme_ref(data, seed\u003d0):"}],"source_content_type":"text/x-python","patch_set":29,"id":"f3576ecd_ad06f398","line":101,"range":{"start_line":101,"start_character":0,"end_line":101,"end_character":52},"updated":"2025-01-20 17:23:55.000000000","message":"ok, so the ref impl sucks, but I\u0027m a little nervous that tests in subsequent patches are going to start to break on dev platforms that don\u0027t have isa-l nor (cough, macos) the kernel impl.\n\nHow about allowing crc32_ref as the final fallback but emitting a big fat warning if it is used.\n\nConversely, why are the ref functions even in this module vs the test module if they\u0027re not intended for use other than for testing against?","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1fcc99d490c636276462a2ad34d9d0ccfa4cc847","unresolved":false,"context_lines":[{"line_number":98,"context_line":"#"},{"line_number":99,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":100,"context_line":"# which is still \u003e1000x faster than a naive python implementation."},{"line_number":101,"context_line":"crc32c \u003d crc32c_isal if crc32c_isal else crc32c_kern"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"def crc64nvme_ref(data, seed\u003d0):"}],"source_content_type":"text/x-python","patch_set":29,"id":"a7ff7f00_3163eddc","line":101,"range":{"start_line":101,"start_character":0,"end_line":101,"end_character":52},"in_reply_to":"e434fec2_5a2b7a55","updated":"2025-03-12 21:02:36.000000000","message":"Done","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":98,"context_line":"#"},{"line_number":99,"context_line":"# i.e., ISA-L is consistently 3-5x faster than kernel sockets,"},{"line_number":100,"context_line":"# which is still \u003e1000x faster than a naive python implementation."},{"line_number":101,"context_line":"crc32c \u003d crc32c_isal if crc32c_isal else crc32c_kern"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"def crc64nvme_ref(data, seed\u003d0):"}],"source_content_type":"text/x-python","patch_set":29,"id":"e434fec2_5a2b7a55","line":101,"range":{"start_line":101,"start_character":0,"end_line":101,"end_character":52},"in_reply_to":"f3576ecd_ad06f398","updated":"2025-01-22 19:00:46.000000000","message":"\u003e How about allowing crc32_ref as the final fallback but emitting a big fat warning if it is used.\n\nYeah, seems reasonable. Again, I hadn\u0027t really been thinking about the BSDs.\n\n\u003e Conversely, why are the ref functions even in this module vs the test module if they\u0027re not intended for use other than for testing against?\n\nWell, it *originally* was for the sake of providing an implementation for py2 -- and then I wasn\u0027t sure what to do with it once we dropped that. But I like it as an implementation of last resort.","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d96953e4cc0bb3dd848d2745a9ede6dc31040012","unresolved":true,"context_lines":[{"line_number":76,"context_line":"    _sock.bind((\"hash\", \"crc32c\"))"},{"line_number":77,"context_line":"except OSError as e:"},{"line_number":78,"context_line":"    if e.errno \u003d\u003d errno.ENOENT:"},{"line_number":79,"context_line":"        # could bind socket, but crc32c is unknown"},{"line_number":80,"context_line":"        _sock.close()"},{"line_number":81,"context_line":"    elif e.errno !\u003d errno.EAFNOSUPPORT:"},{"line_number":82,"context_line":"        raise"}],"source_content_type":"text/x-python","patch_set":33,"id":"4af70009_7b2bc2dc","line":79,"range":{"start_line":79,"start_character":16,"end_line":79,"end_character":20},"updated":"2025-03-13 04:31:57.000000000","message":"\"create\"","commit_id":"e970b2b626b960ebc4523e15dd9894173dfd68aa"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"269e5ab975d275ab8b72cbd0b146918f33ed9dd9","unresolved":false,"context_lines":[{"line_number":76,"context_line":"    _sock.bind((\"hash\", \"crc32c\"))"},{"line_number":77,"context_line":"except OSError as e:"},{"line_number":78,"context_line":"    if e.errno \u003d\u003d errno.ENOENT:"},{"line_number":79,"context_line":"        # could bind socket, but crc32c is unknown"},{"line_number":80,"context_line":"        _sock.close()"},{"line_number":81,"context_line":"    elif e.errno !\u003d errno.EAFNOSUPPORT:"},{"line_number":82,"context_line":"        raise"}],"source_content_type":"text/x-python","patch_set":33,"id":"3dc05070_baa6005e","line":79,"range":{"start_line":79,"start_character":16,"end_line":79,"end_character":20},"in_reply_to":"4af70009_7b2bc2dc","updated":"2025-04-14 03:56:24.000000000","message":"Done","commit_id":"e970b2b626b960ebc4523e15dd9894173dfd68aa"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3213d076f29a7aec66c07f72469c8c3c602059be","unresolved":true,"context_lines":[{"line_number":118,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":119,"context_line":"else:"},{"line_number":120,"context_line":"    warnings.warn(\u0027Using (slow) reference implementation for CRC32-C; \u0027"},{"line_number":121,"context_line":"                  \u0027install ISA-L for faster checksums.\u0027, RuntimeWarning)"},{"line_number":122,"context_line":"    crc32c \u003d crc32c_ref"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"379867a5_43ed5736","line":121,"updated":"2025-03-11 18:43:23.000000000","message":"I think it will be useful to also have some record of which implementation has been selected, particularly given the shenanigans in trying to find isa-l. IDK if a one-off log when s3api is loaded, of registering in the admin swift info?","commit_id":"f110ffb10f74b0689a3fb3219b44d20055e63890"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eaa896e5e203b5640b52e67b12016d72559c7cdd","unresolved":false,"context_lines":[{"line_number":118,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":119,"context_line":"else:"},{"line_number":120,"context_line":"    warnings.warn(\u0027Using (slow) reference implementation for CRC32-C; \u0027"},{"line_number":121,"context_line":"                  \u0027install ISA-L for faster checksums.\u0027, RuntimeWarning)"},{"line_number":122,"context_line":"    crc32c \u003d crc32c_ref"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"db053793_78f201c4","line":121,"in_reply_to":"2e3e8922_e63519f6","updated":"2025-04-02 14:21:29.000000000","message":"ok, this is an action for the s3api patch though","commit_id":"f110ffb10f74b0689a3fb3219b44d20055e63890"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7730e5acf4a222ac46e2d33ea191ebd0e565294b","unresolved":true,"context_lines":[{"line_number":118,"context_line":"    crc32c \u003d crc32c_kern"},{"line_number":119,"context_line":"else:"},{"line_number":120,"context_line":"    warnings.warn(\u0027Using (slow) reference implementation for CRC32-C; \u0027"},{"line_number":121,"context_line":"                  \u0027install ISA-L for faster checksums.\u0027, RuntimeWarning)"},{"line_number":122,"context_line":"    crc32c \u003d crc32c_ref"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":""}],"source_content_type":"text/x-python","patch_set":34,"id":"2e3e8922_e63519f6","line":121,"in_reply_to":"379867a5_43ed5736","updated":"2025-04-02 06:36:04.000000000","message":"Cool idea, maybe an info call could just tell use while has been selected.","commit_id":"f110ffb10f74b0689a3fb3219b44d20055e63890"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"36345e81a7b2b7de430b401a2f465e35864629a5","unresolved":true,"context_lines":[{"line_number":60,"context_line":"    isal.crc32_iscsi.restype \u003d ctypes.c_uint"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"    def crc32c_isal(data, value\u003d0):"},{"line_number":63,"context_line":"        result \u003d isal.crc32_iscsi("},{"line_number":64,"context_line":"            data,"},{"line_number":65,"context_line":"            len(data),"},{"line_number":66,"context_line":"            value ^ 0xffff_ffff,"}],"source_content_type":"text/x-python","patch_set":40,"id":"818e3a90_2d518b98","line":63,"updated":"2025-04-03 04:30:45.000000000","message":"``isal.crc32_iscsi`` itself doesn\u0027t release GIL, but should we release GIL or do some sort of green thread cooperation when calling ``isal.crc32_iscsi``, especially for large data chuck crc32 calcuation? \n\nsince it will be utilizing Intel specific optimized HW, the CPU core used by this thread actually will be free I assume, so it\u0027s like those blocking I/O operations (network calls) which eventlet automatically made cooperative through monkey patching. I have no idea, but just thinking out loud that how can we make green thread calling ``isal.crc32_iscsi`` yielding to others.\n\nif no cooperative yielding implemented with ``isal.crc32_iscsi``, maybe it\u0027s possible that kernel version crc32 actually will be more performant in terms of ttfb and throughputs when testing with many concurrent green threads which proxy-server runs? since kernel crc32 will be monkey patched with the usage of socket.","commit_id":"2b0a51f59bf6759490639643e312828c84456f64"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e43aae7af87ad24e60c718620cfef11b079306c2","unresolved":false,"context_lines":[{"line_number":60,"context_line":"    isal.crc32_iscsi.restype \u003d ctypes.c_uint"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"    def crc32c_isal(data, value\u003d0):"},{"line_number":63,"context_line":"        result \u003d isal.crc32_iscsi("},{"line_number":64,"context_line":"            data,"},{"line_number":65,"context_line":"            len(data),"},{"line_number":66,"context_line":"            value ^ 0xffff_ffff,"}],"source_content_type":"text/x-python","patch_set":40,"id":"5fbf5da7_f7f682de","line":63,"in_reply_to":"818e3a90_2d518b98","updated":"2025-04-07 17:00:45.000000000","message":"since the data size for the crc32 calculation is only 64KB (network chunk size), I don\u0027t think it\u0027s a concern. Green thread cooperation is desirable for very large data chunks I think, but our case is only 64KB typically and isal is super fast, so probably eventlet hub would spend more CPU cycles and it\u0027ll be counter-productive to use green thread cooperative yielding.","commit_id":"2b0a51f59bf6759490639643e312828c84456f64"}],"test/unit/common/utils/test_checksum.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"5c13b41613673d0896448c369c35384818494ded","unresolved":true,"context_lines":[{"line_number":34,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_ref)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"},{"line_number":37,"context_line":"    def test_kern(self):"},{"line_number":38,"context_line":"        self.check_crc_func(checksum.crc32c_kern)"},{"line_number":39,"context_line":"        # Check preferences -- beats out reference, but not ISA-L"},{"line_number":40,"context_line":"        if checksum.crc32c_isal is None:"}],"source_content_type":"text/x-python","patch_set":27,"id":"7c250079_81665d43","line":37,"updated":"2025-01-14 18:09:39.000000000","message":"more tests will be great to cover the failures of ``crc32c_sock.bind``, ``crc32c_sock.setsockopt`` and ``sock.sendall``, to make sure socket would be closed normally, can also be a follow-up.","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eaa896e5e203b5640b52e67b12016d72559c7cdd","unresolved":false,"context_lines":[{"line_number":34,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_ref)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"},{"line_number":37,"context_line":"    def test_kern(self):"},{"line_number":38,"context_line":"        self.check_crc_func(checksum.crc32c_kern)"},{"line_number":39,"context_line":"        # Check preferences -- beats out reference, but not ISA-L"},{"line_number":40,"context_line":"        if checksum.crc32c_isal is None:"}],"source_content_type":"text/x-python","patch_set":27,"id":"82fe818a_3f27ede7","line":37,"in_reply_to":"7c250079_81665d43","updated":"2025-04-02 14:21:29.000000000","message":"Done","commit_id":"4fbb94d26d5d60ac8c9c27013c932c575a618e9e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":true,"context_lines":[{"line_number":30,"context_line":"    def test_ref(self):"},{"line_number":31,"context_line":"        self.check_crc_func(checksum.crc32c_ref)"},{"line_number":32,"context_line":"        # Check preferences -- choice of last resort"},{"line_number":33,"context_line":"        if checksum.crc32c_isal is None and checksum.crc32c_kern is None:"},{"line_number":34,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_ref)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"b545b278_c8a95f89","line":33,"range":{"start_line":33,"start_character":44,"end_line":33,"end_character":72},"updated":"2025-01-20 17:23:55.000000000","message":"this is never True - crc32c_kern is a module function\n\nI\u0027d like it to be True on macos though ;-)","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":true,"context_lines":[{"line_number":30,"context_line":"    def test_ref(self):"},{"line_number":31,"context_line":"        self.check_crc_func(checksum.crc32c_ref)"},{"line_number":32,"context_line":"        # Check preferences -- choice of last resort"},{"line_number":33,"context_line":"        if checksum.crc32c_isal is None and checksum.crc32c_kern is None:"},{"line_number":34,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_ref)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"c72c2af1_45e61423","line":33,"range":{"start_line":33,"start_character":8,"end_line":33,"end_character":39},"updated":"2025-01-20 17:23:55.000000000","message":"this may be True in a py3.6 test env but not sure it\u0027s going to be true in any other zuul test env","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eaa896e5e203b5640b52e67b12016d72559c7cdd","unresolved":false,"context_lines":[{"line_number":30,"context_line":"    def test_ref(self):"},{"line_number":31,"context_line":"        self.check_crc_func(checksum.crc32c_ref)"},{"line_number":32,"context_line":"        # Check preferences -- choice of last resort"},{"line_number":33,"context_line":"        if checksum.crc32c_isal is None and checksum.crc32c_kern is None:"},{"line_number":34,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_ref)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"91a57e37_5947f5bc","line":33,"range":{"start_line":33,"start_character":8,"end_line":33,"end_character":39},"in_reply_to":"11a580fe_cd5cf950","updated":"2025-04-02 14:21:29.000000000","message":"Acknowledged","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":false,"context_lines":[{"line_number":30,"context_line":"    def test_ref(self):"},{"line_number":31,"context_line":"        self.check_crc_func(checksum.crc32c_ref)"},{"line_number":32,"context_line":"        # Check preferences -- choice of last resort"},{"line_number":33,"context_line":"        if checksum.crc32c_isal is None and checksum.crc32c_kern is None:"},{"line_number":34,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_ref)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"b3ae9093_4757f1ea","line":33,"range":{"start_line":33,"start_character":44,"end_line":33,"end_character":72},"in_reply_to":"b545b278_c8a95f89","updated":"2025-01-22 19:00:46.000000000","message":"Acknowledged","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":30,"context_line":"    def test_ref(self):"},{"line_number":31,"context_line":"        self.check_crc_func(checksum.crc32c_ref)"},{"line_number":32,"context_line":"        # Check preferences -- choice of last resort"},{"line_number":33,"context_line":"        if checksum.crc32c_isal is None and checksum.crc32c_kern is None:"},{"line_number":34,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_ref)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"11a580fe_cd5cf950","line":33,"range":{"start_line":33,"start_character":8,"end_line":33,"end_character":39},"in_reply_to":"c72c2af1_45e61423","updated":"2025-01-22 19:00:46.000000000","message":"The experimental pipeline will also run a py37 job that would hit it.","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"917530f27b6fee07e0b3bca9b8e88f5f40aa3626","unresolved":true,"context_lines":[{"line_number":37,"context_line":"    def test_kern(self):"},{"line_number":38,"context_line":"        self.check_crc_func(checksum.crc32c_kern)"},{"line_number":39,"context_line":"        # Check preferences -- beats out reference, but not ISA-L"},{"line_number":40,"context_line":"        if checksum.crc32c_isal is None:"},{"line_number":41,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_kern)"},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"    @unittest.skipIf(checksum.crc32c_isal is None, \u0027No ISA-L CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"7c73807d_ba6d9572","line":40,"range":{"start_line":40,"start_character":8,"end_line":40,"end_character":39},"updated":"2025-01-20 17:23:55.000000000","message":"seems like we\u0027re relying on test running in an env without isa-l - why not mock something to force crc32c_isal to be None","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"639e3272d93d28c0b5872bf1c1b1d84f11f93eb1","unresolved":true,"context_lines":[{"line_number":37,"context_line":"    def test_kern(self):"},{"line_number":38,"context_line":"        self.check_crc_func(checksum.crc32c_kern)"},{"line_number":39,"context_line":"        # Check preferences -- beats out reference, but not ISA-L"},{"line_number":40,"context_line":"        if checksum.crc32c_isal is None:"},{"line_number":41,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_kern)"},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"    @unittest.skipIf(checksum.crc32c_isal is None, \u0027No ISA-L CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"cc720230_4a7b84a1","line":40,"range":{"start_line":40,"start_character":8,"end_line":40,"end_character":39},"in_reply_to":"7c73807d_ba6d9572","updated":"2025-01-22 19:00:46.000000000","message":"Because the setting of `checksum.crc32c` happens at import time.","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"3213d076f29a7aec66c07f72469c8c3c602059be","unresolved":true,"context_lines":[{"line_number":37,"context_line":"    def test_kern(self):"},{"line_number":38,"context_line":"        self.check_crc_func(checksum.crc32c_kern)"},{"line_number":39,"context_line":"        # Check preferences -- beats out reference, but not ISA-L"},{"line_number":40,"context_line":"        if checksum.crc32c_isal is None:"},{"line_number":41,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_kern)"},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"    @unittest.skipIf(checksum.crc32c_isal is None, \u0027No ISA-L CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"de9eb704_fa240194","line":40,"range":{"start_line":40,"start_character":8,"end_line":40,"end_character":39},"in_reply_to":"cc720230_4a7b84a1","updated":"2025-03-11 18:43:23.000000000","message":"right, but how do we guarantee test coverage? does the interface need to change so that we *can* mock something? Like, have the selected impl be properties of a class\nthat are lazy-loaded, so we\u0027d write:\n\n``crc_func \u003d checksum.Checksums.crc32c``\n\nShame we can\u0027t use a module ``__getattr__`` https://peps.python.org/pep-0562/``.\n\nOr pull the CRCHasher class into this patch and make *that* the primary interface?","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4cdd79fb2d5a7c7be20b20238842621bec62a078","unresolved":false,"context_lines":[{"line_number":37,"context_line":"    def test_kern(self):"},{"line_number":38,"context_line":"        self.check_crc_func(checksum.crc32c_kern)"},{"line_number":39,"context_line":"        # Check preferences -- beats out reference, but not ISA-L"},{"line_number":40,"context_line":"        if checksum.crc32c_isal is None:"},{"line_number":41,"context_line":"            self.assertIs(checksum.crc32c, checksum.crc32c_kern)"},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"    @unittest.skipIf(checksum.crc32c_isal is None, \u0027No ISA-L CRC32C\u0027)"}],"source_content_type":"text/x-python","patch_set":29,"id":"2e210c98_a8407783","line":40,"range":{"start_line":40,"start_character":8,"end_line":40,"end_character":39},"in_reply_to":"de9eb704_fa240194","updated":"2025-03-14 11:38:16.000000000","message":"Done","commit_id":"d869cd9b1322d936e2eddf3d01f4d9703950d347"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7730e5acf4a222ac46e2d33ea191ebd0e565294b","unresolved":true,"context_lines":[{"line_number":204,"context_line":"        with mock.patch("},{"line_number":205,"context_line":"                \u0027swift.common.utils.checksum.crc32c_isal\u0027) as mock_isal, \\"},{"line_number":206,"context_line":"                mock.patch(\u0027swift.common.utils.checksum.crc32c_kern\u0027):"},{"line_number":207,"context_line":"            self.assertEqual(mock_isal, checksum.crc32c().crc_func)"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"    def test_crc64nvme_hasher(self):"},{"line_number":210,"context_line":"        # See CRC-64/NVME at"}],"source_content_type":"text/x-python","patch_set":37,"id":"c5c4bb32_2f2443f7","line":207,"updated":"2025-04-02 06:36:04.000000000","message":"I guess this is another place we could throw in a test for precidence:\n\n```\nwith mock.patch(\u0027swift.common.utils.checksum.crc32c_isal\u0027) as mock_isal, \\\n        mock.patch(\u0027swift.common.utils.checksum.crc32c_kern\u0027) as mock_kern:\n   self.assertEqual(mock_isal, checksum.crc32c().crc_func)\n```\n\nBut I guess the mock and assert on 199 is already covering in inverse. But just for completeness sake.","commit_id":"ef29bf933a898ed6e012c35c42794b77de42e1cf"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"eaa896e5e203b5640b52e67b12016d72559c7cdd","unresolved":true,"context_lines":[{"line_number":204,"context_line":"        with mock.patch("},{"line_number":205,"context_line":"                \u0027swift.common.utils.checksum.crc32c_isal\u0027) as mock_isal, \\"},{"line_number":206,"context_line":"                mock.patch(\u0027swift.common.utils.checksum.crc32c_kern\u0027):"},{"line_number":207,"context_line":"            self.assertEqual(mock_isal, checksum.crc32c().crc_func)"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"    def test_crc64nvme_hasher(self):"},{"line_number":210,"context_line":"        # See CRC-64/NVME at"}],"source_content_type":"text/x-python","patch_set":37,"id":"da00265d_38c0bca7","line":207,"in_reply_to":"c5c4bb32_2f2443f7","updated":"2025-04-02 14:21:29.000000000","message":"this is the same as already on line 204-207 isn\u0027t it? except for ``as mock_kern`` but that would be unused","commit_id":"ef29bf933a898ed6e012c35c42794b77de42e1cf"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"edd711bd0ce0d313b9315acc793318c7472b6fdc","unresolved":true,"context_lines":[{"line_number":204,"context_line":"        with mock.patch("},{"line_number":205,"context_line":"                \u0027swift.common.utils.checksum.crc32c_isal\u0027) as mock_isal, \\"},{"line_number":206,"context_line":"                mock.patch(\u0027swift.common.utils.checksum.crc32c_kern\u0027):"},{"line_number":207,"context_line":"            self.assertEqual(mock_isal, checksum.crc32c().crc_func)"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"    def test_crc64nvme_hasher(self):"},{"line_number":210,"context_line":"        # See CRC-64/NVME at"}],"source_content_type":"text/x-python","patch_set":37,"id":"56173b72_7477b3c2","line":207,"in_reply_to":"da00265d_38c0bca7","updated":"2025-04-02 22:18:13.000000000","message":"yeah, that was the point and why it might be more of a nit. It shows that it did pick the first, because making them None is a basically a way we\u0027re saying their disabled.","commit_id":"ef29bf933a898ed6e012c35c42794b77de42e1cf"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"36345e81a7b2b7de430b401a2f465e35864629a5","unresolved":false,"context_lines":[{"line_number":59,"context_line":"                         mock_crc32c_socket.close.call_args_list)"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"    @unittest.skipIf(checksum.crc32c_kern is None, \u0027No kernel CRC32C\u0027)"},{"line_number":62,"context_line":"    def test_kern_socket_close_after_bind_error(self):"},{"line_number":63,"context_line":"        mock_crc32c_socket \u003d mock.MagicMock()"},{"line_number":64,"context_line":"        mock_crc32c_socket.bind.side_effect \u003d OSError(\u0027boom\u0027)"},{"line_number":65,"context_line":"        with mock.patch(\u0027swift.common.utils.checksum.socket.socket\u0027,"}],"source_content_type":"text/x-python","patch_set":40,"id":"ca9b87c6_632ad736","line":62,"updated":"2025-04-03 04:30:45.000000000","message":"Those exception handling test cases look great, thanks for adding them.","commit_id":"2b0a51f59bf6759490639643e312828c84456f64"}]}