)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"733b5e15f18ba81482bb96dd4a150f442106e576","unresolved":true,"context_lines":[{"line_number":7,"context_line":"ring: Create ReplicaToPartToDev helper"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Instead of a list of arrays, let\u0027s just have one contiguous"},{"line_number":10,"context_line":"multidimensional array."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Change-Id: If4941705b43f63ad1257704d716f8753d2613e35"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"d0c136c0_a09aae22","line":10,"updated":"2022-11-21 16:10:17.000000000","message":"contiguous multidimensional array.  that\u0027s cool\n\nhttps://numpy.org/doc/stable/reference/arrays.ndarray.html#internal-memory-layout-of-an-ndarray","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"476595b9f3fc549560c90bed650ee2633f4e7417","unresolved":true,"context_lines":[{"line_number":7,"context_line":"ring: Create ReplicaToPartToDev helper"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Instead of a list of arrays, let\u0027s just have one contiguous"},{"line_number":10,"context_line":"multidimensional array."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Change-Id: If4941705b43f63ad1257704d716f8753d2613e35"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"201ae88e_25a969e5","line":10,"in_reply_to":"d0c136c0_a09aae22","updated":"2022-11-21 18:20:15.000000000","message":"I mean, yeah, that\u0027s what I *really* want -- but I don\u0027t want to add a numpy dependency to swift. Even the sdist is like 10MB!\n\nWell, also, I\u0027m not sure that numpy supports the ragged end we need for fractional replicas -- but maybe we could do something with NONE_DEVs?","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"733b5e15f18ba81482bb96dd4a150f442106e576","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"67a9c44f_d5b94422","updated":"2022-11-21 16:10:17.000000000","message":"this is pretty neat, I think I understand the motivation to be something to do with zipfile serialization - but unless it\u0027s actually faster or more efficient for rebalance or something I\u0027m not sure that the serialization format should be steering the in-memory layout.\n\nIt\u0027s possible I don\u0027t entirely understand the motivation, but even if it takes 2x memory to convert the optimal in-memory format for serialization - it might be worth it to get 2-10% faster rebalance.  Did you do any benchmarks?","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"476595b9f3fc549560c90bed650ee2633f4e7417","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"13d22e6d_41ce0018","in_reply_to":"67a9c44f_d5b94422","updated":"2022-11-21 18:20:15.000000000","message":"There are ancillary benefits (like guaranteeing that all replicas except the last one are the same size, and that the last replica must have a length in the range [1, partitions]) but yeah, mostly I was annoyed by the doubling of memory consumption while serializing.\n\nGiven that memoryview(array.array(\u0027H\u0027)) causes a TypeError on py2, maybe it\u0027s not worth it... I\u0027d have to write and test our own ArrayView or something 😞","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"}],"swift/common/ring/builder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6c1fe8fcd0a0c4b0d3c0b798f9f0a794f592e36c","unresolved":true,"context_lines":[{"line_number":1743,"context_line":"            if dev:"},{"line_number":1744,"context_line":"                dev.setdefault(\u0027replication_ip\u0027, dev[\u0027ip\u0027])"},{"line_number":1745,"context_line":"                dev.setdefault(\u0027replication_port\u0027, dev[\u0027port\u0027])"},{"line_number":1746,"context_line":"        return builder"},{"line_number":1747,"context_line":""},{"line_number":1748,"context_line":"    def save(self, builder_file):"},{"line_number":1749,"context_line":"        \"\"\"Serialize this RingBuilder instance to disk."}],"source_content_type":"text/x-python","patch_set":2,"id":"17fed4e7_a5cb38b5","line":1746,"updated":"2025-05-06 19:06:25.000000000","message":"Surely there should be some upgrade path around here for old builders...","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"}],"swift/common/ring/utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"733b5e15f18ba81482bb96dd4a150f442106e576","unresolved":true,"context_lines":[{"line_number":36,"context_line":"            replica +\u003d len(self)"},{"line_number":37,"context_line":"        start \u003d replica * self.partitions"},{"line_number":38,"context_line":"        end \u003d start + self.partitions"},{"line_number":39,"context_line":"        return memoryview(self.array)[start:end]"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def __iter__(self):"},{"line_number":42,"context_line":"        for i in range(len(self)):"}],"source_content_type":"text/x-python","patch_set":2,"id":"2de6f2b5_8472f860","line":39,"updated":"2022-11-21 16:10:17.000000000","message":"i wonder how manytimes we create/destroy these memoryview objects and what the overhead is compared to carrying around the list of arrays","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"476595b9f3fc549560c90bed650ee2633f4e7417","unresolved":true,"context_lines":[{"line_number":36,"context_line":"            replica +\u003d len(self)"},{"line_number":37,"context_line":"        start \u003d replica * self.partitions"},{"line_number":38,"context_line":"        end \u003d start + self.partitions"},{"line_number":39,"context_line":"        return memoryview(self.array)[start:end]"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def __iter__(self):"},{"line_number":42,"context_line":"        for i in range(len(self)):"}],"source_content_type":"text/x-python","patch_set":2,"id":"afeae904_ab68fc2a","line":39,"in_reply_to":"2de6f2b5_8472f860","updated":"2022-11-21 18:20:15.000000000","message":"Could probably get away with having a single memoryview per instance -- I forget why I didn\u0027t do that. But it\u0027s probably all moot \u0027cause py2...","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6c1fe8fcd0a0c4b0d3c0b798f9f0a794f592e36c","unresolved":true,"context_lines":[{"line_number":36,"context_line":"            replica +\u003d len(self)"},{"line_number":37,"context_line":"        start \u003d replica * self.partitions"},{"line_number":38,"context_line":"        end \u003d start + self.partitions"},{"line_number":39,"context_line":"        return memoryview(self.array)[start:end]"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def __iter__(self):"},{"line_number":42,"context_line":"        for i in range(len(self)):"}],"source_content_type":"text/x-python","patch_set":2,"id":"991752c2_90b6457f","line":39,"in_reply_to":"afeae904_ab68fc2a","updated":"2025-05-06 19:06:25.000000000","message":"Oh hey, we can revisit this now! But the `memoryview`-per-instance doesn\u0027t work out well in practice -- `deepcopy` and `pickle` don\u0027t play well with it. Even if I try to make allowances with something like\n```\n    def __init__(self, partitions, typecode, initializer):\n        self.partitions \u003d partitions\n        self.array \u003d array.array(typecode, initializer)\n        self._view \u003d None\n\n    @property\n    def view(self):\n        if self._view is None:\n            self._view \u003d memoryview(self.array)\n        return self._view\n\n    # The memory view doesn\u0027t play well with pickle\n    def __getstate__(self):\n        return {\n            \u0027partitions\u0027: self.partitions,\n            \u0027array\u0027: self.array,\n        }\n\n    def __setstate__(self, state):\n        self.partitions \u003d state[\u0027partitions\u0027]\n        self.array \u003d state[\u0027array\u0027]\n        # Let the property lazy-assign\n        self._view \u003d None\n```\nI get three test errors. The first,\n```\n    def test_replica_count(self):\n        self.assertEqual(self.ring.replica_count, 3)\n\u003e       self.ring._replica2part2dev_id.array.append(0)\nE       BufferError: cannot resize an array that is exporting buffers\n\ntest/unit/common/ring/test_ring.py:278: BufferError\n```\nI could maybe say, OK, maybe that\u0027s a test bug. Under what possible circumstance would we need to adjust the size of an instantiated `RingData`? But then there\u0027s also\n```\n        # There\u0027s 5 regions now, so the primary nodes + first 2 handoffs\n        # should span all 5 regions\n        part, devs \u003d r.get_nodes(\u0027a1\u0027, \u0027c1\u0027, \u0027o1\u0027)\n        primary_regions \u003d set([d[\u0027region\u0027] for d in devs])\n        primary_zones \u003d set([(d[\u0027region\u0027], d[\u0027zone\u0027]) for d in devs])\n        more_devs \u003d list(r.get_more_nodes(part))\n    \n        seen_regions \u003d set(primary_regions)\n        seen_regions.update([d[\u0027region\u0027] for d in more_devs[:2]])\n\u003e       self.assertEqual(seen_regions, set(range(0, 5)))\nE       AssertionError: Items in the second set but not the first:\nE       1\nE       2\nE       3\nE       4\n```\nand\n```\n    def test_set_replicas_decrease(self):\n        ,,,\n\u003e       rb.validate()\n...\n\u003e           raise exceptions.RingValidationError(\n                \u0027All partitions are not double accounted for: %d !\u003d %d\u0027 %\n                (parts_on_devs, parts_in_map))\nE           swift.common.exceptions.RingValidationError: All partitions are not double accounted for: 78 !\u003d 80\n```\nthat make no sense at all :-(","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"733b5e15f18ba81482bb96dd4a150f442106e576","unresolved":true,"context_lines":[{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def __iter__(self):"},{"line_number":42,"context_line":"        for i in range(len(self)):"},{"line_number":43,"context_line":"            yield self[i]"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"    def __eq__(self, other):"},{"line_number":46,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":2,"id":"8d97edc2_e3657ef0","line":43,"updated":"2022-11-21 16:10:17.000000000","message":"this is pretty wild!","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6c1fe8fcd0a0c4b0d3c0b798f9f0a794f592e36c","unresolved":false,"context_lines":[{"line_number":49,"context_line":"        except Exception:"},{"line_number":50,"context_line":"            # presumably other has no __len__"},{"line_number":51,"context_line":"            return False"},{"line_number":52,"context_line":"        return all(all(y \u003d\u003d z for y, z in zip(w, x))"},{"line_number":53,"context_line":"                   for w, x in zip(self, other))"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"    def __repr__(self):"}],"source_content_type":"text/x-python","patch_set":2,"id":"ee9e8697_353cfa79","line":52,"range":{"start_line":52,"start_character":42,"end_line":52,"end_character":45},"updated":"2025-05-06 19:06:25.000000000","message":"This should be a `zip_longest()`...","commit_id":"549f60081bb1286d5cf3c17e1380495396a4b948"}]}