)]}'
{"test/unit/obj/test_rpc_grpc.py":[{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"eed27e6c9692d00845d752a7c209951e10f5fb89","unresolved":false,"context_lines":[{"line_number":26,"context_line":"        self.socket_path \u003d \"/path/to/rpc.socket\""},{"line_number":27,"context_line":"        self.part_power \u003d 18"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"    def test_vfile_list_partitions(self):"},{"line_number":30,"context_line":"        raise Exception(self.ring)"},{"line_number":31,"context_line":"        with mock.patch(\"swift.obj.rpc_grpc.connections.connections\") as m_conn:"},{"line_number":32,"context_line":"            ret \u003d rpc_grpc.list_partitions(self.socket_path, self.part_power)"}],"source_content_type":"text/x-python","patch_set":2,"id":"bfb3d3c7_fd2a618e","line":29,"updated":"2019-05-17 16:04:56.000000000","message":"This module is mostly boilerplate for protobuf/grpc, so I think we just want to check it calls the index server with correct arguments.","commit_id":"49980b57a250f3f76323e8a512f8bdd8fe9fe2a1"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"db2cb77545541a773bb2488b4c448c94b74417ec","unresolved":false,"context_lines":[{"line_number":27,"context_line":"        self.part_power \u003d 18"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"    def test_vfile_list_partitions(self):"},{"line_number":30,"context_line":"        raise Exception(self.ring)"},{"line_number":31,"context_line":"        with mock.patch(\"swift.obj.rpc_grpc.connections.connections\") as m_conn:"},{"line_number":32,"context_line":"            ret \u003d rpc_grpc.list_partitions(self.socket_path, self.part_power)"},{"line_number":33,"context_line":"            arg \u003d fmgr_pb2.ListPartitionsInfo(partition_bits\u003dself.part_power)"}],"source_content_type":"text/x-python","patch_set":2,"id":"9fb8cfa7_b981e2fd","line":30,"range":{"start_line":30,"start_character":7,"end_line":30,"end_character":34},"updated":"2019-06-10 09:34:14.000000000","message":"Probably, this is left by mistake? Because, we don\u0027t define the self.ring and we don\u0027t have to call raise Exception here?","commit_id":"49980b57a250f3f76323e8a512f8bdd8fe9fe2a1"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"a58e38ed6b1674a9b49d2bb990450e78371d4e8a","unresolved":false,"context_lines":[{"line_number":27,"context_line":"        self.part_power \u003d 18"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"    def test_vfile_list_partitions(self):"},{"line_number":30,"context_line":"        raise Exception(self.ring)"},{"line_number":31,"context_line":"        with mock.patch(\"swift.obj.rpc_grpc.connections.connections\") as m_conn:"},{"line_number":32,"context_line":"            ret \u003d rpc_grpc.list_partitions(self.socket_path, self.part_power)"},{"line_number":33,"context_line":"            arg \u003d fmgr_pb2.ListPartitionsInfo(partition_bits\u003dself.part_power)"}],"source_content_type":"text/x-python","patch_set":2,"id":"9fb8cfa7_35439715","line":30,"range":{"start_line":30,"start_character":7,"end_line":30,"end_character":34},"in_reply_to":"9fb8cfa7_b981e2fd","updated":"2019-06-11 16:24:59.000000000","message":"Right, doesn\u0027t look good :/","commit_id":"49980b57a250f3f76323e8a512f8bdd8fe9fe2a1"}],"test/unit/obj/test_vfile.py":[{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"ac2916e4d1b9e8fb14a230d4625f35a95d1eeab5","unresolved":false,"context_lines":[{"line_number":29,"context_line":"    def list_partitions(self, socket_path, part_power):"},{"line_number":30,"context_line":"        # TODO: need review, I\u0027m not sure if the list_partition would return"},{"line_number":31,"context_line":"        # only part numbers (or path)"},{"line_number":32,"context_line":"        return [\"1\", \"2\", \"3\"]"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"class TestVFile(unittest.TestCase):"}],"source_content_type":"text/x-python","patch_set":1,"id":"bfb3d3c7_943221a7","line":32,"updated":"2019-05-17 14:07:22.000000000","message":"it returns the response from protobuf, in this case, unicode.\n(you can see the translation from the protobuf type to native type here: https://developers.google.com/protocol-buffers/docs/proto3)","commit_id":"265671603b0bde8ab121ae5d600aa3aacb5419f0"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"ac2916e4d1b9e8fb14a230d4625f35a95d1eeab5","unresolved":false,"context_lines":[{"line_number":43,"context_line":"    def tearDown(self):"},{"line_number":44,"context_line":"        rmtree(self.testdir, ignore_errors\u003d1)"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"    # TODO: perhaps, this is just a vfile.list_partitions? not a listdir?"},{"line_number":47,"context_line":"    def test_vfile_listdir(self):"},{"line_number":48,"context_line":"        with mock.patch(\"swift.obj.vfile.rpc\"):"},{"line_number":49,"context_line":"            ret \u003d vfile.listdir(self.object_dir)"}],"source_content_type":"text/x-python","patch_set":1,"id":"bfb3d3c7_b4ef25e8","line":46,"updated":"2019-05-17 14:07:22.000000000","message":"Yes, I think we want to test separately :list_partitions, list_partition, list_suffix etc..\n\nvfile.listdir() is \"routing\" to these separate calls based on the path","commit_id":"265671603b0bde8ab121ae5d600aa3aacb5419f0"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"eed27e6c9692d00845d752a7c209951e10f5fb89","unresolved":false,"context_lines":[{"line_number":28,"context_line":"import mock"},{"line_number":29,"context_line":""},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"class MockRpc(mock.MagicMock):"},{"line_number":32,"context_line":"    def list_partitions(self, socket_path, part_power):"},{"line_number":33,"context_line":"        # TODO: need review, I\u0027m not sure if the list_partition would return"},{"line_number":34,"context_line":"        # only part numbers (or path)"}],"source_content_type":"text/x-python","patch_set":2,"id":"bfb3d3c7_bd1e296e","line":31,"updated":"2019-05-17 16:04:56.000000000","message":"I left this as I think it will be used for other functions","commit_id":"49980b57a250f3f76323e8a512f8bdd8fe9fe2a1"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"eed27e6c9692d00845d752a7c209951e10f5fb89","unresolved":false,"context_lines":[{"line_number":51,"context_line":"                                    object_ring\u003dFakeRing(part_power\u003d18)),"},{"line_number":52,"context_line":"                      StoragePolicy(1, \u0027one\u0027, True,"},{"line_number":53,"context_line":"                                    object_ring\u003dFakeRing(part_power\u003d20))])"},{"line_number":54,"context_line":"    def test_vfile_listdir_partitions(self):"},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"        tests \u003d ["},{"line_number":57,"context_line":"            {\"path\": \"/srv/node/sda/objects\","}],"source_content_type":"text/x-python","patch_set":2,"id":"bfb3d3c7_a0fe0003","line":54,"updated":"2019-05-17 16:04:56.000000000","message":"In these I want to test that vfile.listdir() makes the correct RPC calls and gets the part power correctly from the ring (it is critical otherwise the index server will give incorrect partitions)","commit_id":"49980b57a250f3f76323e8a512f8bdd8fe9fe2a1"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"db2cb77545541a773bb2488b4c448c94b74417ec","unresolved":false,"context_lines":[{"line_number":59,"context_line":"            {\"path\": \"/sdb/objects\","},{"line_number":60,"context_line":"             \"expected\": [\"/sdb/losf/rpc.socket\", 18]},"},{"line_number":61,"context_line":"            {\"path\": \"/sdc/objects-1\","},{"line_number":62,"context_line":"             \"expected\": [\"/sdc/losf-1/rpc.socket\", 20]},"},{"line_number":63,"context_line":"        ]"},{"line_number":64,"context_line":"        with mock.patch("},{"line_number":65,"context_line":"                \"swift.obj.vfile.rpc.list_partitions\") as m_list_partitions:"}],"source_content_type":"text/x-python","patch_set":2,"id":"9fb8cfa7_d96ff6f9","line":62,"updated":"2019-06-10 09:34:14.000000000","message":"Probably, I have to peal out the magic of index server here and why the part power is required for the rpc. I reached out to golang index server [1] that wants to hash the key with specific part_power, maybe it\u0027s as well as Swift object key hashing.\n\nMy questions are basically 2 points:\n\n1.why the hashing is placed in index server? In my understanding, the required prefix (or key itself?) can be calculated by vfile so we may not have to pass the part power via grpc. (but i understand the protocl change affects your production environ tho, just curious)\n\n2.Does it have the risk of key conflict when more than 1 policies have same partition power. if the key in the leveldb consists from just a part_power and the prefix, and then, 2 different ring with same part (the devices and rebalance stats are different) deployed with losf diskfile in a Swift cluster, one policy may overwrite the partition stat of the other policy on the backend leveldb... is that right?\n   \n\n1: https://github.com/openstack/swift/blob/feature/losf/go/swift-rpc-losf/encoding.go#L116-L140","commit_id":"49980b57a250f3f76323e8a512f8bdd8fe9fe2a1"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"a58e38ed6b1674a9b49d2bb990450e78371d4e8a","unresolved":false,"context_lines":[{"line_number":59,"context_line":"            {\"path\": \"/sdb/objects\","},{"line_number":60,"context_line":"             \"expected\": [\"/sdb/losf/rpc.socket\", 18]},"},{"line_number":61,"context_line":"            {\"path\": \"/sdc/objects-1\","},{"line_number":62,"context_line":"             \"expected\": [\"/sdc/losf-1/rpc.socket\", 20]},"},{"line_number":63,"context_line":"        ]"},{"line_number":64,"context_line":"        with mock.patch("},{"line_number":65,"context_line":"                \"swift.obj.vfile.rpc.list_partitions\") as m_list_partitions:"}],"source_content_type":"text/x-python","patch_set":2,"id":"9fb8cfa7_f5c3bf87","line":62,"in_reply_to":"9fb8cfa7_d96ff6f9","updated":"2019-06-11 16:24:59.000000000","message":"The function you mention in the index server [1] does not need the part_power.\nIt will take a md5 in ASCII format, and turn it to bytes, so that it uses 16 bytes instead of 32.\n\n1) Very little magic, I hope I can explain it clearly:\nThe part power is required only for a few functions.\n\nWhat do we need it for:\n- when we want to list partitions: we do not store partitions in the index-server, only \u003cobject-hash\u003e\u003cfilename\u003e. So, to list the partitions, we start from the first key in the database, extract the partition (we need part_power here). Then we skip to the next key that has a different partition (need part_power again). And at the end of that process we have a list of all partitions.\n- When we want to list suffix within a partition. We need to get all suffixes from the first key in the partition, to the last key in the partition.\n\nIn theory we could have the python code iterate over keys and run this algorithm. In practice it is way too slow, so we have a call in the RPC server that is \"list_partitions()\", and it must have the part_power to do its work.\n\nWe have considered other options, I\u0027m happy to discuss them if you wish.\n\nThere are slides here, page 19 and 20 that explain that: https://fr.slideshare.net/AlexandreLecuyer/openstack-swift-lots-of-small-files \n\nWe can discuss it online if I failed to explain clearly or if you have other questions.\n\n2) there is one index server per disk, per policy. You would have /srv/node/losf and /srv/node/losf-1 so no risk of conflict.","commit_id":"49980b57a250f3f76323e8a512f8bdd8fe9fe2a1"}]}