)]}'
{"swift/common/direct_client.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":74,"context_line":"    if older_than:"},{"line_number":75,"context_line":"        qs +\u003d \u0027\u0026older_than\u003d%s\u0027 % quote(older_than)"},{"line_number":76,"context_line":"    if policy_index is not None:"},{"line_number":77,"context_line":"        qs +\u003d \u0027\u0026policy_index\u003d%d\u0027 % policy_index"},{"line_number":78,"context_line":"    with Timeout(conn_timeout):"},{"line_number":79,"context_line":"        conn \u003d http_connect(node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027], part,"},{"line_number":80,"context_line":"                            \u0027GET\u0027, path, query_string\u003dqs,"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_f94aaeab","line":77,"updated":"2015-08-05 07:56:58.000000000","message":"these are all new public interfaces - correct?  exposed to the client as well?","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":74,"context_line":"    if older_than:"},{"line_number":75,"context_line":"        qs +\u003d \u0027\u0026older_than\u003d%s\u0027 % quote(older_than)"},{"line_number":76,"context_line":"    if policy_index is not None:"},{"line_number":77,"context_line":"        qs +\u003d \u0027\u0026policy_index\u003d%d\u0027 % policy_index"},{"line_number":78,"context_line":"    with Timeout(conn_timeout):"},{"line_number":79,"context_line":"        conn \u003d http_connect(node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027], part,"},{"line_number":80,"context_line":"                            \u0027GET\u0027, path, query_string\u003dqs,"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_952a6f8f","line":77,"in_reply_to":"1a4dcd0f_f94aaeab","updated":"2015-08-10 21:27:40.000000000","message":"I do not want to expose these interfaces \u0027public\u0027. I want to expose them in only account/container servers. But I have to pay attentions that this does not expose these parameters to client side. Thanks.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"}],"swift/common/middleware/change_policy.py":[{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"1e1674d81081f6b6c83b1817e7cfd666b5212e01","unresolved":false,"context_lines":[{"line_number":86,"context_line":"                current, prev \u003d self._get_policy_indexes(req)"},{"line_number":87,"context_line":"                # Previous policy index of a container is stored as sysmeta if"},{"line_number":88,"context_line":"                # a policy changing status is \"Running\""},{"line_number":89,"context_line":"                if prev is not None:"},{"line_number":90,"context_line":"                    return HTTPConflict(request\u003dreq,"},{"line_number":91,"context_line":"                                        body\u003d\"Previous changing policy \""},{"line_number":92,"context_line":"                                             \"process is not finished yet.\")"},{"line_number":93,"context_line":"                if req_policy_idx !\u003d current:"},{"line_number":94,"context_line":"                    req.headers.update({"},{"line_number":95,"context_line":"                        \u0027X-Backend-Storage-Policy-Index\u0027: req_policy_idx,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5aef4532_b671d978","line":92,"range":{"start_line":89,"start_character":0,"end_line":92,"end_character":76},"updated":"2016-03-09 11:25:37.000000000","message":"IMO, the response should have the information about the policy which the container will be changed to.","commit_id":"68bdfdb60e7383a3d213fcd630b445c877bda561"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"540e226a7d342a255a7b120235d6d12042a84485","unresolved":false,"context_lines":[{"line_number":86,"context_line":"                current, prev \u003d self._get_policy_indexes(req)"},{"line_number":87,"context_line":"                # Previous policy index of a container is stored as sysmeta if"},{"line_number":88,"context_line":"                # a policy changing status is \"Running\""},{"line_number":89,"context_line":"                if prev is not None:"},{"line_number":90,"context_line":"                    return HTTPConflict(request\u003dreq,"},{"line_number":91,"context_line":"                                        body\u003d\"Previous changing policy \""},{"line_number":92,"context_line":"                                             \"process is not finished yet.\")"},{"line_number":93,"context_line":"                if req_policy_idx !\u003d current:"},{"line_number":94,"context_line":"                    req.headers.update({"},{"line_number":95,"context_line":"                        \u0027X-Backend-Storage-Policy-Index\u0027: req_policy_idx,"}],"source_content_type":"text/x-python","patch_set":8,"id":"9a061dce_968e392c","line":92,"range":{"start_line":89,"start_character":0,"end_line":92,"end_character":76},"in_reply_to":"5aef4532_b671d978","updated":"2016-04-06 22:47:49.000000000","message":"Sorry I cannot understand your comment precisely. Basically, HEAD response to the container contains information about previous policy name and current policy name. I think that information satisfies your needs.","commit_id":"68bdfdb60e7383a3d213fcd630b445c877bda561"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"6e84f0bd2a74201c04529f1851083309e7789c53","unresolved":false,"context_lines":[{"line_number":86,"context_line":"                current, prev \u003d self._get_policy_indexes(req)"},{"line_number":87,"context_line":"                # Previous policy index of a container is stored as sysmeta if"},{"line_number":88,"context_line":"                # a policy changing status is \"Running\""},{"line_number":89,"context_line":"                if prev is not None:"},{"line_number":90,"context_line":"                    return HTTPConflict(request\u003dreq,"},{"line_number":91,"context_line":"                                        body\u003d\"Previous changing policy \""},{"line_number":92,"context_line":"                                             \"process is not finished yet.\")"},{"line_number":93,"context_line":"                if req_policy_idx !\u003d current:"},{"line_number":94,"context_line":"                    req.headers.update({"},{"line_number":95,"context_line":"                        \u0027X-Backend-Storage-Policy-Index\u0027: req_policy_idx,"}],"source_content_type":"text/x-python","patch_set":8,"id":"9a061dce_7876efd7","line":92,"range":{"start_line":89,"start_character":0,"end_line":92,"end_character":76},"in_reply_to":"9a061dce_968e392c","updated":"2016-04-07 01:28:20.000000000","message":"Thank you for your reply and I\u0027m sorry for my misunderstanding.\nYou are right. The information in HEAD response satisfies me.","commit_id":"68bdfdb60e7383a3d213fcd630b445c877bda561"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"a37645be9c212a4069629506e065f820370e27fb","unresolved":false,"context_lines":[{"line_number":95,"context_line":"                        \u0027X-Backend-Storage-Policy-Index\u0027: req_policy_idx,"},{"line_number":96,"context_line":"                        self.prev_idx_header: current,"},{"line_number":97,"context_line":"                        self.obj_queued_header: False})"},{"line_number":98,"context_line":"        # handle for Container GET/HEAD"},{"line_number":99,"context_line":"        elif container and not obj and req.method in (\u0027GET\u0027, \u0027HEAD\u0027):"},{"line_number":100,"context_line":"            _, prev \u003d self._get_policy_indexes(req)"},{"line_number":101,"context_line":"            if prev is not None:"},{"line_number":102,"context_line":"                # Use X-Backend-Storage-Policy-Index header for conveying"},{"line_number":103,"context_line":"                # a previous policy index"},{"line_number":104,"context_line":"                req.headers.update({\u0027X-Backend-Storage-Policy-Index\u0027: prev})"},{"line_number":105,"context_line":"        # handle for Object GET/HEAD and DELETE"},{"line_number":106,"context_line":"        elif obj and req.method in (\u0027GET\u0027, \u0027HEAD\u0027, \u0027DELETE\u0027):"},{"line_number":107,"context_line":"            current, prev \u003d self._get_policy_indexes(req)"}],"source_content_type":"text/x-python","patch_set":8,"id":"9a061dce_5eb2f306","line":104,"range":{"start_line":98,"start_character":0,"end_line":104,"end_character":76},"updated":"2016-04-07 02:49:56.000000000","message":"In my understanding, if users PUT object to the container during policy changing, the object will be stored in the new policy.\nHowever, it seems that users get a list of objects only in prev policy when users GET the container during policy changing.\nTherefore, before finish of policy changing, users cannot confirm that the object is in the container by GETting container.\nIs this by design?","commit_id":"68bdfdb60e7383a3d213fcd630b445c877bda561"},{"author":{"_account_id":860,"name":"Donagh McCabe","email":"donagh.mccabe@gmail.com","username":"donagh-mccabe"},"change_message_id":"3beafd54e9d4f10f7dfbd43ab77b601d3bb506d2","unresolved":false,"context_lines":[{"line_number":90,"context_line":"            req_policy_idx \u003d self._convert_policy_to_index(req)"},{"line_number":91,"context_line":"            if req_policy_idx is not None:"},{"line_number":92,"context_line":"                # Changing Policies operation is only allowed by swift_owner"},{"line_number":93,"context_line":"                if req.environ.get(\u0027reseller_request\u0027) is not True:"},{"line_number":94,"context_line":"                    return HTTPForbidden()"},{"line_number":95,"context_line":"                current, prev \u003d self._get_policy_indexes(req)"},{"line_number":96,"context_line":"                # Previous policy index of a container is stored as sysmeta if"}],"source_content_type":"text/x-python","patch_set":13,"id":"dada55a8_c46474ba","line":93,"range":{"start_line":93,"start_character":36,"end_line":93,"end_character":52},"updated":"2016-07-20 16:22:21.000000000","message":"The comment is changes, but not the code?","commit_id":"a67db3efcc8c3c29ed6599e3a3d541dd6ce3a32c"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"1b06d87fab8d5f6ba813c89f7e38d24810133024","unresolved":false,"context_lines":[{"line_number":90,"context_line":"            req_policy_idx \u003d self._convert_policy_to_index(req)"},{"line_number":91,"context_line":"            if req_policy_idx is not None:"},{"line_number":92,"context_line":"                # Changing Policies operation is only allowed by swift_owner"},{"line_number":93,"context_line":"                if req.environ.get(\u0027reseller_request\u0027) is not True:"},{"line_number":94,"context_line":"                    return HTTPForbidden()"},{"line_number":95,"context_line":"                current, prev \u003d self._get_policy_indexes(req)"},{"line_number":96,"context_line":"                # Previous policy index of a container is stored as sysmeta if"}],"source_content_type":"text/x-python","patch_set":13,"id":"dada55a8_f46b25b7","line":93,"range":{"start_line":93,"start_character":36,"end_line":93,"end_character":52},"in_reply_to":"dada55a8_c46474ba","updated":"2016-07-20 17:55:29.000000000","message":"Nice catch. Actually, I wonder which is better when I write this patch. Now, I am going to revert above comment to reseller_admin.","commit_id":"a67db3efcc8c3c29ed6599e3a3d541dd6ce3a32c"}],"swift/container/backend.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":613,"context_line":"                    query_args.append(prefix)"},{"line_number":614,"context_line":"                if older_than:"},{"line_number":615,"context_line":"                    query +\u003d \u0027 created_at \u003c ? AND\u0027"},{"line_number":616,"context_line":"                    query_args.append(older_than)"},{"line_number":617,"context_line":"                if self.get_db_version(conn) \u003c 1:"},{"line_number":618,"context_line":"                    query +\u003d \u0027 +deleted \u003d 0\u0027"},{"line_number":619,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_39d816ef","line":616,"updated":"2015-08-05 07:56:58.000000000","message":"this column is not indexed - should be very slow/expensive","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":613,"context_line":"                    query_args.append(prefix)"},{"line_number":614,"context_line":"                if older_than:"},{"line_number":615,"context_line":"                    query +\u003d \u0027 created_at \u003c ? AND\u0027"},{"line_number":616,"context_line":"                    query_args.append(older_than)"},{"line_number":617,"context_line":"                if self.get_db_version(conn) \u003c 1:"},{"line_number":618,"context_line":"                    query +\u003d \u0027 +deleted \u003d 0\u0027"},{"line_number":619,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_b018d976","line":616,"in_reply_to":"1a4dcd0f_39d816ef","updated":"2015-08-10 21:27:40.000000000","message":"I got it. I will consider this point.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":619,"context_line":"                else:"},{"line_number":620,"context_line":"                    query +\u003d \u0027 deleted \u003d 0\u0027"},{"line_number":621,"context_line":"                orig_tail_query \u003d \u0027\u0027\u0027"},{"line_number":622,"context_line":"                    ORDER BY name, created_at DESC LIMIT ?"},{"line_number":623,"context_line":"                \u0027\u0027\u0027"},{"line_number":624,"context_line":"                orig_tail_args \u003d [limit - len(results)]"},{"line_number":625,"context_line":"                # storage policy filter"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_f9f86e4b","line":622,"updated":"2015-08-05 07:56:58.000000000","message":"does this change just not have any effect on the ordering of the listing or are our functional tests really that loose?","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":619,"context_line":"                else:"},{"line_number":620,"context_line":"                    query +\u003d \u0027 deleted \u003d 0\u0027"},{"line_number":621,"context_line":"                orig_tail_query \u003d \u0027\u0027\u0027"},{"line_number":622,"context_line":"                    ORDER BY name, created_at DESC LIMIT ?"},{"line_number":623,"context_line":"                \u0027\u0027\u0027"},{"line_number":624,"context_line":"                orig_tail_args \u003d [limit - len(results)]"},{"line_number":625,"context_line":"                # storage policy filter"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_f091819b","line":622,"in_reply_to":"1a4dcd0f_f9f86e4b","updated":"2015-08-10 21:27:40.000000000","message":"In container/server.py, I will dedup list by name so this does not effect on current functional tests. Of course, I must add functional tests to check latest object\u0027s meta info is available.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":624,"context_line":"                orig_tail_args \u003d [limit - len(results)]"},{"line_number":625,"context_line":"                # storage policy filter"},{"line_number":626,"context_line":"                policy_tail_query \u003d \u0027 AND storage_policy_index IN(%s)\u0027 % \\"},{"line_number":627,"context_line":"                    \u0027,\u0027.join(storage_policy_index) + orig_tail_query"},{"line_number":628,"context_line":"                policy_tail_args \u003d orig_tail_args"},{"line_number":629,"context_line":"                tail_query, tail_args \u003d \\"},{"line_number":630,"context_line":"                    policy_tail_query, policy_tail_args"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_f4f2b5d1","line":627,"updated":"2015-08-05 07:56:58.000000000","message":"this column is also un-indexed - when this condition filters more rows than it selects it can potentially rip through many pages.\n\nI\u0027m not sure I\u0027m comfortable with this.  How will it be used?\n\nIt would be helpful to examine the results of these queries through sqlite\u0027s explain - and potentially publishing those results.  Perhaps more useful would be to run benchmarks and publish those.\n\nI would start with 1M rows.\n\nEarly on when most rows are in the \"wrong\" storage policy and created_at is near the bottom - I think you\u0027ll get 10K rows out pretty quickly.\n\nOnce the transfer has been running for awhile and most rows are in the \"right\" storage policy (i.e. being filtered by the queries from the transfer to \"find work to do\") and the created_at param is closer to the end - it will become more expensive to respond with 10K rows (perhaps prohibitively expensive).\n\nIf the object transfers can lock up large databases for 100\u0027s of ms I think that could cause a problem for object writes and replication.  If the slowdown scales linearly up to multiple seconds as the container grows to 4-6M objects - it could potentially cause replication or async pendings to \"loose ground\" - which would be a bummer, but you might just be able to shut down the transferrer.  It\u0027d be a bigger problem if there\u0027s no controls on these interfaces and a misbehaved client could trivially interfere with the consistency engine by abusing an inefficient interface.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":624,"context_line":"                orig_tail_args \u003d [limit - len(results)]"},{"line_number":625,"context_line":"                # storage policy filter"},{"line_number":626,"context_line":"                policy_tail_query \u003d \u0027 AND storage_policy_index IN(%s)\u0027 % \\"},{"line_number":627,"context_line":"                    \u0027,\u0027.join(storage_policy_index) + orig_tail_query"},{"line_number":628,"context_line":"                policy_tail_args \u003d orig_tail_args"},{"line_number":629,"context_line":"                tail_query, tail_args \u003d \\"},{"line_number":630,"context_line":"                    policy_tail_query, policy_tail_args"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_f0d661c2","line":627,"in_reply_to":"1a4dcd0f_f4f2b5d1","updated":"2015-08-10 21:27:40.000000000","message":"I understand your concerns. I will focus on sqlite\u0027s performance issues more. The purpose of this fix is to get objects whether they are already migrated or not.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"}],"swift/container/server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":493,"context_line":"        if isinstance(policy_index, tuple) and len(policy_index) \u003e 1:"},{"line_number":494,"context_line":"            # Dedup"},{"line_number":495,"context_line":"            seen \u003d set()"},{"line_number":496,"context_line":"            container_list \u003d [x for x in container_list if x[0] not in seen and not seen.add(x[0])] "},{"line_number":497,"context_line":"        return self.create_listing(req, out_content_type, info, resp_headers,"},{"line_number":498,"context_line":"                                   broker.metadata, container_list, container)"},{"line_number":499,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_cfba84c0","line":496,"updated":"2015-08-05 07:56:58.000000000","message":"so if you have some objects in sp0 and some in sp1 - the idea is you only want to see the name once?  If we\u0027re going to do this work anyway; can we just not do the filter in the database?  get back all the rows and then flatten them?  Or going the other way - make the database do the unique on the name field (it\u0027s already indexed).","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":493,"context_line":"        if isinstance(policy_index, tuple) and len(policy_index) \u003e 1:"},{"line_number":494,"context_line":"            # Dedup"},{"line_number":495,"context_line":"            seen \u003d set()"},{"line_number":496,"context_line":"            container_list \u003d [x for x in container_list if x[0] not in seen and not seen.add(x[0])] "},{"line_number":497,"context_line":"        return self.create_listing(req, out_content_type, info, resp_headers,"},{"line_number":498,"context_line":"                                   broker.metadata, container_list, container)"},{"line_number":499,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_684b452c","line":496,"in_reply_to":"1a4dcd0f_cfba84c0","updated":"2015-08-10 21:27:40.000000000","message":"In backend.py fix, we sort container list by name (ascending) and created timestamp (descending), so I would like to cut older objects whose name are the same in this clause. I can move this filter in broker.list_objects_iter method.\n\nHowever, I \u0027can\u0027 write SQL to do so but it might be too complicated and too expensive so i do not want to filter the list in DB side.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":593,"context_line":"        if new_policy_index is not None and src_policy_index !\u003d \"\":"},{"line_number":594,"context_line":"            return HTTPConflict(request\u003dreq,"},{"line_number":595,"context_line":"                                body\u003d\"Previous changing policy process is \""},{"line_number":596,"context_line":"                                     \"not finished yet.\")"},{"line_number":597,"context_line":"        elif (new_policy_index is not None and"},{"line_number":598,"context_line":"              new_policy_index !\u003d broker.storage_policy_index):"},{"line_number":599,"context_line":"            # Change storage policy and store previous policy idx as sysmeta"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_2ffbe8c2","line":596,"updated":"2015-08-05 07:56:58.000000000","message":"it\u0027s seems perhaps not all nodes will be able to consistently return a 409, what happens if one node writes down a new policy and others reject it?  I\u0027d guess the newest timestamp for the sysmeta update would win?","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":593,"context_line":"        if new_policy_index is not None and src_policy_index !\u003d \"\":"},{"line_number":594,"context_line":"            return HTTPConflict(request\u003dreq,"},{"line_number":595,"context_line":"                                body\u003d\"Previous changing policy process is \""},{"line_number":596,"context_line":"                                     \"not finished yet.\")"},{"line_number":597,"context_line":"        elif (new_policy_index is not None and"},{"line_number":598,"context_line":"              new_policy_index !\u003d broker.storage_policy_index):"},{"line_number":599,"context_line":"            # Change storage policy and store previous policy idx as sysmeta"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_e8bbd573","line":596,"in_reply_to":"1a4dcd0f_2ffbe8c2","updated":"2015-08-10 21:27:40.000000000","message":"Thanks for suggestion. It is a very important point so I have to think out how to handle this negative case.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"}],"swift/obj/transferrer.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":117,"context_line":"        :param older_than: filter objects to be older than this timestamp"},{"line_number":118,"context_line":"        :param policy_index: filter objects to be stored by this policy"},{"line_number":119,"context_line":"        :returns: a tuple of (response headers, response body)"},{"line_number":120,"context_line":"        \"\"\""},{"line_number":121,"context_line":"        def _eat_client_exception(*args, **kwargs):"},{"line_number":122,"context_line":"            try:"},{"line_number":123,"context_line":"                return direct_get_container(*args,"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_d4e1b9d1","line":120,"updated":"2015-08-05 07:56:58.000000000","message":"are you sure doing all these backend connections by hand is better than using the internal client?  I tend to think about things like node error limiting, timeout configuration, stuff that proxy has been taught to do over the years that make it really good at talking to backend nodes.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":117,"context_line":"        :param older_than: filter objects to be older than this timestamp"},{"line_number":118,"context_line":"        :param policy_index: filter objects to be stored by this policy"},{"line_number":119,"context_line":"        :returns: a tuple of (response headers, response body)"},{"line_number":120,"context_line":"        \"\"\""},{"line_number":121,"context_line":"        def _eat_client_exception(*args, **kwargs):"},{"line_number":122,"context_line":"            try:"},{"line_number":123,"context_line":"                return direct_get_container(*args,"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_4b5d2314","line":120,"in_reply_to":"1a4dcd0f_d4e1b9d1","updated":"2015-08-10 21:27:40.000000000","message":"Thanks for your comments. As you suggested, it is better to use the InternalClient in this part. The reason why I avoided to use InternalClient is because I would not like to expose \u0027older_than\u0027 and \u0027policy_index\u0027 parameter to proxy. My standing point where I would not like to expose these new interfaces to client side is not changed now, but I will think out the way to remove hand-made connections like below without exposing new interfaces to client side.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":170,"context_line":"            # Each transferrer is responsible for sending objects or parts"},{"line_number":171,"context_line":"            # of objects stored in its local devices"},{"line_number":172,"context_line":"            if is_local_device(ips, self.port, node[\"ip\"], node[\"port\"]):"},{"line_number":173,"context_line":"                dev_path \u003d join(self.devices_dir, node[\u0027device\u0027])"},{"line_number":174,"context_line":"                if self.mount_check and not ismount(dev_path):"},{"line_number":175,"context_line":"                    self.logger.warn(_(\u0027%s is not mounted\u0027), node[\u0027device\u0027])"},{"line_number":176,"context_line":"                    continue"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_3441fda9","line":173,"updated":"2015-08-05 07:56:58.000000000","message":"is the *part* for this object pre-filtered before it get\u0027s to this method to make sure that at least *one* of replicas of this object are on this node - or in that case would we just loop over all the nodes and find that node of the devices are local and return None (way down at the bottom)?","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":170,"context_line":"            # Each transferrer is responsible for sending objects or parts"},{"line_number":171,"context_line":"            # of objects stored in its local devices"},{"line_number":172,"context_line":"            if is_local_device(ips, self.port, node[\"ip\"], node[\"port\"]):"},{"line_number":173,"context_line":"                dev_path \u003d join(self.devices_dir, node[\u0027device\u0027])"},{"line_number":174,"context_line":"                if self.mount_check and not ismount(dev_path):"},{"line_number":175,"context_line":"                    self.logger.warn(_(\u0027%s is not mounted\u0027), node[\u0027device\u0027])"},{"line_number":176,"context_line":"                    continue"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_0b34fbf8","line":173,"in_reply_to":"1a4dcd0f_3441fda9","updated":"2015-08-10 21:27:40.000000000","message":"Your comment is right. I just would like to make sure that at least *one* replica is on this node. So I can probably remove this if-statement with accomplishing this objective according to your comment. Thanks.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":239,"context_line":"                                                      sc[\u0027name\u0027])"},{"line_number":240,"context_line":"            container_path \u003d cmeta.get(\u0027x-container-sysmeta-uri\u0027)"},{"line_number":241,"context_line":"            prev_policy_idx \u003d int("},{"line_number":242,"context_line":"                cmeta.get(\u0027x-container-sysmeta-prev-storage-policy-index\u0027))"},{"line_number":243,"context_line":"            # Time when a special container is created is the same as"},{"line_number":244,"context_line":"            # the requested time of policy changing"},{"line_number":245,"context_line":"            last_put_timestamp \u003d cmeta.get(\u0027x-backend-put-timestamp\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_cfeda426","line":242,"updated":"2015-08-05 07:56:58.000000000","message":"what if the container has misplaced objects - I\u0027d guess the reconciler will just move them into the current \u0027x-backend-storage-policy-index\u0027?  In fact - what prevents the reconciler from just doing the transferrers job for it?","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":239,"context_line":"                                                      sc[\u0027name\u0027])"},{"line_number":240,"context_line":"            container_path \u003d cmeta.get(\u0027x-container-sysmeta-uri\u0027)"},{"line_number":241,"context_line":"            prev_policy_idx \u003d int("},{"line_number":242,"context_line":"                cmeta.get(\u0027x-container-sysmeta-prev-storage-policy-index\u0027))"},{"line_number":243,"context_line":"            # Time when a special container is created is the same as"},{"line_number":244,"context_line":"            # the requested time of policy changing"},{"line_number":245,"context_line":"            last_put_timestamp \u003d cmeta.get(\u0027x-backend-put-timestamp\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_0e82c452","line":242,"in_reply_to":"1a4dcd0f_cfeda426","updated":"2015-08-10 21:27:40.000000000","message":"Misplaced objects will be handled as you suggested.\n\nReconciler just sends objects from \u0027previous\u0027 to \u0027current\u0027 policy so I can use the sending objects part. However, objects placed in \u0027previous\u0027 policy should be accessible in the status of policy migration, and the existing reconciling logic does not have this point. Moreover, container\u0027s link to previous policy should be removed after the completion of policy migration. Reconciler cannot determine it.\n\nJust an idea to reuse reconciler functions for policy migration is to create a daemon to *periodically* check if policy migration is termintated or not by querying \"are there remained objects whose policy should be migrated\" to container DB. Thoghts?","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":245,"context_line":"            last_put_timestamp \u003d cmeta.get(\u0027x-backend-put-timestamp\u0027)"},{"line_number":246,"context_line":"            acc_con \u003d split_path(container_path, 2, 2)"},{"line_number":247,"context_line":"            headers, objs \u003d self._get_transferred_objects("},{"line_number":248,"context_line":"                acc_con[0], acc_con[1], last_put_timestamp, prev_policy_idx)"},{"line_number":249,"context_line":"            new_policy_idx \u003d int(headers.get(\u0027x-backend-storage-policy-index\u0027))"},{"line_number":250,"context_line":"            if objs and prev_policy_idx !\u003d new_policy_idx:"},{"line_number":251,"context_line":"                for obj in objs:"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_1403a110","line":248,"updated":"2015-08-05 07:56:58.000000000","message":"so it seems like every transferrer will get a list of objects to be transferred from every container that\u0027s currently being moved in the whole cluster all the time - that might be ok - is that right?\n\nI guess the other way would be to keep a set of containers being transferred in mind as it walks across all the objects - that seems worse.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":245,"context_line":"            last_put_timestamp \u003d cmeta.get(\u0027x-backend-put-timestamp\u0027)"},{"line_number":246,"context_line":"            acc_con \u003d split_path(container_path, 2, 2)"},{"line_number":247,"context_line":"            headers, objs \u003d self._get_transferred_objects("},{"line_number":248,"context_line":"                acc_con[0], acc_con[1], last_put_timestamp, prev_policy_idx)"},{"line_number":249,"context_line":"            new_policy_idx \u003d int(headers.get(\u0027x-backend-storage-policy-index\u0027))"},{"line_number":250,"context_line":"            if objs and prev_policy_idx !\u003d new_policy_idx:"},{"line_number":251,"context_line":"                for obj in objs:"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_0edde413","line":248,"in_reply_to":"1a4dcd0f_1403a110","updated":"2015-08-10 21:27:40.000000000","message":"Transferrer\u0027s behavior you explained is right.\n\nSorry, your \"the other way\" does not make sense to me. After migrating all objects to be transferred from one container, that container info will be removed in the line 256-257.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"b4184b4bb1c91078abf41e8ae77db034761fe4d9","unresolved":false,"context_lines":[{"line_number":210,"context_line":"        \"\"\""},{"line_number":211,"context_line":"        for sc in self._iter_containers():"},{"line_number":212,"context_line":"            empty \u003d True"},{"line_number":213,"context_line":"            acc, con \u003d sc[\u0027name\u0027].split(\u0027:\u0027)"},{"line_number":214,"context_line":"            for obj in self.swift.iter_objects(POLICY_CHANGE_ACCOUNT,"},{"line_number":215,"context_line":"                                               sc[\u0027name\u0027]):"},{"line_number":216,"context_line":"                empty \u003d False"}],"source_content_type":"text/x-python","patch_set":7,"id":"7af24918_12019538","line":213,"range":{"start_line":213,"start_character":12,"end_line":213,"end_character":44},"updated":"2016-03-03 13:02:50.000000000","message":"If sc[\u0027name\u0027] has two \u0027:\u0027 characters, the result of sc[\u0027name\u0027].split(\u0027:\u0027) is a triple.\nThen, \"ValueError: too many values to unpack\" will be caused.\n\nThe implementation should be\n\"acc, con \u003d sc[\u0027name\u0027].split(\u0027:\u0027, 1)\"","commit_id":"a858b422018feb9f1be62f43ad591860899e1824"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"4d123c4659f68c397ee1e63dcb587c49a91f0eb1","unresolved":false,"context_lines":[{"line_number":36,"context_line":"    transfer_type \u003d \u0027application/x-transfer-%s-to-%s\u0027 % (prev_idx, current_idx)"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"    def translator(obj_info):"},{"line_number":39,"context_line":"        nextts \u003d slightly_later_timestamp(obj_info[1])"},{"line_number":40,"context_line":"        return {"},{"line_number":41,"context_line":"            \u0027name\u0027: obj_info[0], \u0027deleted\u0027: 0, \u0027created_at\u0027: nextts,"},{"line_number":42,"context_line":"            \u0027etag\u0027: nextts, \u0027content_type\u0027: transfer_type, \u0027size\u0027: 0,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5aef4532_730db136","line":39,"range":{"start_line":39,"start_character":8,"end_line":39,"end_character":54},"updated":"2016-03-09 06:44:33.000000000","message":"It seems that the transferrer will use the x-timestamp incremented twice.\nWhy is this?","commit_id":"68bdfdb60e7383a3d213fcd630b445c877bda561"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"540e226a7d342a255a7b120235d6d12042a84485","unresolved":false,"context_lines":[{"line_number":36,"context_line":"    transfer_type \u003d \u0027application/x-transfer-%s-to-%s\u0027 % (prev_idx, current_idx)"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"    def translator(obj_info):"},{"line_number":39,"context_line":"        nextts \u003d slightly_later_timestamp(obj_info[1])"},{"line_number":40,"context_line":"        return {"},{"line_number":41,"context_line":"            \u0027name\u0027: obj_info[0], \u0027deleted\u0027: 0, \u0027created_at\u0027: nextts,"},{"line_number":42,"context_line":"            \u0027etag\u0027: nextts, \u0027content_type\u0027: transfer_type, \u0027size\u0027: 0,"}],"source_content_type":"text/x-python","patch_set":8,"id":"9a061dce_d65a71f1","line":39,"range":{"start_line":39,"start_character":8,"end_line":39,"end_character":54},"in_reply_to":"5aef4532_730db136","updated":"2016-04-06 22:47:49.000000000","message":"Thanks for your review. That is my mistake. I will reflect your comment in the next patch.","commit_id":"68bdfdb60e7383a3d213fcd630b445c877bda561"},{"author":{"_account_id":14766,"name":"Kazuhiro MIYAHARA","email":"miyahara.kazuhiro@lab.ntt.co.jp","username":"miyahara"},"change_message_id":"4d123c4659f68c397ee1e63dcb587c49a91f0eb1","unresolved":false,"context_lines":[{"line_number":217,"context_line":"                prev_policy_idx, new_policy_idx \u003d \\"},{"line_number":218,"context_line":"                    self._extract_policy_indices(obj)"},{"line_number":219,"context_line":"                if prev_policy_idx !\u003d new_policy_idx:"},{"line_number":220,"context_line":"                    obj_next_timestamp \u003d slightly_later_timestamp(obj[\u0027hash\u0027])"},{"line_number":221,"context_line":"                    success \u003d self.transfer_object(acc, con, obj[\u0027name\u0027],"},{"line_number":222,"context_line":"                                                   prev_policy_idx,"},{"line_number":223,"context_line":"                                                   new_policy_idx,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5aef4532_f330a1ea","line":220,"range":{"start_line":220,"start_character":20,"end_line":220,"end_character":78},"updated":"2016-03-09 06:44:33.000000000","message":"This is the second increment.","commit_id":"68bdfdb60e7383a3d213fcd630b445c877bda561"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":214,"context_line":"                               \u0027X-Timestamp\u0027: resp.headers[\u0027x-timestamp\u0027],"},{"line_number":215,"context_line":"                               \u0027X-Container-Sysmeta-Prev-Storage-Policy-Index\u0027:"},{"line_number":216,"context_line":"                                resp.headers[\u0027x-container-sysmeta-prev-index\u0027]"},{"line_number":217,"context_line":"                               }"},{"line_number":218,"context_line":"            special_environ \u003d {\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027}"},{"line_number":219,"context_line":"            preq \u003d Request.blank(special_path, headers\u003dspecial_headers,"},{"line_number":220,"context_line":"                                 environ\u003dspecial_environ)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_f9738e13","line":217,"updated":"2015-08-05 07:56:58.000000000","message":"so this container never has any objects in it - it\u0027s just a scratch plce to keep some metadata about the work?","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":8859,"name":"Daisuke Morita","email":"morita.daisuke@ntti3.com","username":"morrisson"},"change_message_id":"7cf7b7eca212a2d9bc5eca9d129e9c0f30230e39","unresolved":false,"context_lines":[{"line_number":214,"context_line":"                               \u0027X-Timestamp\u0027: resp.headers[\u0027x-timestamp\u0027],"},{"line_number":215,"context_line":"                               \u0027X-Container-Sysmeta-Prev-Storage-Policy-Index\u0027:"},{"line_number":216,"context_line":"                                resp.headers[\u0027x-container-sysmeta-prev-index\u0027]"},{"line_number":217,"context_line":"                               }"},{"line_number":218,"context_line":"            special_environ \u003d {\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027}"},{"line_number":219,"context_line":"            preq \u003d Request.blank(special_path, headers\u003dspecial_headers,"},{"line_number":220,"context_line":"                                 environ\u003dspecial_environ)"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_6ed6e8f2","line":217,"in_reply_to":"1a4dcd0f_f9738e13","updated":"2015-08-10 21:27:40.000000000","message":"Exactly. Like reconciler.","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"828c067836fd13e4c0328fbcea4c246615d43671","unresolved":false,"context_lines":[{"line_number":224,"context_line":"            self.container_name \u003d container_path_hash"},{"line_number":225,"context_line":"            # TODO: How to recover to put a special container in the case of"},{"line_number":226,"context_line":"            # failure. Can other container related daemons re-put a special"},{"line_number":227,"context_line":"            # container"},{"line_number":228,"context_line":"            self.PUT(preq)"},{"line_number":229,"context_line":"            self.account_name \u003d p_acc_name"},{"line_number":230,"context_line":"            self.container_name \u003d p_cont_name"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4dcd0f_595ba287","line":227,"updated":"2015-08-05 07:56:58.000000000","message":"how indeed...","commit_id":"723063c4b48787cd3a4fac5ae001f00c9be2e473"}]}