)]}'
{"swift/proxy/controllers/obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c0c816e0f52e21693c5c9d008ececca3846e911","unresolved":false,"context_lines":[{"line_number":2113,"context_line":"        if client_cl:"},{"line_number":2114,"context_line":"            policy_index \u003d int(headers.get(\u0027X-Backend-Storage-Policy-Index\u0027))"},{"line_number":2115,"context_line":"            policy \u003d POLICIES.get_by_index(policy_index)"},{"line_number":2116,"context_line":"            if 2147483648 \u003e int(client_cl) \u003e\u003d policy.ec_segment_size:"},{"line_number":2117,"context_line":"                # TODO - remove condition when get_segment_info handles"},{"line_number":2118,"context_line":"                # larger numbers for data_len.  Currently, data_len is int32"},{"line_number":2119,"context_line":"                # https://bitbucket.org/kmgreen2/pyeclib/pull-requests/34/"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_c71eb062","line":2116,"range":{"start_line":2116,"start_character":28,"end_line":2116,"end_character":68},"updated":"2016-02-20 22:31:27.000000000","message":"We can\u0027t get an expected-fragment-archive size when client_cl \u003c policy.ec_segment_size?","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"fabc645e73e41f9966c1058290f76667465e05f9","unresolved":false,"context_lines":[{"line_number":2113,"context_line":"        if client_cl:"},{"line_number":2114,"context_line":"            policy_index \u003d int(headers.get(\u0027X-Backend-Storage-Policy-Index\u0027))"},{"line_number":2115,"context_line":"            policy \u003d POLICIES.get_by_index(policy_index)"},{"line_number":2116,"context_line":"            if 2147483648 \u003e int(client_cl) \u003e\u003d policy.ec_segment_size:"},{"line_number":2117,"context_line":"                # TODO - remove condition when get_segment_info handles"},{"line_number":2118,"context_line":"                # larger numbers for data_len.  Currently, data_len is int32"},{"line_number":2119,"context_line":"                # https://bitbucket.org/kmgreen2/pyeclib/pull-requests/34/"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_e7a494ba","line":2116,"range":{"start_line":2116,"start_character":28,"end_line":2116,"end_character":68},"in_reply_to":"bae84128_c71eb062","updated":"2016-02-20 22:46:20.000000000","message":"oops, that shouldn\u0027t be there.  I will take that away and see if everything is still ok.","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c0c816e0f52e21693c5c9d008ececca3846e911","unresolved":false,"context_lines":[{"line_number":2115,"context_line":"            policy \u003d POLICIES.get_by_index(policy_index)"},{"line_number":2116,"context_line":"            if 2147483648 \u003e int(client_cl) \u003e\u003d policy.ec_segment_size:"},{"line_number":2117,"context_line":"                # TODO - remove condition when get_segment_info handles"},{"line_number":2118,"context_line":"                # larger numbers for data_len.  Currently, data_len is int32"},{"line_number":2119,"context_line":"                # https://bitbucket.org/kmgreen2/pyeclib/pull-requests/34/"},{"line_number":2120,"context_line":"                #  enable-get_segment_info-to-handle-larger/diff"},{"line_number":2121,"context_line":"                #"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_2c45c316","line":2118,"range":{"start_line":2118,"start_character":59,"end_line":2118,"end_character":76},"updated":"2016-02-20 22:31:27.000000000","message":"Well that\u0027s fun. Thanks for noticing that early!","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"fabc645e73e41f9966c1058290f76667465e05f9","unresolved":false,"context_lines":[{"line_number":2115,"context_line":"            policy \u003d POLICIES.get_by_index(policy_index)"},{"line_number":2116,"context_line":"            if 2147483648 \u003e int(client_cl) \u003e\u003d policy.ec_segment_size:"},{"line_number":2117,"context_line":"                # TODO - remove condition when get_segment_info handles"},{"line_number":2118,"context_line":"                # larger numbers for data_len.  Currently, data_len is int32"},{"line_number":2119,"context_line":"                # https://bitbucket.org/kmgreen2/pyeclib/pull-requests/34/"},{"line_number":2120,"context_line":"                #  enable-get_segment_info-to-handle-larger/diff"},{"line_number":2121,"context_line":"                #"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_8796684a","line":2118,"range":{"start_line":2118,"start_character":59,"end_line":2118,"end_character":76},"in_reply_to":"bae84128_2c45c316","updated":"2016-02-20 22:46:20.000000000","message":"well, it was easy to notice that one of the functests with a max size obj fails this call :-)\n\nKota helped to look into what was happening exactly with that large size.","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c0c816e0f52e21693c5c9d008ececca3846e911","unresolved":false,"context_lines":[{"line_number":2126,"context_line":"                    (info.get(\u0027num_segments\u0027) - 1) + \\"},{"line_number":2127,"context_line":"                    info.get(\u0027last_fragment_size\u0027)"},{"line_number":2128,"context_line":"                if info.get(\u0027last_segment_size\u0027) \u003e info.get(\u0027segment_size\u0027):"},{"line_number":2129,"context_line":"                    # TODO - find out why this happens"},{"line_number":2130,"context_line":"                    # This happens in test_slo_copy and test_slo_copy_account."},{"line_number":2131,"context_line":"                    # Using these calcuations will result in a 499 on PUT"},{"line_number":2132,"context_line":"                    # because the fsize !\u003d upload_size. actual is +80"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_ecf11b6c","line":2129,"range":{"start_line":2129,"start_character":29,"end_line":2129,"end_character":54},"updated":"2016-02-20 22:31:27.000000000","message":"Looks like in addition to the configured segment_size, there\u0027s also a min_segment_size from the EC library [1]. If the last segment would be below that, it will instead be combined with the previous one [2][3].\n\nBut it seems like that should all be accounted-for in last_fragment_size... hmm... Maybe we can add some logging to see what we got in info, and what the actual fragment lengths are as we send them down to the object server?\n\n[1] https://github.com/tsg-/pyeclib/blob/e4c48ee/src/c/pyeclib_c/pyeclib_c.c#L373-L374\n\n[2] https://github.com/tsg-/pyeclib/blob/e4c48ee/src/c/pyeclib_c/pyeclib_c.c#L379-L385\n\n[3] https://github.com/tsg-/pyeclib/blob/e4c48ee/src/c/pyeclib_c/pyeclib_c.c#L414-L424","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"fabc645e73e41f9966c1058290f76667465e05f9","unresolved":false,"context_lines":[{"line_number":2126,"context_line":"                    (info.get(\u0027num_segments\u0027) - 1) + \\"},{"line_number":2127,"context_line":"                    info.get(\u0027last_fragment_size\u0027)"},{"line_number":2128,"context_line":"                if info.get(\u0027last_segment_size\u0027) \u003e info.get(\u0027segment_size\u0027):"},{"line_number":2129,"context_line":"                    # TODO - find out why this happens"},{"line_number":2130,"context_line":"                    # This happens in test_slo_copy and test_slo_copy_account."},{"line_number":2131,"context_line":"                    # Using these calcuations will result in a 499 on PUT"},{"line_number":2132,"context_line":"                    # because the fsize !\u003d upload_size. actual is +80"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_c777d038","line":2129,"range":{"start_line":2129,"start_character":29,"end_line":2129,"end_character":54},"in_reply_to":"bae84128_ecf11b6c","updated":"2016-02-20 22:46:20.000000000","message":"ok, I will look more into how this size deviation really happens, and prob get back to you with questions.","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"55c53dccd1f3bf282bdfe525557c5b15b17e7b29","unresolved":false,"context_lines":[{"line_number":2129,"context_line":"                    # TODO - find out why some tests in this case fail"},{"line_number":2130,"context_line":"                    # This happens in test_slo_copy and test_slo_copy_account."},{"line_number":2131,"context_line":"                    # Using these calcuations will result in a 499 on PUT"},{"line_number":2132,"context_line":"                    # because the fsize !\u003d upload_size. actual is +80"},{"line_number":2133,"context_line":"                    expected_frag_size \u003d None"},{"line_number":2134,"context_line":""},{"line_number":2135,"context_line":"        self.app.logger.thread_locals \u003d logger_thread_locals"}],"source_content_type":"text/x-python","patch_set":2,"id":"bae84128_4069d124","line":2132,"updated":"2016-02-22 08:03:39.000000000","message":"I don\u0027t know why the PyECLib rounds smaller segment into previous one but it comes from performance efficiency. Perhaps, the reason PyECLib calculates min_segment_size is because liberasurecode will pad data to make good memory alignment if the segment is small enough. And if the segment smaller than min segment, the padding will be bigger than actual payload so it will be wasted and PyECLib *might* expect an user handles to prevent to put such a small segment but swift never does.\n\nI wrote up an unit test[1] PyECLib lies the segment info different from input one.\n\n1: https://bitbucket.org/bloodeagle40234/pyeclib/commits/0a082b03b5b9c11552275b390d6a33a016c9c0ff","commit_id":"70a4106bd3ccc436de460db08ee935dffc3cf203"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"55ffdffe6edf14ec9999c9278ba2c318401c4f77","unresolved":false,"context_lines":[{"line_number":2129,"context_line":"                    # TODO - find out why some tests in this case fail"},{"line_number":2130,"context_line":"                    # This happens in test_slo_copy and test_slo_copy_account."},{"line_number":2131,"context_line":"                    # Using these calcuations will result in a 499 on PUT"},{"line_number":2132,"context_line":"                    # because the fsize !\u003d upload_size. actual is +80"},{"line_number":2133,"context_line":"                    expected_frag_size \u003d None"},{"line_number":2134,"context_line":""},{"line_number":2135,"context_line":"        self.app.logger.thread_locals \u003d logger_thread_locals"}],"source_content_type":"text/x-python","patch_set":2,"id":"bae84128_83f2bc6d","line":2132,"in_reply_to":"bae84128_4069d124","updated":"2016-02-22 10:26:07.000000000","message":"Oh, sorry. I added wrong explanation for this. the padding is not for performance but for data slicing. For example, if we set ec_data \u003d 4 but the segments is just 1 byte, pyeclib has to pad 3 bytes into the segment in order to slice the segment into 4 1-byte data fragments. (Strictly the padding is calculated from both the ec algorithm and ec_data so above is just an example) I hope it makes you sense.","commit_id":"70a4106bd3ccc436de460db08ee935dffc3cf203"},{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"91be026e038d1e9798bc67433df8e61371d130c6","unresolved":false,"context_lines":[{"line_number":2113,"context_line":"        if client_cl:"},{"line_number":2114,"context_line":"            policy_index \u003d int(headers.get(\u0027X-Backend-Storage-Policy-Index\u0027))"},{"line_number":2115,"context_line":"            policy \u003d POLICIES.get_by_index(policy_index)"},{"line_number":2116,"context_line":"            # TODO: PyECLib \u003c\u003d 1.2.0 looks to return the segment info"},{"line_number":2117,"context_line":"            # different from the input for aligned data efficiency but"},{"line_number":2118,"context_line":"            # Swift never does. So calculate the fragment length Swift"},{"line_number":2119,"context_line":"            # will actually send to object sever by making two different"}],"source_content_type":"text/x-python","patch_set":3,"id":"3afc51ec_56169b55","line":2116,"updated":"2016-03-15 04:16:45.000000000","message":"So the desired behavior here is for get_segment_info() to not assume the case where a tiny final segment is merged into the previous one?\n\nIn some cases, we may end up with a 1-byte last fragment, but we don\u0027t have a choice given the current Swift data path design - we\u0027ll make the change to PyECLib.\n\nget_segment_info() is in use by no PyECLib users other than Swift at this point, so we should be able to get this in quickly.","commit_id":"b5a243e75a8033988063d2c1c90ac373bc0050d2"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"5e67ef05d9fa8fec3947e2ccbfaa4589ec6a7b7c","unresolved":false,"context_lines":[{"line_number":2116,"context_line":"            # TODO: PyECLib \u003c\u003d 1.2.0 looks to return the segment info"},{"line_number":2117,"context_line":"            # different from the input for aligned data efficiency but"},{"line_number":2118,"context_line":"            # Swift never does. So calculate the fragment length Swift"},{"line_number":2119,"context_line":"            # will actually send to object sever by making two different"},{"line_number":2120,"context_line":"            # get_segment_info calls (until PyECLib fixed)."},{"line_number":2121,"context_line":"            # policy.fragment_size makes the call using segment size,"},{"line_number":2122,"context_line":"            # and the next call is to get info for the last segment"}],"source_content_type":"text/x-python","patch_set":3,"id":"bae84128_b956ba27","line":2119,"range":{"start_line":2119,"start_character":43,"end_line":2119,"end_character":48},"updated":"2016-02-23 22:36:40.000000000","message":"nit: \"server\"","commit_id":"b5a243e75a8033988063d2c1c90ac373bc0050d2"},{"author":{"_account_id":7485,"name":"Tushar Gohad","email":"tushar.gohad@intel.com","username":"tsg"},"change_message_id":"5292713e168bd291b6ed8f19207962ab7c687475","unresolved":false,"context_lines":[{"line_number":2117,"context_line":"            # different from the input for aligned data efficiency but"},{"line_number":2118,"context_line":"            # Swift never does. So calculate the fragment length Swift"},{"line_number":2119,"context_line":"            # will actually send to object sever by making two different"},{"line_number":2120,"context_line":"            # get_segment_info calls (until PyECLib fixed)."},{"line_number":2121,"context_line":"            # policy.fragment_size makes the call using segment size,"},{"line_number":2122,"context_line":"            # and the next call is to get info for the last segment"},{"line_number":2123,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"1af94dfe_122c2caa","line":2120,"updated":"2016-03-17 06:20:45.000000000","message":"Remove \"until PyECLib fixed\" part per Sam\u0027s latest comment?  :-)","commit_id":"b5a243e75a8033988063d2c1c90ac373bc0050d2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ddfdb9f98e1dbf35bbce2d3010c16d8be249ddd4","unresolved":false,"context_lines":[{"line_number":2127,"context_line":""},{"line_number":2128,"context_line":"            # calculate the tail fragment_size by hand and add it to"},{"line_number":2129,"context_line":"            # expected_frag_size"},{"line_number":2130,"context_line":"            last_segment_size \u003d int(client_cl) % policy.ec_segment_size"},{"line_number":2131,"context_line":"            if last_segment_size:"},{"line_number":2132,"context_line":"                last_info \u003d policy.pyeclib_driver.get_segment_info("},{"line_number":2133,"context_line":"                    last_segment_size, policy.ec_segment_size)"}],"source_content_type":"text/x-python","patch_set":3,"id":"bae84128_827c16ed","line":2130,"updated":"2016-02-28 21:16:18.000000000","message":"Maybe more obviously correct?\n\n  num_fragments, last_segment_size \u003d divmod(\n      int(client_cl), policy.ec_segment_size)\n  expected_frag_size \u003d policy.fragment_size * num_fragments","commit_id":"b5a243e75a8033988063d2c1c90ac373bc0050d2"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"89a3ef92c179d50e1c13da26e17413ced3932866","unresolved":false,"context_lines":[{"line_number":2127,"context_line":""},{"line_number":2128,"context_line":"            # calculate the tail fragment_size by hand and add it to"},{"line_number":2129,"context_line":"            # expected_frag_size"},{"line_number":2130,"context_line":"            last_segment_size \u003d int(client_cl) % policy.ec_segment_size"},{"line_number":2131,"context_line":"            if last_segment_size:"},{"line_number":2132,"context_line":"                last_info \u003d policy.pyeclib_driver.get_segment_info("},{"line_number":2133,"context_line":"                    last_segment_size, policy.ec_segment_size)"}],"source_content_type":"text/x-python","patch_set":3,"id":"fa0719c6_e9e61eb3","line":2130,"in_reply_to":"bae84128_827c16ed","updated":"2016-03-22 08:25:35.000000000","message":"Yeah, that is nice!","commit_id":"b5a243e75a8033988063d2c1c90ac373bc0050d2"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"7c0c816e0f52e21693c5c9d008ececca3846e911","unresolved":false,"context_lines":[{"line_number":1500,"context_line":"                                connection_id,))"},{"line_number":1501,"context_line":"            self.assertTrue(size \u003e int(info[\u0027backend-content-length\u0027]) \u003e 0,"},{"line_number":1502,"context_line":"                            \"invalid backend-content-length for conn %r\" % ("},{"line_number":1503,"context_line":"                                connection_id,))"},{"line_number":1504,"context_line":""},{"line_number":1505,"context_line":"            # email.parser.FeedParser doesn\u0027t know how to take a multipart"},{"line_number":1506,"context_line":"            # message and boundary together and parse it; it only knows how"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_2745dc16","line":1503,"updated":"2016-02-20 22:31:27.000000000","message":"Surely somewhere we can make an exact assertion involving int(info[\u0027backend-content-length\u0027]) and len(something)... I still haven\u0027t figured out where, yet, though...","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"fabc645e73e41f9966c1058290f76667465e05f9","unresolved":false,"context_lines":[{"line_number":1500,"context_line":"                                connection_id,))"},{"line_number":1501,"context_line":"            self.assertTrue(size \u003e int(info[\u0027backend-content-length\u0027]) \u003e 0,"},{"line_number":1502,"context_line":"                            \"invalid backend-content-length for conn %r\" % ("},{"line_number":1503,"context_line":"                                connection_id,))"},{"line_number":1504,"context_line":""},{"line_number":1505,"context_line":"            # email.parser.FeedParser doesn\u0027t know how to take a multipart"},{"line_number":1506,"context_line":"            # message and boundary together and parse it; it only knows how"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_07909842","line":1503,"in_reply_to":"bae84128_2745dc16","updated":"2016-02-20 22:46:20.000000000","message":"Yes, that would be good.  I might need some help figuring out how to calculate the expected in the test :-)","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2a8d332e3aa6d139bed50f05712247e273cbec89","unresolved":false,"context_lines":[{"line_number":1521,"context_line":"            # attach the body to frag_archives list"},{"line_number":1522,"context_line":"            self.assertEqual(obj_part[\u0027X-Document\u0027], \u0027object body\u0027)"},{"line_number":1523,"context_line":"            frag_archives.append(obj_part.get_payload())"},{"line_number":1524,"context_line":""},{"line_number":1525,"context_line":"            # validate some footer metadata"},{"line_number":1526,"context_line":"            self.assertEqual(footer_part[\u0027X-Document\u0027], \u0027object metadata\u0027)"},{"line_number":1527,"context_line":"            footer_metadata \u003d json.loads(footer_part.get_payload())"}],"source_content_type":"text/x-python","patch_set":1,"id":"bae84128_02d7e323","line":1524,"updated":"2016-02-21 04:50:44.000000000","message":"So I think this is where we want something like\n\n self.assertEqual(int(info[\u0027backend-content-length\u0027]),\n                  len(frag_archives[-1]))\n\nWe may also want to check that all of the connections got the same backend-content-length.","commit_id":"32d47d390fcb3f99d81726aea7d1cc24d7e247bc"}]}