)]}'
{"swift/container/backend.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"baa2520b3a00e9707dc655290c6149d07ccc33a0","unresolved":true,"context_lines":[{"line_number":2232,"context_line":"                sql +\u003d \"AND name \u003e ? \""},{"line_number":2233,"context_line":"                args.append(str(last_upper))"},{"line_number":2234,"context_line":"            obj_count \u003d connection.execute(sql, args).fetchone()"},{"line_number":2235,"context_line":"            return obj_count"},{"line_number":2236,"context_line":""},{"line_number":2237,"context_line":"    def _get_next_shard_range_upper(self, shard_size, last_upper\u003dNone):"},{"line_number":2238,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":4,"id":"e837caf9_d8796264","line":2235,"updated":"2021-05-17 22:43:20.000000000","message":"There is no great way to do this.. but at least it hits the name, delete index:\n\n QUERY PLAN\n`--SCAN TABLE object USING COVERING INDEX ix_object_deleted_name\n\nStill need to write tests for this function.","commit_id":"4075ecc13d76e3cc6a34c8c4f608d1f1c8a07ef9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"182b5209b6a5012a55a2cd58562761af16eb3201","unresolved":true,"context_lines":[{"line_number":2232,"context_line":"                sql +\u003d \"AND name \u003e ? \""},{"line_number":2233,"context_line":"                args.append(str(last_upper))"},{"line_number":2234,"context_line":"            obj_count \u003d connection.execute(sql, args).fetchone()"},{"line_number":2235,"context_line":"            return obj_count"},{"line_number":2236,"context_line":""},{"line_number":2237,"context_line":"    def _get_next_shard_range_upper(self, shard_size, last_upper\u003dNone):"},{"line_number":2238,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":4,"id":"4c3f806f_1f2831f4","line":2235,"in_reply_to":"e837caf9_d8796264","updated":"2021-05-18 14:08:25.000000000","message":"When handling the final shard (next upper is None) we could just repeat _get_next_shard_upper with some smaller threshold shard_size and if we get an upper then we know the final shard is too small, so throw it away and extend the penultimate shard up to MAX, or if we still get None then we know the final shard has enough objects to be viable.\n\nOr if we want to be a little more \u0027accurate\u0027, use two thresholds:\n\n  upper \u003d _get_next_shard_range_upper(small_enough_to_ignore, last_upper)\n  if upper is None:\n      # extend penultimate shard to MAX and we\u0027re done\n  else:\n      upper \u003d _get_next_shard_range_upper(big_enough_to_keep, last_upper)\n      if upper is None:\n          # keep the final shard - it\u0027s OK - we\u0027re done\n      else:\n          # rollback penultimate shard and get final 2 shards again with \n          #   shard_size \u003d (shard_size + big_enough_to_keep)/2","commit_id":"4075ecc13d76e3cc6a34c8c4f608d1f1c8a07ef9"}],"swift/container/sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"871f5d9f207dff1997c7693d7f9a3803eec594d9","unresolved":true,"context_lines":[{"line_number":442,"context_line":"    paths.sort(key\u003dsort_key, reverse\u003dTrue)"},{"line_number":443,"context_line":"    return paths"},{"line_number":444,"context_line":""},{"line_number":445,"context_line":"def get_rows_per_shard(rows_per_shard, shard_container_threshold,"},{"line_number":446,"context_line":"                       expansion_limit, obj_count):"},{"line_number":447,"context_line":"    if rows_per_shard !\u003d \u0027auto\u0027:"},{"line_number":448,"context_line":"        return rows_per_shard"},{"line_number":449,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"9c77f00a_1fbc9cf2","line":446,"range":{"start_line":445,"start_character":23,"end_line":446,"end_character":38},"updated":"2021-05-14 10:02:06.000000000","message":"can just pass in a namespace, either args from s-m-s-r or the ContainerSharder instance, both of which have these as attributes","commit_id":"e2705462ec5e6dbee8a18ff65c579d016d7b3d9b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"99da3837ed1a4f986b7ededcc018397a20dbca7e","unresolved":true,"context_lines":[{"line_number":443,"context_line":"    return paths"},{"line_number":444,"context_line":""},{"line_number":445,"context_line":"def get_rows_per_shard(rows_per_shard, shard_container_threshold,"},{"line_number":446,"context_line":"                       expansion_limit, obj_count):"},{"line_number":447,"context_line":"    if rows_per_shard !\u003d \u0027auto\u0027:"},{"line_number":448,"context_line":"        return rows_per_shard"},{"line_number":449,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"8570525c_04bdbe83","line":446,"in_reply_to":"","updated":"2021-05-14 12:01:24.000000000","message":"yeah good call, and it should probably be the obj_count left to shard maybe.. seeing as the point is to calculate the correct rows_per_shard that will best fill the rest of the objs leaving no little ones.","commit_id":"e2705462ec5e6dbee8a18ff65c579d016d7b3d9b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"871f5d9f207dff1997c7693d7f9a3803eec594d9","unresolved":true,"context_lines":[{"line_number":455,"context_line":""},{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    left_over \u003d obj_count % rows_per_shard"},{"line_number":458,"context_line":"    number_of_shards \u003d obj_count // rows_per_shard"},{"line_number":459,"context_line":"    max_extra_per_shard \u003d (expansion_limit - rows_per_shard) * number_of_shards"},{"line_number":460,"context_line":""},{"line_number":461,"context_line":"    if left_over \u003c max_extra_per_shard:"}],"source_content_type":"text/x-python","patch_set":2,"id":"fb366e1c_c5d32d82","line":458,"range":{"start_line":458,"start_character":4,"end_line":458,"end_character":20},"updated":"2021-05-14 10:02:06.000000000","message":"which number of shards is this? the desired or the current?","commit_id":"e2705462ec5e6dbee8a18ff65c579d016d7b3d9b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"99da3837ed1a4f986b7ededcc018397a20dbca7e","unresolved":false,"context_lines":[{"line_number":455,"context_line":""},{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    left_over \u003d obj_count % rows_per_shard"},{"line_number":458,"context_line":"    number_of_shards \u003d obj_count // rows_per_shard"},{"line_number":459,"context_line":"    max_extra_per_shard \u003d (expansion_limit - rows_per_shard) * number_of_shards"},{"line_number":460,"context_line":""},{"line_number":461,"context_line":"    if left_over \u003c max_extra_per_shard:"}],"source_content_type":"text/x-python","patch_set":2,"id":"5bb98cd9_45773af4","line":458,"in_reply_to":"fb366e1c_c5d32d82","updated":"2021-05-14 12:01:24.000000000","message":"the number of shards we need for the rest we need based on obj_count. if we end up bathing this size will be recalculated for what\u0027s left next time round given a new obj_count.","commit_id":"e2705462ec5e6dbee8a18ff65c579d016d7b3d9b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"871f5d9f207dff1997c7693d7f9a3803eec594d9","unresolved":true,"context_lines":[{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    left_over \u003d obj_count % rows_per_shard"},{"line_number":458,"context_line":"    number_of_shards \u003d obj_count // rows_per_shard"},{"line_number":459,"context_line":"    max_extra_per_shard \u003d (expansion_limit - rows_per_shard) * number_of_shards"},{"line_number":460,"context_line":""},{"line_number":461,"context_line":"    if left_over \u003c max_extra_per_shard:"},{"line_number":462,"context_line":"        rows_per_shard +\u003d math.ceil(left_over / number_of_shards)"}],"source_content_type":"text/x-python","patch_set":2,"id":"d87ea275_bedecff0","line":459,"range":{"start_line":459,"start_character":4,"end_line":459,"end_character":79},"updated":"2021-05-14 10:02:06.000000000","message":"this doesn\u0027t feel right...\n\n1. the var is perhaps mis-named, this is just max_extra, right?\n\n2. if obj_count is 1490K, rows_per_shard is 500k and expansion_limit is 750k then max_extra is 500k, left_over is 490k, so rows_per_shard becomes 745k, whereas 3 shards with 500k, 500k, 490k would have been fine.\n\nso I think max_extra is too large??\n\nI\u0027m wondering about a different approach: rather than trying to make other shards as big as possible, just make sure the final shard isn\u0027t too small?\nhttps://review.opendev.org/c/openstack/swift/+/791440","commit_id":"e2705462ec5e6dbee8a18ff65c579d016d7b3d9b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"99da3837ed1a4f986b7ededcc018397a20dbca7e","unresolved":false,"context_lines":[{"line_number":456,"context_line":""},{"line_number":457,"context_line":"    left_over \u003d obj_count % rows_per_shard"},{"line_number":458,"context_line":"    number_of_shards \u003d obj_count // rows_per_shard"},{"line_number":459,"context_line":"    max_extra_per_shard \u003d (expansion_limit - rows_per_shard) * number_of_shards"},{"line_number":460,"context_line":""},{"line_number":461,"context_line":"    if left_over \u003c max_extra_per_shard:"},{"line_number":462,"context_line":"        rows_per_shard +\u003d math.ceil(left_over / number_of_shards)"}],"source_content_type":"text/x-python","patch_set":2,"id":"644bc885_27dc2d1d","line":459,"in_reply_to":"","updated":"2021-05-14 12:01:24.000000000","message":"yeah, it might be miss named. and yeah in your example case it\u0027ll get it wrong. well not wrong just not really optimised :)\n\nI guess if left_over gets large enough, say above shrink_size we can just return the current derived rows_per_shard as we haven\u0027t spread it out yet and call it a day.\n\nI\u0027m in my phone so I look forward to looking at your approach on my Monday!","commit_id":"e2705462ec5e6dbee8a18ff65c579d016d7b3d9b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"365eca52576f95c75156d455efd4dd000d031bfe","unresolved":true,"context_lines":[{"line_number":465,"context_line":""},{"line_number":466,"context_line":"    # each shard will grow by at most min_left_over/num_whole_shards"},{"line_number":467,"context_line":"    if left_over \u003c min_left_over:"},{"line_number":468,"context_line":"        rows_per_shard +\u003d math.ceil(float(left_over) / num_whole_shards)"},{"line_number":469,"context_line":""},{"line_number":470,"context_line":"    return int(rows_per_shard)"},{"line_number":471,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"a47c8aae_ab442cd4","line":468,"updated":"2021-05-23 23:04:24.000000000","message":"sigh, this float typecast is so py2 behaves properly.","commit_id":"43e6eee246736a6d9d2af6a3761ca3377b65d35c"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"eeb7d12ae81ce4f79ceea0ff76073e094b63c8b7","unresolved":true,"context_lines":[{"line_number":7270,"context_line":"                (\u0027auto\u0027, 200, 75, 1005, 100),"},{"line_number":7271,"context_line":"                # rows_per_shard \u003e obj_count return rows_per_shard"},{"line_number":7272,"context_line":"                (\u0027auto\u0027, 200, 75, 199, 100)):"},{"line_number":7273,"context_line":"            do_test(r_p_s, cont_threshold, exp_limit, obj_count, expected)"}],"source_content_type":"text/x-python","patch_set":1,"id":"2448d197_2d6c207f","line":7273,"updated":"2021-05-14 01:26:29.000000000","message":"Probably need a bunch more cases here, but wanted to add something initially.","commit_id":"12a29196179a4c43063bf163e20ac068844dc8d2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"130a6d2f185ed19310d49df22772f5d3f8610f2f","unresolved":true,"context_lines":[{"line_number":7267,"context_line":"                # let\u0027s add a bit extra"},{"line_number":7268,"context_line":"                (\u0027auto\u0027, 100, 75, 1005, 51),"},{"line_number":7269,"context_line":"                # rows_per_shard \u003e exp_limit return rows_per_shard"},{"line_number":7270,"context_line":"                (\u0027auto\u0027, 200, 75, 1005, 100),"},{"line_number":7271,"context_line":"                # rows_per_shard \u003e obj_count return rows_per_shard"},{"line_number":7272,"context_line":"                (\u0027auto\u0027, 200, 75, 199, 100)):"},{"line_number":7273,"context_line":"            do_test(r_p_s, cont_threshold, exp_limit, obj_count, expected)"}],"source_content_type":"text/x-python","patch_set":2,"id":"ed2b3d7c_f995b534","line":7270,"updated":"2021-05-14 04:36:32.000000000","message":"It seems like 101 would still be preferable here :-/ Why have an extra little shard with just 5 objects in it?\n\nI\u0027m *really* tempted to say that we ought to target\n\n rows_per_shard \u003d shard_container_threshold * 3 // 5\n\nor\n\n rows_per_shard \u003d shard_container_threshold * 11 // 20","commit_id":"e2705462ec5e6dbee8a18ff65c579d016d7b3d9b"}]}