)]}'
{"etc/container-server.conf-sample":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"91859f260595371541d0d56ec77c8a18712bfda6","unresolved":true,"context_lines":[{"line_number":492,"context_line":"# Minimum size of the final shard range. If this is greater than one then the"},{"line_number":493,"context_line":"# final shard range may be extended to more than rows_per_shard in order to"},{"line_number":494,"context_line":"# avoid a further shard range with less than minimum-shard-size rows."},{"line_number":495,"context_line":"# minimum_shard_size \u003d 1"},{"line_number":496,"context_line":"#"},{"line_number":497,"context_line":"# The container-sharder accepts the following configuration options as defined"},{"line_number":498,"context_line":"# in the container-replicator section:"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"9bf25c78_8c0160e6","line":495,"updated":"2021-06-17 00:37:01.000000000","message":"IDK that I like this default. It preserves existing behavior, but we know that existing behavior is not ideal.","commit_id":"ecb19c1f193734e85d3d63bd5231babe4aef5db4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dc54d1e8ae06113e8ac656762fedb94d825eec94","unresolved":true,"context_lines":[{"line_number":492,"context_line":"# Minimum size of the final shard range. If this is greater than one then the"},{"line_number":493,"context_line":"# final shard range may be extended to more than rows_per_shard in order to"},{"line_number":494,"context_line":"# avoid a further shard range with less than minimum-shard-size rows."},{"line_number":495,"context_line":"# minimum_shard_size \u003d 1"},{"line_number":496,"context_line":"#"},{"line_number":497,"context_line":"# The container-sharder accepts the following configuration options as defined"},{"line_number":498,"context_line":"# in the container-replicator section:"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"4ececd5b_1b81148a","line":495,"in_reply_to":"9bf25c78_8c0160e6","updated":"2021-06-24 13:41:48.000000000","message":"changing to rows_per_shard // 5 \u003d\u003d 100k","commit_id":"ecb19c1f193734e85d3d63bd5231babe4aef5db4"}],"swift/cli/manage_shard_ranges.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"556b95c0c01224dfbf4b078a41c351649daf2d35","unresolved":true,"context_lines":[{"line_number":913,"context_line":"        if args.conf_file:"},{"line_number":914,"context_line":"            # load values from conf file"},{"line_number":915,"context_line":"            confs.append(readconf(args.conf_file, \u0027container-sharder\u0027))"},{"line_number":916,"context_line":"        conf_args \u003d ContainerSharderConf(confs)"},{"line_number":917,"context_line":"    except (OSError, IOError) as exc:"},{"line_number":918,"context_line":"        print(\u0027Error opening config file %s: %s\u0027 % (args.conf_file, exc),"},{"line_number":919,"context_line":"              file\u003dsys.stderr)"}],"source_content_type":"text/x-python","patch_set":5,"id":"a0e11232_6ce4fcd1","line":916,"updated":"2021-06-24 14:11:17.000000000","message":"rows_per_shard from the command line may influence the default minimum-shard-size so we need to pass command line args to the ContainerSharderConf","commit_id":"5923417b1442fd2a0921f574a53c228acc586bc2"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3725a1cd77cbebaf35451091e7ad814c9e11e154","unresolved":true,"context_lines":[{"line_number":913,"context_line":"        if args.conf_file:"},{"line_number":914,"context_line":"            # load values from conf file"},{"line_number":915,"context_line":"            confs.append(readconf(args.conf_file, \u0027container-sharder\u0027))"},{"line_number":916,"context_line":"        conf_args \u003d ContainerSharderConf(confs)"},{"line_number":917,"context_line":"    except (OSError, IOError) as exc:"},{"line_number":918,"context_line":"        print(\u0027Error opening config file %s: %s\u0027 % (args.conf_file, exc),"},{"line_number":919,"context_line":"              file\u003dsys.stderr)"}],"source_content_type":"text/x-python","patch_set":5,"id":"fb249637_4bbc8f01","line":916,"in_reply_to":"a0e11232_6ce4fcd1","updated":"2021-06-25 06:41:01.000000000","message":"I\u0027m a bit confused by the config handling here. If it appears in the first config then it\u0027ll take presendence. From what it seems in the node from this patch.\n\nSo can\u0027t we just:\n\n  conf \u003d {}\n  if args.conf_file:\n    conf \u003d readconf(...)\n  conf.update(dict((k, v) for k, v in vars(args).items()\n              if v !\u003d USE_SHARDER_DEFAULT))\n  conf_args \u003d ContainerSharderConf(conf)","commit_id":"5923417b1442fd2a0921f574a53c228acc586bc2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f99d50e35f3685ff8fe7b018c6631baf15b4e6aa","unresolved":true,"context_lines":[{"line_number":913,"context_line":"        if args.conf_file:"},{"line_number":914,"context_line":"            # load values from conf file"},{"line_number":915,"context_line":"            confs.append(readconf(args.conf_file, \u0027container-sharder\u0027))"},{"line_number":916,"context_line":"        conf_args \u003d ContainerSharderConf(confs)"},{"line_number":917,"context_line":"    except (OSError, IOError) as exc:"},{"line_number":918,"context_line":"        print(\u0027Error opening config file %s: %s\u0027 % (args.conf_file, exc),"},{"line_number":919,"context_line":"              file\u003dsys.stderr)"}],"source_content_type":"text/x-python","patch_set":5,"id":"f72039cd_71563a3c","line":916,"in_reply_to":"fb249637_4bbc8f01","updated":"2021-06-28 09:03:53.000000000","message":"You\u0027re right, it can be simpler, I got a bit carried away with the generalised overly conf thing :( will fix","commit_id":"5923417b1442fd2a0921f574a53c228acc586bc2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a44edddc8ef8c90c9ca970bb8193b6ddc92489ec","unresolved":true,"context_lines":[{"line_number":910,"context_line":"    try:"},{"line_number":911,"context_line":"        conf \u003d {}"},{"line_number":912,"context_line":"        if args.conf_file:"},{"line_number":913,"context_line":"            conf \u003d readconf(args.conf_file, \u0027container-sharder\u0027)"},{"line_number":914,"context_line":"        conf.update(dict((k, v) for k, v in vars(args).items()"},{"line_number":915,"context_line":"                         if v !\u003d USE_SHARDER_DEFAULT))"},{"line_number":916,"context_line":"        conf_args \u003d ContainerSharderConf(conf)"}],"source_content_type":"text/x-python","patch_set":7,"id":"710a5080_04890b47","line":913,"updated":"2021-06-30 20:51:48.000000000","message":"This *can* pick up a minimum_shard_size from the [container-sharder] section... yeah?","commit_id":"89fdce188f5efd8c04ac855c1f2ac644f3ff6866"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6a4b4435f458864aec347b23574033a654093953","unresolved":true,"context_lines":[{"line_number":910,"context_line":"    try:"},{"line_number":911,"context_line":"        conf \u003d {}"},{"line_number":912,"context_line":"        if args.conf_file:"},{"line_number":913,"context_line":"            conf \u003d readconf(args.conf_file, \u0027container-sharder\u0027)"},{"line_number":914,"context_line":"        conf.update(dict((k, v) for k, v in vars(args).items()"},{"line_number":915,"context_line":"                         if v !\u003d USE_SHARDER_DEFAULT))"},{"line_number":916,"context_line":"        conf_args \u003d ContainerSharderConf(conf)"}],"source_content_type":"text/x-python","patch_set":7,"id":"efecd7a9_3fc359e0","line":913,"in_reply_to":"710a5080_04890b47","updated":"2021-07-01 17:06:13.000000000","message":"should do but I should check if there is test coverage for that now you have asked","commit_id":"89fdce188f5efd8c04ac855c1f2ac644f3ff6866"}],"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"91859f260595371541d0d56ec77c8a18712bfda6","unresolved":true,"context_lines":[{"line_number":643,"context_line":"        if not operands[0][1] \u003c operands[1][1]:"},{"line_number":644,"context_line":"            raise ValueError("},{"line_number":645,"context_line":"                \u0027%s (%d) must be less than %s (%d)\u0027"},{"line_number":646,"context_line":"                % (operands[0] + operands[1]))"},{"line_number":647,"context_line":""},{"line_number":648,"context_line":""},{"line_number":649,"context_line":"DEFAULT_SHARDER_CONF \u003d vars(ContainerSharderConf())"}],"source_content_type":"text/x-python","patch_set":2,"id":"afcd3976_9768023c","line":646,"updated":"2021-06-17 00:37:01.000000000","message":"This feels a bit awkward -- I\u0027m guessing it\u0027s been done to make it easy to add future validations? Still, something like\n\n ordered_args \u003d [\n     (\u0027minimum_shard_size\u0027, \u0027rows_per_shard\u0027),\n     (\u0027shrink_threshold\u0027, \u0027rows_per_shard\u0027),\n     (\u0027rows_per_shard\u0027, \u0027shard_container_threshold\u0027),\n     (\u0027shrink_threshold\u0027, \u0027expansion_limit\u0027),\n     (\u0027expansion_limit\u0027, \u0027shard_container_threshold\u0027),\n ]\n for smaller, larger in ordered_args:\n     if conf[smaller] \u003e\u003d conf[larger]:\n         raise ValueError(\u0027%s (%d) must be less than %s (%d)\u0027\n                          % (smaller, conf[smaller], larger, conf[larger]))\n\nseems more obvious to me. Dictionary lookups are cheap.","commit_id":"ecb19c1f193734e85d3d63bd5231babe4aef5db4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"dc54d1e8ae06113e8ac656762fedb94d825eec94","unresolved":true,"context_lines":[{"line_number":643,"context_line":"        if not operands[0][1] \u003c operands[1][1]:"},{"line_number":644,"context_line":"            raise ValueError("},{"line_number":645,"context_line":"                \u0027%s (%d) must be less than %s (%d)\u0027"},{"line_number":646,"context_line":"                % (operands[0] + operands[1]))"},{"line_number":647,"context_line":""},{"line_number":648,"context_line":""},{"line_number":649,"context_line":"DEFAULT_SHARDER_CONF \u003d vars(ContainerSharderConf())"}],"source_content_type":"text/x-python","patch_set":2,"id":"cc7ea3c8_55a26a41","line":646,"in_reply_to":"afcd3976_9768023c","updated":"2021-06-24 13:41:48.000000000","message":"I did have an eye on further validations. Will rework it.","commit_id":"ecb19c1f193734e85d3d63bd5231babe4aef5db4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"556b95c0c01224dfbf4b078a41c351649daf2d35","unresolved":true,"context_lines":[{"line_number":578,"context_line":"        if not confs:"},{"line_number":579,"context_line":"            confs \u003d [{}]"},{"line_number":580,"context_line":"        elif not isinstance(confs, (list, tuple)):"},{"line_number":581,"context_line":"            confs \u003d [confs]"},{"line_number":582,"context_line":""},{"line_number":583,"context_line":"        def get_val(key, validator, default):"},{"line_number":584,"context_line":"            \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"40c23a99_b767516e","line":581,"updated":"2021-06-24 14:11:17.000000000","message":"we need to support overriding confs so that s-m-s-r can pass in its CLI args with a rows_per_shard value that is then used to deduce default minimum-shard-size","commit_id":"5923417b1442fd2a0921f574a53c228acc586bc2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"556b95c0c01224dfbf4b078a41c351649daf2d35","unresolved":true,"context_lines":[{"line_number":641,"context_line":"            \u0027expansion_limit\u0027, int, self.expansion_limit)"},{"line_number":642,"context_line":"        self.rows_per_shard \u003d get_val("},{"line_number":643,"context_line":"            \u0027rows_per_shard\u0027, config_positive_int_value,"},{"line_number":644,"context_line":"            max(self.shard_container_threshold // 2, 1))"},{"line_number":645,"context_line":"        self.minimum_shard_size \u003d get_val("},{"line_number":646,"context_line":"            \u0027minimum_shard_size\u0027, config_positive_int_value,"},{"line_number":647,"context_line":"            max(self.rows_per_shard // 5, 1))"}],"source_content_type":"text/x-python","patch_set":5,"id":"0a46f0dc_7992b69c","line":644,"updated":"2021-06-24 14:11:17.000000000","message":"we need to allow rows_per_shard in the conf dict so that we can set a sensible default for minimum_shard_size","commit_id":"5923417b1442fd2a0921f574a53c228acc586bc2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"556b95c0c01224dfbf4b078a41c351649daf2d35","unresolved":true,"context_lines":[{"line_number":670,"context_line":"        logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":671,"context_line":"        ContainerReplicator.__init__(self, conf, logger\u003dlogger)"},{"line_number":672,"context_line":"        # rows_per_shard is not configurable for the daemon"},{"line_number":673,"context_line":"        conf.pop(\u0027rows_per_shard\u0027, None)"},{"line_number":674,"context_line":"        ContainerSharderConf.__init__(self, conf)"},{"line_number":675,"context_line":"        if conf.get(\u0027auto_create_account_prefix\u0027):"},{"line_number":676,"context_line":"            self.logger.warning(\u0027Option auto_create_account_prefix is \u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"b867f75a_7a9d95fe","line":673,"updated":"2021-06-24 14:11:17.000000000","message":"we still do not yet support rows_per_shard in config files","commit_id":"5923417b1442fd2a0921f574a53c228acc586bc2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2ca8a1c89acd2840c71122f9f1a438dcd7c082d9","unresolved":true,"context_lines":[{"line_number":628,"context_line":"        self.expansion_limit \u003d get_val("},{"line_number":629,"context_line":"            \u0027expansion_limit\u0027, int, self.expansion_limit)"},{"line_number":630,"context_line":"        self.rows_per_shard \u003d get_val("},{"line_number":631,"context_line":"            \u0027rows_per_shard\u0027, config_positive_int_value,"},{"line_number":632,"context_line":"            max(self.shard_container_threshold // 2, 1))"},{"line_number":633,"context_line":"        self.minimum_shard_size \u003d get_val("},{"line_number":634,"context_line":"            \u0027minimum_shard_size\u0027, config_positive_int_value,"}],"source_content_type":"text/x-python","patch_set":7,"id":"1888073d_88e11c0e","line":631,"updated":"2021-06-28 19:49:02.000000000","message":"Feels like a bit of a red flag that we\u0027re not updating the example config, but I suppose it\u0027s not actually exposed there.","commit_id":"89fdce188f5efd8c04ac855c1f2ac644f3ff6866"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2ca8a1c89acd2840c71122f9f1a438dcd7c082d9","unresolved":true,"context_lines":[{"line_number":658,"context_line":"        logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":659,"context_line":"        ContainerReplicator.__init__(self, conf, logger\u003dlogger)"},{"line_number":660,"context_line":"        # rows_per_shard is not configurable for the daemon"},{"line_number":661,"context_line":"        conf.pop(\u0027rows_per_shard\u0027, None)"},{"line_number":662,"context_line":"        ContainerSharderConf.__init__(self, conf)"},{"line_number":663,"context_line":"        if conf.get(\u0027auto_create_account_prefix\u0027):"},{"line_number":664,"context_line":"            self.logger.warning(\u0027Option auto_create_account_prefix is \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"a8a555b4_efbf24c7","line":661,"updated":"2021-06-28 19:49:02.000000000","message":"Why not?","commit_id":"89fdce188f5efd8c04ac855c1f2ac644f3ff6866"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"69516e53cbb66e17506a56b2d32b83cebe3eba78","unresolved":true,"context_lines":[{"line_number":658,"context_line":"        logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":659,"context_line":"        ContainerReplicator.__init__(self, conf, logger\u003dlogger)"},{"line_number":660,"context_line":"        # rows_per_shard is not configurable for the daemon"},{"line_number":661,"context_line":"        conf.pop(\u0027rows_per_shard\u0027, None)"},{"line_number":662,"context_line":"        ContainerSharderConf.__init__(self, conf)"},{"line_number":663,"context_line":"        if conf.get(\u0027auto_create_account_prefix\u0027):"},{"line_number":664,"context_line":"            self.logger.warning(\u0027Option auto_create_account_prefix is \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"67eb3a39_7125b07a","line":661,"in_reply_to":"a8a555b4_efbf24c7","updated":"2021-06-29 12:57:53.000000000","message":"This is merely preserving existing behaviour. I didn\u0027t think it appropriate to change that in this patch, although it is anomalous and unnecessary.","commit_id":"89fdce188f5efd8c04ac855c1f2ac644f3ff6866"}],"test/probe/test_sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"91859f260595371541d0d56ec77c8a18712bfda6","unresolved":true,"context_lines":[{"line_number":2808,"context_line":"            \u0027swift-manage-shard-ranges\u0027,"},{"line_number":2809,"context_line":"            self.get_db_file(self.brain.part, self.brain.nodes[0]),"},{"line_number":2810,"context_line":"            \u0027find_and_replace\u0027, \u00273\u0027, \u0027--enable\u0027, \u0027--minimum-shard-size\u0027, \u00272\u0027])"},{"line_number":2811,"context_line":"        self.assert_container_state(self.brain.nodes[0], \u0027unsharded\u0027, 2)"},{"line_number":2812,"context_line":""},{"line_number":2813,"context_line":"        # \"Run container-replicator to replicate them to other nodes.\""},{"line_number":2814,"context_line":"        self.replicators.once()"}],"source_content_type":"text/x-python","patch_set":2,"id":"d1bb182b_17af579b","line":2811,"updated":"2021-06-17 00:37:01.000000000","message":"👍","commit_id":"ecb19c1f193734e85d3d63bd5231babe4aef5db4"}]}