)]}'
{"bin/swift-container-sharder":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":17,"context_line":"from swift.container.sharder import ContainerSharder"},{"line_number":18,"context_line":"from swift.common.utils import parse_options"},{"line_number":19,"context_line":"from swift.common.daemon import run_daemon"},{"line_number":20,"context_line":"from optparse import OptionParser"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"},{"line_number":23,"context_line":"    parser \u003d OptionParser(\"%prog CONFIG [options]\")"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"5f7c97a3_39e54b39","line":20,"updated":"2018-05-17 23:43:44.000000000","message":"Off-topic: ugh, that\u0027s right -- at some point we still need to move everything over to argparse...","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"}],"doc/saio/swift/internal-client.conf":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"4288132a59208eb728c6f276fa6cca381501a5c0","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"5f7c97a3_61bb2172","updated":"2018-05-17 10:19:50.000000000","message":"Hmm.. ok. this is just copied from etc/internal-client.conf","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"}],"etc/container-server.conf-sample":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c6d454d754f283340c7d8fd8d18db2c52b0d15b4","unresolved":false,"context_lines":[{"line_number":390,"context_line":"# cleaving a shard ranges before the sharder will progress to the next shard"},{"line_number":391,"context_line":"# range. The default of \u0027auto\u0027 causes the container ring quorum value to be"},{"line_number":392,"context_line":"# used."},{"line_number":393,"context_line":"# shard_replication_quorum \u003d auto"},{"line_number":394,"context_line":"#"},{"line_number":395,"context_line":"# The sharder uses an internal client to create and make requests to"},{"line_number":396,"context_line":"# containers. The absolute path to the client config file can be configured."}],"source_content_type":"application/octet-stream","patch_set":3,"id":"5f7c97a3_08dd5c6f","line":393,"updated":"2018-05-08 20:02:26.000000000","message":"YAS!\n\nIt\u0027s like handoffs delete hardcore mode for sharding!!!","commit_id":"1ea5f815dfc62e4c33df684743c5a0e4813bb5be"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c6d454d754f283340c7d8fd8d18db2c52b0d15b4","unresolved":false,"context_lines":[{"line_number":418,"context_line":"# in the container-replicator section:"},{"line_number":419,"context_line":"#"},{"line_number":420,"context_line":"# per_diff \u003d 1000"},{"line_number":421,"context_line":"# max_diffs \u003d 100"},{"line_number":422,"context_line":"# concurrency \u003d 8"},{"line_number":423,"context_line":"# node_timeout \u003d 10"},{"line_number":424,"context_line":"# conn_timeout \u003d 0.5"}],"source_content_type":"application/octet-stream","patch_set":3,"id":"5f7c97a3_e88ca080","line":421,"updated":"2018-05-08 20:02:26.000000000","message":"You\u0027d think 100K is alot, but if you have to usync a 500K db - meh...","commit_id":"1ea5f815dfc62e4c33df684743c5a0e4813bb5be"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"4288132a59208eb728c6f276fa6cca381501a5c0","unresolved":false,"context_lines":[{"line_number":355,"context_line":"# below which a \u0027donor\u0027 shard container will be considered for shrinking into"},{"line_number":356,"context_line":"# another \u0027acceptor\u0027 shard container. shard_shrink_point is a percentage of"},{"line_number":357,"context_line":"# shard_container_size e.g. the default value of 5 means 5% of the"},{"line_number":358,"context_line":"# shard_container_size. shard_shrink_point \u003d 5"},{"line_number":359,"context_line":"#"},{"line_number":360,"context_line":"# When auto-sharding is enabled shard_shrink_merge_point defines the maximum"},{"line_number":361,"context_line":"# allowed size of an acceptor shard container after having a donor merged into"}],"source_content_type":"application/octet-stream","patch_set":9,"id":"5f7c97a3_015eed44","line":358,"range":{"start_line":358,"start_character":24,"end_line":358,"end_character":46},"updated":"2018-05-17 10:19:50.000000000","message":"nits) I think this is the option key-value so we should make it as a new line.","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":355,"context_line":"# below which a \u0027donor\u0027 shard container will be considered for shrinking into"},{"line_number":356,"context_line":"# another \u0027acceptor\u0027 shard container. shard_shrink_point is a percentage of"},{"line_number":357,"context_line":"# shard_container_size e.g. the default value of 5 means 5% of the"},{"line_number":358,"context_line":"# shard_container_size. shard_shrink_point \u003d 5"},{"line_number":359,"context_line":"#"},{"line_number":360,"context_line":"# When auto-sharding is enabled shard_shrink_merge_point defines the maximum"},{"line_number":361,"context_line":"# allowed size of an acceptor shard container after having a donor merged into"}],"source_content_type":"application/octet-stream","patch_set":9,"id":"5f7c97a3_91378be4","line":358,"range":{"start_line":358,"start_character":24,"end_line":358,"end_character":46},"in_reply_to":"5f7c97a3_015eed44","updated":"2018-05-18 17:55:43.000000000","message":"Done","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":355,"context_line":"# below which a \u0027donor\u0027 shard container will be considered for shrinking into"},{"line_number":356,"context_line":"# another \u0027acceptor\u0027 shard container. shard_shrink_point is a percentage of"},{"line_number":357,"context_line":"# shard_container_size e.g. the default value of 5 means 5% of the"},{"line_number":358,"context_line":"# shard_container_size. shard_shrink_point \u003d 5"},{"line_number":359,"context_line":"#"},{"line_number":360,"context_line":"# When auto-sharding is enabled shard_shrink_merge_point defines the maximum"},{"line_number":361,"context_line":"# allowed size of an acceptor shard container after having a donor merged into"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"5f7c97a3_39beabff","line":358,"range":{"start_line":358,"start_character":24,"end_line":358,"end_character":46},"updated":"2018-05-17 23:43:44.000000000","message":"Still needs to move to its own line.","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":355,"context_line":"# below which a \u0027donor\u0027 shard container will be considered for shrinking into"},{"line_number":356,"context_line":"# another \u0027acceptor\u0027 shard container. shard_shrink_point is a percentage of"},{"line_number":357,"context_line":"# shard_container_size e.g. the default value of 5 means 5% of the"},{"line_number":358,"context_line":"# shard_container_size. shard_shrink_point \u003d 5"},{"line_number":359,"context_line":"#"},{"line_number":360,"context_line":"# When auto-sharding is enabled shard_shrink_merge_point defines the maximum"},{"line_number":361,"context_line":"# allowed size of an acceptor shard container after having a donor merged into"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"5f7c97a3_5131f3eb","line":358,"range":{"start_line":358,"start_character":24,"end_line":358,"end_character":46},"in_reply_to":"5f7c97a3_39beabff","updated":"2018-05-18 17:55:43.000000000","message":"Done","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":414,"context_line":"#"},{"line_number":415,"context_line":"# Each time the sharder dumps stats to the recon cache file it includes a list"},{"line_number":416,"context_line":"# of containers that appear to need sharding but are not yet sharding. By"},{"line_number":417,"context_line":"# default this list is limited to the top 5 containers, ordered by size. The"},{"line_number":418,"context_line":"# limit may be changed by setting recon_candidates_limit to an integer value. A"},{"line_number":419,"context_line":"# negative value implies no limit."},{"line_number":420,"context_line":"# recon_candidates_limit \u003d 5"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"5f7c97a3_d912170a","line":417,"range":{"start_line":417,"start_character":67,"end_line":417,"end_character":71},"updated":"2018-05-17 23:43:44.000000000","message":"Slightly ambiguous -- are we talking object count, max rowid, or size-on-disk?","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":414,"context_line":"#"},{"line_number":415,"context_line":"# Each time the sharder dumps stats to the recon cache file it includes a list"},{"line_number":416,"context_line":"# of containers that appear to need sharding but are not yet sharding. By"},{"line_number":417,"context_line":"# default this list is limited to the top 5 containers, ordered by size. The"},{"line_number":418,"context_line":"# limit may be changed by setting recon_candidates_limit to an integer value. A"},{"line_number":419,"context_line":"# negative value implies no limit."},{"line_number":420,"context_line":"# recon_candidates_limit \u003d 5"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"5f7c97a3_8cd68ec3","line":417,"range":{"start_line":417,"start_character":67,"end_line":417,"end_character":71},"in_reply_to":"5f7c97a3_cee563de","updated":"2018-05-18 17:55:43.000000000","message":"I propose changing to shard_container_threshold because\n\n1. shard_container_size sounds a lot like the size of a shard container, and that is in fact precisely 1/2 this value\n\n2. we may in the future be able to improve our estimation of container \u0027size\u0027, e.g. to include deleted rows, so it might be best to keep the option name a little abstract\n\nhope that is ok.","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":414,"context_line":"#"},{"line_number":415,"context_line":"# Each time the sharder dumps stats to the recon cache file it includes a list"},{"line_number":416,"context_line":"# of containers that appear to need sharding but are not yet sharding. By"},{"line_number":417,"context_line":"# default this list is limited to the top 5 containers, ordered by size. The"},{"line_number":418,"context_line":"# limit may be changed by setting recon_candidates_limit to an integer value. A"},{"line_number":419,"context_line":"# negative value implies no limit."},{"line_number":420,"context_line":"# recon_candidates_limit \u003d 5"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"5f7c97a3_d1cf43ae","line":417,"range":{"start_line":417,"start_character":67,"end_line":417,"end_character":71},"in_reply_to":"5f7c97a3_d912170a","updated":"2018-05-18 17:55:43.000000000","message":"Done","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"98f79823679f482ade4a3e2fc2793e26ac3c8449","unresolved":false,"context_lines":[{"line_number":414,"context_line":"#"},{"line_number":415,"context_line":"# Each time the sharder dumps stats to the recon cache file it includes a list"},{"line_number":416,"context_line":"# of containers that appear to need sharding but are not yet sharding. By"},{"line_number":417,"context_line":"# default this list is limited to the top 5 containers, ordered by size. The"},{"line_number":418,"context_line":"# limit may be changed by setting recon_candidates_limit to an integer value. A"},{"line_number":419,"context_line":"# negative value implies no limit."},{"line_number":420,"context_line":"# recon_candidates_limit \u003d 5"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"5f7c97a3_cee563de","line":417,"range":{"start_line":417,"start_character":67,"end_line":417,"end_character":71},"in_reply_to":"5f7c97a3_d912170a","updated":"2018-05-18 08:55:02.000000000","message":"I thought it\u0027s count. Since we define shard_container_size as the object *COUNT*, size means count always under the sharding world in my understand that feels me nervous a bit because I may always look for the definition when I review sharding staff if I\u0027m not sure the meaning.\n\nIf there is not so big impact the name change, using *COUNT* (e.g. shard_container_count) might be nice to be concrete meaning for better maintainability.","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"}],"setup.cfg":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"4288132a59208eb728c6f276fa6cca381501a5c0","unresolved":false,"context_lines":[{"line_number":72,"context_line":"    keystonemiddleware\u003e\u003d4.17.0"},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"[entry_points]"},{"line_number":75,"context_line":"console_scripts \u003d"},{"line_number":76,"context_line":"    swift-manage-shard-ranges \u003d swift.cli.manage_shard_ranges:main"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"paste.app_factory \u003d"},{"line_number":79,"context_line":"    proxy \u003d swift.proxy.server:app_factory"}],"source_content_type":"text/x-ttcn-cfg","patch_set":9,"id":"5f7c97a3_ed4ac0aa","line":76,"range":{"start_line":75,"start_character":0,"end_line":76,"end_character":66},"updated":"2018-05-17 10:19:50.000000000","message":"just wondering why we use console_scripts instead of making bin/swift-mange-shard-ranges in the bin dir. Not so much problem exists, I suppose tho.\n\nExactly, i can touch swift-manage-shard-ranges after this patch installed into my local.","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"98f79823679f482ade4a3e2fc2793e26ac3c8449","unresolved":false,"context_lines":[{"line_number":72,"context_line":"    keystonemiddleware\u003e\u003d4.17.0"},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"[entry_points]"},{"line_number":75,"context_line":"console_scripts \u003d"},{"line_number":76,"context_line":"    swift-manage-shard-ranges \u003d swift.cli.manage_shard_ranges:main"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"paste.app_factory \u003d"},{"line_number":79,"context_line":"    proxy \u003d swift.proxy.server:app_factory"}],"source_content_type":"text/x-ttcn-cfg","patch_set":9,"id":"5f7c97a3_734bdcaf","line":76,"range":{"start_line":75,"start_character":0,"end_line":76,"end_character":66},"in_reply_to":"5f7c97a3_857c3773","updated":"2018-05-18 08:55:02.000000000","message":"Make sense. This is the first step to avoid the stubs. The reason why we don\u0027t do it about sharder is the sharder requires some args handling inside.","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"db21d8a1748950f429d00b75313f7b1969f0e619","unresolved":false,"context_lines":[{"line_number":72,"context_line":"    keystonemiddleware\u003e\u003d4.17.0"},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"[entry_points]"},{"line_number":75,"context_line":"console_scripts \u003d"},{"line_number":76,"context_line":"    swift-manage-shard-ranges \u003d swift.cli.manage_shard_ranges:main"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"paste.app_factory \u003d"},{"line_number":79,"context_line":"    proxy \u003d swift.proxy.server:app_factory"}],"source_content_type":"text/x-ttcn-cfg","patch_set":9,"id":"5f7c97a3_857c3773","line":76,"range":{"start_line":75,"start_character":0,"end_line":76,"end_character":66},"in_reply_to":"5f7c97a3_ed4ac0aa","updated":"2018-05-17 19:09:18.000000000","message":"*shrug* Seemed unnecessary to provide our own stub file like https://github.com/openstack/swift/blob/master/bin/swift-ring-builder -- may as well let setuptools do that for us. See also: https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html\n\nAt some point, I\u0027d kinda like to get rid of more of those bin scripts...","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"}],"swift/cli/manage_shard_ranges.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c194e21d2ca14d3b4dcc8eeaf85cc219c14ecf51","unresolved":false,"context_lines":[{"line_number":126,"context_line":""},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"def db_info(broker, args):"},{"line_number":129,"context_line":"    print(\u0027Sharding enabled \u003d %s\u0027 % sharding_enabled(broker))"},{"line_number":130,"context_line":"    print(\u0027db_state \u003d %s\u0027 % broker.get_db_state_text())"},{"line_number":131,"context_line":"    print(\u0027Metadata:\u0027)"},{"line_number":132,"context_line":"    for k, (v, t) in broker.metadata.items():"}],"source_content_type":"text/x-python","patch_set":1,"id":"5f7c97a3_542ca546","line":129,"range":{"start_line":129,"start_character":36,"end_line":129,"end_character":61},"updated":"2018-05-09 15:53:33.000000000","message":"this returns True when the broker has shard ranges but does not yet have its own shard range i.e. is not fully enabled for sharding.\n\nso after doing a \u0027replace\u0027 then \u0027info\u0027 show sharding enabled true, which is misleading - the enable command is till needed.","commit_id":"3cdecd0fcde436bbabb149271b24c131bc06873a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c6d454d754f283340c7d8fd8d18db2c52b0d15b4","unresolved":false,"context_lines":[{"line_number":281,"context_line":"    parser.add_argument(\u0027--verbose\u0027, \u0027-v\u0027, action\u003d\u0027count\u0027,"},{"line_number":282,"context_line":"                        help\u003d\u0027Increase output verbosity\u0027)"},{"line_number":283,"context_line":"    subparsers \u003d parser.add_subparsers("},{"line_number":284,"context_line":"        help\u003d\u0027Sub-command help\u0027, title\u003d\u0027Sub-commands\u0027)"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"    # find"},{"line_number":287,"context_line":"    find_parser \u003d subparsers.add_parser("}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_e84fc031","line":284,"updated":"2018-05-08 20:02:26.000000000","message":"FYI, I always prefer when the CLI scripts \"parser\" objects are defined outside of main so they\u0027re importable (easier to monkey-patch and extend with one-off hack scripts)","commit_id":"1ea5f815dfc62e4c33df684743c5a0e4813bb5be"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c194e21d2ca14d3b4dcc8eeaf85cc219c14ecf51","unresolved":false,"context_lines":[{"line_number":281,"context_line":"    parser.add_argument(\u0027--verbose\u0027, \u0027-v\u0027, action\u003d\u0027count\u0027,"},{"line_number":282,"context_line":"                        help\u003d\u0027Increase output verbosity\u0027)"},{"line_number":283,"context_line":"    subparsers \u003d parser.add_subparsers("},{"line_number":284,"context_line":"        help\u003d\u0027Sub-command help\u0027, title\u003d\u0027Sub-commands\u0027)"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"    # find"},{"line_number":287,"context_line":"    find_parser \u003d subparsers.add_parser("}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_55b99a2b","line":284,"in_reply_to":"5f7c97a3_e84fc031","updated":"2018-05-09 15:53:33.000000000","message":"Done","commit_id":"1ea5f815dfc62e4c33df684743c5a0e4813bb5be"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"90db6b91a4cb29c337b36f6020bfede6703038b1","unresolved":false,"context_lines":[{"line_number":99,"context_line":""},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"def find_ranges(broker, args):"},{"line_number":102,"context_line":"    shard_data, delta_t \u003d _find_ranges(broker, args, sys.stderr)"},{"line_number":103,"context_line":"    print(json.dumps(shard_data, sort_keys\u003dTrue, indent\u003d2))"},{"line_number":104,"context_line":"    print(\u0027Found %d ranges in %gs (total object count %s)\u0027 %"},{"line_number":105,"context_line":"          (len(shard_data), delta_t,"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_0e88e86e","line":102,"updated":"2018-05-09 21:41:08.000000000","message":"why stderr?!","commit_id":"e689e864aea535d6f25be22903df4e3356d4b98c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"88863bba6b03d0b3a4c342509059b28b6dc6addc","unresolved":false,"context_lines":[{"line_number":99,"context_line":""},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"def find_ranges(broker, args):"},{"line_number":102,"context_line":"    shard_data, delta_t \u003d _find_ranges(broker, args, sys.stderr)"},{"line_number":103,"context_line":"    print(json.dumps(shard_data, sort_keys\u003dTrue, indent\u003d2))"},{"line_number":104,"context_line":"    print(\u0027Found %d ranges in %gs (total object count %s)\u0027 %"},{"line_number":105,"context_line":"          (len(shard_data), delta_t,"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_f14c719f","line":102,"in_reply_to":"5f7c97a3_0e88e86e","updated":"2018-05-09 22:53:49.000000000","message":"Finding shard ranges can take a while, particularly when you get into the hundreds of millions of rows. If you\u0027re feeling cavalier you can just do something like\n\n swift-manage-shard-ranges $DB find_and_replace --enable $SHARD_SIZE\n\nbut if you fail to write the shard ranges, you\u0027ve done a bunch of work and lost it all. Instead you could do something like\n\n swift-manage-shard-ranges $DB find $SHARD_SIZE \u003e shard_data.json\n swift-manage-shard-ranges $DB replace shard_data.json\n swift-manage-shard-ranges $DB enable\n\nand retry each step until it succeeds. So, while you\u0027re redirecting stdout (as opposed to teeing it, since you don\u0027t really need to look at the hundreds of shards produced), we\u0027ll drop our progress info (lines 90-91) to stderr so you know the process hasn\u0027t just hung for 45 minutes.\n\nGot better ideas? I guess maybe we could have `find` take an `output_file` arg...","commit_id":"e689e864aea535d6f25be22903df4e3356d4b98c"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"a97a6265c2249a9a48de890eccc4afe91a1f1b5b","unresolved":false,"context_lines":[{"line_number":99,"context_line":""},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"def find_ranges(broker, args):"},{"line_number":102,"context_line":"    shard_data, delta_t \u003d _find_ranges(broker, args, sys.stderr)"},{"line_number":103,"context_line":"    print(json.dumps(shard_data, sort_keys\u003dTrue, indent\u003d2))"},{"line_number":104,"context_line":"    print(\u0027Found %d ranges in %gs (total object count %s)\u0027 %"},{"line_number":105,"context_line":"          (len(shard_data), delta_t,"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_84728b7a","line":102,"in_reply_to":"5f7c97a3_f14c719f","updated":"2018-05-10 14:02:23.000000000","message":"That output file argument is the best solution, you can pass -o /foo to the tool then.\n\nThe problem for me is that 1) the printout is unconditional, there\u0027s no -q, so one must redirect outputs, and 2) it comes out to different streams depending on circumstances. I know it may be asking for too much for a technical tool, but still...","commit_id":"e689e864aea535d6f25be22903df4e3356d4b98c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"4288132a59208eb728c6f276fa6cca381501a5c0","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"def _find_ranges(broker, args, status_file\u003dNone):"},{"line_number":83,"context_line":"    start \u003d last_report \u003d time.time()"},{"line_number":84,"context_line":"    limit \u003d 5 if status_file else -1"},{"line_number":85,"context_line":"    shard_data, last_found \u003d broker.find_shard_ranges("},{"line_number":86,"context_line":"        args.rows_per_shard, limit\u003dlimit)"},{"line_number":87,"context_line":"    if shard_data:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_28c74648","line":84,"range":{"start_line":84,"start_character":4,"end_line":84,"end_character":36},"updated":"2018-05-17 10:19:50.000000000","message":"Perhaps, this always results in limit \u003d 5?\nIt looks like all callers in this file set either sys.stdout or sys.stderr.\n\nAnd the next question is... what happen if the rows_per_shard and the limit is smaller than enough sharding.\n\nProbably I need to learn more about sharding but the case I concerned is likely:\n\n1. I have 1 million rows in the container db to be attempted to shard. (but might not know the actual number at first)\n2. I set rows_per_shard to 1,000 and the limit is 5 by default, so find_ranges will report just the first 5 shard ranges.\n3. Oops, unfortunately, I called `find_and_replace --enable --force` for the state.\n\nThen... does the container broker start to work sharding with 1000 x 5 shards and the rest of 0.95 million rows db??? Or any protections (or cancellation commands) exist?","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"db21d8a1748950f429d00b75313f7b1969f0e619","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"def _find_ranges(broker, args, status_file\u003dNone):"},{"line_number":83,"context_line":"    start \u003d last_report \u003d time.time()"},{"line_number":84,"context_line":"    limit \u003d 5 if status_file else -1"},{"line_number":85,"context_line":"    shard_data, last_found \u003d broker.find_shard_ranges("},{"line_number":86,"context_line":"        args.rows_per_shard, limit\u003dlimit)"},{"line_number":87,"context_line":"    if shard_data:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_45729f7e","line":84,"range":{"start_line":84,"start_character":4,"end_line":84,"end_character":36},"in_reply_to":"5f7c97a3_28c74648","updated":"2018-05-17 19:09:18.000000000","message":"This is just for batching so you\u0027ll get periodic progress when running interactively -- we find `limit` at a time until `last_found` is True, regardless.","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"98f79823679f482ade4a3e2fc2793e26ac3c8449","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"def _find_ranges(broker, args, status_file\u003dNone):"},{"line_number":83,"context_line":"    start \u003d last_report \u003d time.time()"},{"line_number":84,"context_line":"    limit \u003d 5 if status_file else -1"},{"line_number":85,"context_line":"    shard_data, last_found \u003d broker.find_shard_ranges("},{"line_number":86,"context_line":"        args.rows_per_shard, limit\u003dlimit)"},{"line_number":87,"context_line":"    if shard_data:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_b3adf432","line":84,"range":{"start_line":84,"start_character":4,"end_line":84,"end_character":36},"in_reply_to":"5f7c97a3_45729f7e","updated":"2018-05-18 08:55:02.000000000","message":"Got it. find_shard_ranges is re-called at line95 in the while loop.\n\nHmm. I tried to improve the code like as\n\ndiff --git a/swift/cli/manage_shard_ranges.py b/swift/cli/manage_shard_ranges.py\nindex acbc364..8e05e2e 100644\n--- a/swift/cli/manage_shard_ranges.py\n+++ b/swift/cli/manage_shard_ranges.py\n@@ -82,19 +82,19 @@ def _check_own_shard_range(broker, args):\n def _find_ranges(broker, args, status_file\u003dNone):\n     start \u003d last_report \u003d time.time()\n     limit \u003d 5 if status_file else -1\n-    shard_data, last_found \u003d broker.find_shard_ranges(\n-        args.rows_per_shard, limit\u003dlimit)\n-    if shard_data:\n-        while not last_found:\n-            if last_report + 10 \u003c time.time():\n-                print(\u0027Found %d ranges in %gs; looking for more...\u0027 % (\n-                    len(shard_data), time.time() - start), file\u003dstatus_file)\n-                last_report \u003d time.time()\n-            # prefix doesn\u0027t matter since we aren\u0027t persisting it\n-            found_ranges \u003d make_shard_ranges(broker, shard_data, \u0027.shards_\u0027)\n-            more_shard_data, last_found \u003d broker.find_shard_ranges(\n-                args.rows_per_shard, existing_ranges\u003dfound_ranges, limit\u003d5)\n-            shard_data.extend(more_shard_data)\n+    shard_data \u003d []\n+    last_found \u003d False\n+    while not last_found:\n+        if last_report + 10 \u003c time.time():\n+            print(\u0027Found %d ranges in %gs; looking for more...\u0027 % (\n+                  len(shard_data), time.time() - start), file\u003dstatus_file)\n+            last_report \u003d time.time()\n+        # prefix doesn\u0027t matter since we aren\u0027t persisting it\n+        found_ranges \u003d make_shard_ranges(broker, shard_data, \u0027.shards_\u0027)\n+        more_shard_data, last_found \u003d broker.find_shard_ranges(\n+            args.rows_per_shard, existing_ranges\u003dfound_ranges, limit\u003d5)\n+        shard_data.extend(more_shard_data)\n+\n     return shard_data, time.time() - start\n\nbut it doesn\u0027t reduce the code so much. so either is fine to me. Perhaps, mine would be easy to know, find_shard_ranges called for each time.","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"def _find_ranges(broker, args, status_file\u003dNone):"},{"line_number":83,"context_line":"    start \u003d last_report \u003d time.time()"},{"line_number":84,"context_line":"    limit \u003d 5 if status_file else -1"},{"line_number":85,"context_line":"    shard_data, last_found \u003d broker.find_shard_ranges("},{"line_number":86,"context_line":"        args.rows_per_shard, limit\u003dlimit)"},{"line_number":87,"context_line":"    if shard_data:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_ec614213","line":84,"range":{"start_line":84,"start_character":4,"end_line":84,"end_character":36},"in_reply_to":"5f7c97a3_b3adf432","updated":"2018-05-18 17:55:43.000000000","message":"with the diff limit is never -1\n\nshall we follow this up on master?","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"98f79823679f482ade4a3e2fc2793e26ac3c8449","unresolved":false,"context_lines":[{"line_number":93,"context_line":"            # prefix doesn\u0027t matter since we aren\u0027t persisting it"},{"line_number":94,"context_line":"            found_ranges \u003d make_shard_ranges(broker, shard_data, \u0027.shards_\u0027)"},{"line_number":95,"context_line":"            more_shard_data, last_found \u003d broker.find_shard_ranges("},{"line_number":96,"context_line":"                args.rows_per_shard, existing_ranges\u003dfound_ranges, limit\u003d5)"},{"line_number":97,"context_line":"            shard_data.extend(more_shard_data)"},{"line_number":98,"context_line":"    return shard_data, time.time() - start"},{"line_number":99,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_0e5fdbe5","line":96,"range":{"start_line":96,"start_character":73,"end_line":96,"end_character":74},"updated":"2018-05-18 08:55:02.000000000","message":"Hah? This is not each `limit` but each 5 shards by hard-code. so IIRC, limit\u003dlimit correctly.","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":93,"context_line":"            # prefix doesn\u0027t matter since we aren\u0027t persisting it"},{"line_number":94,"context_line":"            found_ranges \u003d make_shard_ranges(broker, shard_data, \u0027.shards_\u0027)"},{"line_number":95,"context_line":"            more_shard_data, last_found \u003d broker.find_shard_ranges("},{"line_number":96,"context_line":"                args.rows_per_shard, existing_ranges\u003dfound_ranges, limit\u003d5)"},{"line_number":97,"context_line":"            shard_data.extend(more_shard_data)"},{"line_number":98,"context_line":"    return shard_data, time.time() - start"},{"line_number":99,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_cc7006e0","line":96,"range":{"start_line":96,"start_character":73,"end_line":96,"end_character":74},"in_reply_to":"5f7c97a3_0e5fdbe5","updated":"2018-05-18 17:55:43.000000000","message":"if limit is -1 at line 84 then find_shard_ranges at line 85 should return all shard ranges so line 96 wouldn\u0027t be reached. But I agree its not clear. And I think you pointed out before that maybe limit is always 5, if the func is vener called with status_file\u003dNone.","commit_id":"a0250b49b4cab8e91c9faf4d808b6e32fb7dc5f2"}],"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":424,"context_line":"        min \u003d \u0027, greater than %s\u0027 % minimum if minimum is not None else \u0027\u0027"},{"line_number":425,"context_line":"        max \u003d \u0027, less than %s\u0027 % maximum if maximum is not None else \u0027\u0027"},{"line_number":426,"context_line":"        raise ValueError(\u0027Config option must be a number%s%s, not \"%s\".\u0027 %"},{"line_number":427,"context_line":"                         (min, max, value))"},{"line_number":428,"context_line":""},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"def config_auto_int_value(value, default):"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_991d7f11","line":427,"range":{"start_line":427,"start_character":26,"end_line":427,"end_character":34},"updated":"2018-05-17 23:43:44.000000000","message":"nit: shadow built-ins","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":424,"context_line":"        min \u003d \u0027, greater than %s\u0027 % minimum if minimum is not None else \u0027\u0027"},{"line_number":425,"context_line":"        max \u003d \u0027, less than %s\u0027 % maximum if maximum is not None else \u0027\u0027"},{"line_number":426,"context_line":"        raise ValueError(\u0027Config option must be a number%s%s, not \"%s\".\u0027 %"},{"line_number":427,"context_line":"                         (min, max, value))"},{"line_number":428,"context_line":""},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"def config_auto_int_value(value, default):"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_3149d75f","line":427,"range":{"start_line":427,"start_character":26,"end_line":427,"end_character":34},"in_reply_to":"5f7c97a3_991d7f11","updated":"2018-05-18 17:55:43.000000000","message":"Done","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"}],"swift/container/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":758,"context_line":"            than or equal to this value will not be removed."},{"line_number":759,"context_line":"        :param upper: defines the upper bound of object names that will be"},{"line_number":760,"context_line":"            removed; names less than or equal to this value will be removed;"},{"line_number":761,"context_line":"            names greater than this value will not be removed."},{"line_number":762,"context_line":"        :param max_row: if specified only rows less than or equal to max_row"},{"line_number":763,"context_line":"            will be removed"},{"line_number":764,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_f9d11331","line":761,"updated":"2018-05-17 23:43:44.000000000","message":"Worth calling out how the empty string is a special case?","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":758,"context_line":"            than or equal to this value will not be removed."},{"line_number":759,"context_line":"        :param upper: defines the upper bound of object names that will be"},{"line_number":760,"context_line":"            removed; names less than or equal to this value will be removed;"},{"line_number":761,"context_line":"            names greater than this value will not be removed."},{"line_number":762,"context_line":"        :param max_row: if specified only rows less than or equal to max_row"},{"line_number":763,"context_line":"            will be removed"},{"line_number":764,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_71dbcf7c","line":761,"in_reply_to":"5f7c97a3_f9d11331","updated":"2018-05-18 17:55:43.000000000","message":"Done","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"}],"swift/container/sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c194e21d2ca14d3b4dcc8eeaf85cc219c14ecf51","unresolved":false,"context_lines":[{"line_number":197,"context_line":"    def __init__(self, ref, cursor\u003d\u0027\u0027, max_row\u003dNone, cleave_to_row\u003dNone,"},{"line_number":198,"context_line":"                 last_cleave_to_row\u003dNone, cleaving_done\u003dFalse,"},{"line_number":199,"context_line":"                 misplaced_done\u003dFalse):"},{"line_number":200,"context_line":"        self.ref \u003d ref"},{"line_number":201,"context_line":"        self._cursor \u003d None"},{"line_number":202,"context_line":"        self.cursor \u003d cursor"},{"line_number":203,"context_line":"        self.max_row \u003d max_row"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f6a8fd7_21e064cc","line":200,"range":{"start_line":200,"start_character":8,"end_line":200,"end_character":22},"updated":"2018-05-09 15:53:33.000000000","message":"I\u0027ve been burnt by calling \n\n  context \u003d CleavingContext(broker)\n\nwe should make it blow up if ref is not a string","commit_id":"3cdecd0fcde436bbabb149271b24c131bc06873a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c6d454d754f283340c7d8fd8d18db2c52b0d15b4","unresolved":false,"context_lines":[{"line_number":1082,"context_line":"                len(created_ranges))"},{"line_number":1083,"context_line":"        return len(created_ranges)"},{"line_number":1084,"context_line":""},{"line_number":1085,"context_line":"    def _cleave_shard_range(self, broker, cleaving_context, shard_range):"},{"line_number":1086,"context_line":"        self.logger.info(\"Cleaving \u0027%s\u0027: %r from row %s\","},{"line_number":1087,"context_line":"                         broker.path, shard_range,"},{"line_number":1088,"context_line":"                         cleaving_context.last_cleave_to_row)"}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_68cf30b0","line":1085,"updated":"2018-05-08 20:02:26.000000000","message":"If you understand how this method works you have a good grasp of the sharder daemon!","commit_id":"1ea5f815dfc62e4c33df684743c5a0e4813bb5be"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":299,"context_line":"        logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":300,"context_line":"        super(ContainerSharder, self).__init__(conf, logger\u003dlogger)"},{"line_number":301,"context_line":"        self.shards_account_prefix \u003d ("},{"line_number":302,"context_line":"            (conf.get(\u0027auto_create_account_prefix\u0027) or \u0027.\u0027) + \u0027shards_\u0027)"},{"line_number":303,"context_line":""},{"line_number":304,"context_line":"        def percent_value(key, default):"},{"line_number":305,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_1932cfa8","line":302,"range":{"start_line":302,"start_character":63,"end_line":302,"end_character":70},"updated":"2018-05-17 23:43:44.000000000","message":"Maybe should be configurable?","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":299,"context_line":"        logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":300,"context_line":"        super(ContainerSharder, self).__init__(conf, logger\u003dlogger)"},{"line_number":301,"context_line":"        self.shards_account_prefix \u003d ("},{"line_number":302,"context_line":"            (conf.get(\u0027auto_create_account_prefix\u0027) or \u0027.\u0027) + \u0027shards_\u0027)"},{"line_number":303,"context_line":""},{"line_number":304,"context_line":"        def percent_value(key, default):"},{"line_number":305,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_6ca1b295","line":302,"range":{"start_line":302,"start_character":63,"end_line":302,"end_character":70},"in_reply_to":"5f7c97a3_1932cfa8","updated":"2018-05-18 17:55:43.000000000","message":"follow on","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":36,"context_line":"    # Ensure these are all lowercase"},{"line_number":37,"context_line":"    pass_through_headers \u003d [\u0027x-container-read\u0027, \u0027x-container-write\u0027,"},{"line_number":38,"context_line":"                            \u0027x-container-sync-key\u0027, \u0027x-container-sync-to\u0027,"},{"line_number":39,"context_line":"                            \u0027x-versions-location\u0027, \u0027x-container-sharding\u0027]"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def __init__(self, app, account_name, container_name, **kwargs):"},{"line_number":42,"context_line":"        super(ContainerController, self).__init__(app)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_f9ba73df","line":39,"updated":"2018-05-17 23:43:44.000000000","message":"I don\u0027t think this is needed -- we do the translation to sysmeta before calling generate_request_headers, yeah?","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":36,"context_line":"    # Ensure these are all lowercase"},{"line_number":37,"context_line":"    pass_through_headers \u003d [\u0027x-container-read\u0027, \u0027x-container-write\u0027,"},{"line_number":38,"context_line":"                            \u0027x-container-sync-key\u0027, \u0027x-container-sync-to\u0027,"},{"line_number":39,"context_line":"                            \u0027x-versions-location\u0027, \u0027x-container-sharding\u0027]"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"    def __init__(self, app, account_name, container_name, **kwargs):"},{"line_number":42,"context_line":"        super(ContainerController, self).__init__(app)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_8ccc4e62","line":39,"in_reply_to":"5f7c97a3_f9ba73df","updated":"2018-05-18 17:55:43.000000000","message":"Done","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":265,"context_line":"        if req.environ.get(\u0027reseller_request\u0027, False) and \\"},{"line_number":266,"context_line":"                \u0027X-Container-Sharding\u0027 in req.headers:"},{"line_number":267,"context_line":"            req.headers[get_sys_meta_prefix(\u0027container\u0027) + \u0027Sharding\u0027] \u003d \\"},{"line_number":268,"context_line":"                config_true_value(req.headers[\u0027X-Container-Sharding\u0027])"},{"line_number":269,"context_line":"        length_limit \u003d self.get_name_length_limit()"},{"line_number":270,"context_line":"        if len(self.container_name) \u003e length_limit:"},{"line_number":271,"context_line":"            resp \u003d HTTPBadRequest(request\u003dreq)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_99cb3f71","line":268,"updated":"2018-05-17 23:43:44.000000000","message":"Should this get str()ed?","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":265,"context_line":"        if req.environ.get(\u0027reseller_request\u0027, False) and \\"},{"line_number":266,"context_line":"                \u0027X-Container-Sharding\u0027 in req.headers:"},{"line_number":267,"context_line":"            req.headers[get_sys_meta_prefix(\u0027container\u0027) + \u0027Sharding\u0027] \u003d \\"},{"line_number":268,"context_line":"                config_true_value(req.headers[\u0027X-Container-Sharding\u0027])"},{"line_number":269,"context_line":"        length_limit \u003d self.get_name_length_limit()"},{"line_number":270,"context_line":"        if len(self.container_name) \u003e length_limit:"},{"line_number":271,"context_line":"            resp \u003d HTTPBadRequest(request\u003dreq)"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_6c269219","line":268,"in_reply_to":"5f7c97a3_99cb3f71","updated":"2018-05-18 17:55:43.000000000","message":"looks like HeaderEnvironProxy does it for us but no harm doing here to","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8505f0d8ee930e9814a55172a6ad869e5d2300eb","unresolved":false,"context_lines":[{"line_number":317,"context_line":"        if req.environ.get(\u0027reseller_request\u0027, False) and \\"},{"line_number":318,"context_line":"                \u0027X-Container-Sharding\u0027 in req.headers:"},{"line_number":319,"context_line":"            req.headers[get_sys_meta_prefix(\u0027container\u0027) + \u0027Sharding\u0027] \u003d \\"},{"line_number":320,"context_line":"                config_true_value(req.headers[\u0027X-Container-Sharding\u0027])"},{"line_number":321,"context_line":"        account_partition, accounts, container_count \u003d \\"},{"line_number":322,"context_line":"            self.account_info(self.account_name, req)"},{"line_number":323,"context_line":"        if not accounts:"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_39c8eb72","line":320,"updated":"2018-05-17 23:43:44.000000000","message":"ditto","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"aeaa0fd34763a7c84b8142f849e2303f2ca9c03e","unresolved":false,"context_lines":[{"line_number":317,"context_line":"        if req.environ.get(\u0027reseller_request\u0027, False) and \\"},{"line_number":318,"context_line":"                \u0027X-Container-Sharding\u0027 in req.headers:"},{"line_number":319,"context_line":"            req.headers[get_sys_meta_prefix(\u0027container\u0027) + \u0027Sharding\u0027] \u003d \\"},{"line_number":320,"context_line":"                config_true_value(req.headers[\u0027X-Container-Sharding\u0027])"},{"line_number":321,"context_line":"        account_partition, accounts, container_count \u003d \\"},{"line_number":322,"context_line":"            self.account_info(self.account_name, req)"},{"line_number":323,"context_line":"        if not accounts:"}],"source_content_type":"text/x-python","patch_set":10,"id":"5f7c97a3_0c7d9ee6","line":320,"in_reply_to":"5f7c97a3_39c8eb72","updated":"2018-05-18 17:55:43.000000000","message":"Done","commit_id":"ecad8e5a7a5dbabdc37377f0d70d8dde1013321e"}],"test/probe/test_sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2f05f981fb930f40363640e50618144e194ae0ac","unresolved":false,"context_lines":[{"line_number":395,"context_line":"        # verify parameterised listing of a container during sharding"},{"line_number":396,"context_line":"        all_obj_names \u003d self._make_object_names(4 * self.max_shard_size)"},{"line_number":397,"context_line":"        obj_names \u003d all_obj_names[::2]"},{"line_number":398,"context_line":"        self.put_objects(obj_names)"},{"line_number":399,"context_line":"        # choose some names approx in middle of each expected shard range"},{"line_number":400,"context_line":"        markers \u003d ["},{"line_number":401,"context_line":"            obj_names[i] for i in range(self.max_shard_size / 4,"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_7b7f25ab","line":398,"updated":"2018-05-18 21:01:25.000000000","message":"Hit an error:\n\n ERROR: test_sharding_listing (test.probe.test_sharder.TestContainerShardingUTF8)\n ----------------------------------------------------------------------\n Traceback (most recent call last):\n   File \"/vagrant/swift/test/probe/test_sharder.py\", line 398, in test_sharding_listing\n     self.put_objects(obj_names)\n   File \"/vagrant/swift/test/probe/test_sharder.py\", line 137, in put_objects\n     client.put_object(self.url, self.token, self.container_name, obj)\n   File \"/vagrant/python-swiftclient/swiftclient/client.py\", line 1342, in put_object\n     raise ClientException.from_response(resp, \u0027Object PUT failed\u0027, body)\n ClientException: Object PUT failed: http://127.0.0.1:8080/v1/AUTH_test/container-d02faebe-fcbd-41bc-a1a3-35103e9c371c/obj-%C3%A4%C3%AA%C3%AC%C3%B2%C3%BB-0000ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo 500 Internal Error   An error occurred\n\n\nproxy-server says\n\n May 18 20:07:02 saio proxy-server: Error: An error occurred:\n Traceback (most recent call last):\n   File \"/vagrant/swift/swift/common/middleware/catch_errors.py\", line 41, in handle_request\n     resp \u003d self._app_call(env)\n   File \"/vagrant/swift/swift/common/wsgi.py\", line 1088, in _app_call\n     resp \u003d self.app(env, self._start_response)\n   File \"/vagrant/swift/swift/common/middleware/gatekeeper.py\", line 123, in __call__\n     return self.app(env, gatekeeper_response)\n   File \"/vagrant/swift/swift/common/middleware/healthcheck.py\", line 47, in __call__\n     if req.path \u003d\u003d \u0027/healthcheck\u0027:\n   File \"/vagrant/swift/swift/common/swob.py\", line 934, in path\n     self.environ[\u0027PATH_INFO\u0027])\n   File \"/usr/lib/python2.7/urllib.py\", line 1298, in quote\n     return \u0027\u0027.join(map(quoter, s))\n KeyError: u\u0027\\xc3\u0027 (txn: tx55498733f9324c63818d9-005aff3266)\n\nHow are we getting *unicode* into PATH_INFO??\n\nOn the plus side (I guess?) it\u0027s sporadic.","commit_id":"26418140108281ae5ac11004ebf33b7b3e08c74d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2f05f981fb930f40363640e50618144e194ae0ac","unresolved":false,"context_lines":[{"line_number":1807,"context_line":"        # second shard range"},{"line_number":1808,"context_line":"        self.sharders.once(additional_args\u003d\u0027--partitions\u003d%s\u0027 % shard_part,"},{"line_number":1809,"context_line":"                           number\u003dmisplaced_node[\u0027id\u0027] + 1)"},{"line_number":1810,"context_line":"        self.assert_container_listing([\u0027alpha\u0027])"},{"line_number":1811,"context_line":"        self.assert_container_object_count(0)  # root not yet updated"},{"line_number":1812,"context_line":""},{"line_number":1813,"context_line":"        # run sharder to get root updated"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_5bd6e951","line":1810,"updated":"2018-05-18 21:01:25.000000000","message":"Hit a reliable failure here. Not sure *why*, though; logs show the sharders.once call doing about what I\u0027d expect:\n\n May 18 19:45:45 saio container-sharder-6031: Begin container sharder \"once\" mode\n May 18 19:45:45 saio container-sharder-6031: Container sharder cycle starting, auto-sharding True\n May 18 19:45:45 saio container-sharder-6031: (Override partitions: 5)\n May 18 19:45:45 saio container-sharder-6031: Starting processing .shards_AUTH_test/container-d336441e-5488-4410-abb2-00741a8695ab-f45c0b267d4dc33864c628616724c0b7-1526672722.94967-0 state sharded\n May 18 19:45:45 saio container-sharder-6031: Updating shard from root {\u0027object_count\u0027: 0, \u0027lower\u0027: \u0027\u0027, \u0027name\u0027: \u0027.shards_AUTH_test/container-d336441e-5488-4410-abb2-00741a8695ab-f45c0b267d4dc33864c628616724c0b7-1526672722.94967-0\u0027, \u0027upper\u0027: \u0027obj-0024\u0027, \u0027deleted\u0027: 1, \u0027bytes_used\u0027: 0, \u0027meta_timestamp\u0027: \u00271526672743.90304\u0027, \u0027state_timestamp\u0027: \u00271526672739.02419\u0027, \u0027epoch\u0027: \u00271526672739.02419\u0027, \u0027state\u0027: 70, \u0027timestamp\u0027: \u00271526672743.73401\u0027}\n May 18 19:45:45 saio container-sharder-6031: Looking for misplaced objects in .shards_AUTH_test/container-d336441e-5488-4410-abb2-00741a8695ab-f45c0b267d4dc33864c628616724c0b7-1526672722.94967-0 (/srv/node3/sdb3/containers/5/0e2/05099f87b91b40918372ddd01e8880e2/05099f87b91b40918372ddd01e8880e2_1526672739.02419.db)\n May 18 19:45:45 saio container-sharder-6031: misplaced object source bounds [(\u0027\u0027, \u0027\u0027)]\n May 18 19:45:45 saio container-sharder-6031: got 1 objects from /srv/node3/sdb3/containers/5/0e2/05099f87b91b40918372ddd01e8880e2/05099f87b91b40918372ddd01e8880e2_1526672739.02419.db in 0.000236034393311s\n May 18 19:45:45 saio container-sharder-6031: moving misplaced objects found in range ShardRange\u003cMinBound to MaxBound as of 1526672739.02419, (0, 0) as of 1526672739.05606, active as of 1526672722.94967\u003e\n May 18 19:45:45 saio container-sharder-6031: Replicating db /srv/node3/sdb3/containers/17/c18/110295c542f1b637af97636b0de55c18/110295c542f1b637af97636b0de55c18.db\n May 18 19:45:45 saio container-sharder-6031: /srv/node3/sdb3/containers/17/c18/110295c542f1b637af97636b0de55c18/110295c542f1b637af97636b0de55c18.db synced 2 shard ranges to 127.0.0.1:6021/sdb2\n May 18 19:45:45 saio container-sharder-6031: /srv/node3/sdb3/containers/17/c18/110295c542f1b637af97636b0de55c18/110295c542f1b637af97636b0de55c18.db usyncing chunks to 127.0.0.1:6021/sdb2, starting at row 75\n May 18 19:45:45 saio container-sharder-6031: /srv/node3/sdb3/containers/17/c18/110295c542f1b637af97636b0de55c18/110295c542f1b637af97636b0de55c18.db usyncing chunks to 127.0.0.1:6021/sdb2, finished at row 76 (0.00512815s)\n May 18 19:45:45 saio container-sharder-6031: /srv/node3/sdb3/containers/17/c18/110295c542f1b637af97636b0de55c18/110295c542f1b637af97636b0de55c18.db synced 2 shard ranges to 127.0.0.1:6041/sdb4\n May 18 19:45:45 saio container-sharder-6031: /srv/node3/sdb3/containers/17/c18/110295c542f1b637af97636b0de55c18/110295c542f1b637af97636b0de55c18.db usyncing chunks to 127.0.0.1:6041/sdb4, starting at row 75\n May 18 19:45:45 saio container-sharder-6031: /srv/node3/sdb3/containers/17/c18/110295c542f1b637af97636b0de55c18/110295c542f1b637af97636b0de55c18.db usyncing chunks to 127.0.0.1:6041/sdb4, finished at row 76 (0.00424099s)\n May 18 19:45:45 saio container-sharder-6031: Moved 1 misplaced objects\n May 18 19:45:45 saio container-sharder-6031: Finished handling misplaced objects\n May 18 19:45:45 saio container-sharder-6031: Finished processing .shards_AUTH_test/container-d336441e-5488-4410-abb2-00741a8695ab-f45c0b267d4dc33864c628616724c0b7-1526672722.94967-0 state sharded\n May 18 19:45:45 saio container-sharder-6031: Since Fri May 18 19:45:45 2018 visited - attempted:1 success:1 failure:0 skipped:0 completed:0\n May 18 19:45:45 saio container-sharder-6031: Since Fri May 18 19:45:45 2018 scanned - attempted:0 success:0 failure:0 found:0 min_time:0 max_time:0\n May 18 19:45:45 saio container-sharder-6031: Since Fri May 18 19:45:45 2018 created - attempted:0 success:0 failure:0\n May 18 19:45:45 saio container-sharder-6031: Since Fri May 18 19:45:45 2018 cleaved - attempted:0 success:0 failure:0 min_time:0 max_time:0\n May 18 19:45:45 saio container-sharder-6031: Since Fri May 18 19:45:45 2018 misplaced - attempted:1 success:1 failure:0 found:1 placed:1 unplaced:0\n May 18 19:45:45 saio container-sharder-6031: Since Fri May 18 19:45:45 2018 audit_root - attempted:0 success:0 failure:0\n May 18 19:45:45 saio container-sharder-6031: Since Fri May 18 19:45:45 2018 audit_shard - attempted:1 success:1 failure:0\n May 18 19:45:45 saio container-sharder-6031: Container sharder \"once\" mode completed: 0.15s","commit_id":"26418140108281ae5ac11004ebf33b7b3e08c74d"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c194e21d2ca14d3b4dcc8eeaf85cc219c14ecf51","unresolved":false,"context_lines":[{"line_number":102,"context_line":"            timestamps[0], timestamps[1], timestamps[3])"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"class TestSharder(BaseTestSharder):"},{"line_number":106,"context_line":"    def test_init(self):"},{"line_number":107,"context_line":"        def do_test(conf, expected):"},{"line_number":108,"context_line":"            with mock.patch("}],"source_content_type":"text/x-python","patch_set":1,"id":"5f7c97a3_88ae83e5","line":105,"range":{"start_line":105,"start_character":6,"end_line":105,"end_character":17},"updated":"2018-05-09 15:53:33.000000000","message":"rename to TestContainerSharder ?","commit_id":"3cdecd0fcde436bbabb149271b24c131bc06873a"}]}