)]}'
{"specs/in_progress/container_sharding.rst":[{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"717d05efa20b6c9e92ff8665f4bc7a5864bbcbae","unresolved":false,"context_lines":[{"line_number":266,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"},{"line_number":267,"context_line":"As discussed at the Vancouver summit, to speed up finding the right place to put an object, the root container now stores a copy of all distributed nodes, this way an object will no right away where is _should_ go and not potentially product many extra container GET\u0027s to find the location."},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"The container-sharder has been changed to updated the root container and the trie getting split when sharding. And will need some work to audit/check if the parent shard container and parent knows about the container, if not, quarantine(?)"},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"CountingTrie and the candidate subtrie algorithm"},{"line_number":272,"context_line":"--------------------------------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_598c1e35","line":269,"updated":"2015-07-13 15:14:24.000000000","message":"should be \"will know right away\" and I think \"product\" was meant to be \"produce\".  Also \"changed to update\"","commit_id":"f6863e5514cf0e95bfa8efe7ff6756e93ebad28d"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"582cce7375ad6c1d20f59c8fd73aca76edde4c69","unresolved":false,"context_lines":[{"line_number":266,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"},{"line_number":267,"context_line":"As discussed at the Vancouver summit, to speed up finding the right place to put an object, the root container now stores a copy of all distributed nodes, this way an object will no right away where is _should_ go and not potentially product many extra container GET\u0027s to find the location."},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"The container-sharder has been changed to updated the root container and the trie getting split when sharding. And will need some work to audit/check if the parent shard container and parent knows about the container, if not, quarantine(?)"},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"CountingTrie and the candidate subtrie algorithm"},{"line_number":272,"context_line":"--------------------------------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_483c2fc8","line":269,"in_reply_to":"9a41bdd9_598c1e35","updated":"2015-07-14 02:42:53.000000000","message":"Done","commit_id":"f6863e5514cf0e95bfa8efe7ff6756e93ebad28d"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"717d05efa20b6c9e92ff8665f4bc7a5864bbcbae","unresolved":false,"context_lines":[{"line_number":468,"context_line":"#. If there are misplaced objects deal with them first, as fixing a problem is more important then sharding."},{"line_number":469,"context_line":""},{"line_number":470,"context_line":"#. If there are good candidate subtries, if none, see if this container needs to merge back into patent, if not continue on to next container."},{"line_number":471,"context_line":""},{"line_number":472,"context_line":"#. Split the trie creating a new container, then update the existing one, deleting sharded objects and nodes."},{"line_number":473,"context_line":""},{"line_number":474,"context_line":"And now for a more detailed version, all the work happens in the _one_shard_pass method, which is called by run once, or continually"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_19a3f650","line":471,"updated":"2015-07-13 15:14:24.000000000","message":"is \"patent\" supposed to be \"parent\"?","commit_id":"f6863e5514cf0e95bfa8efe7ff6756e93ebad28d"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"582cce7375ad6c1d20f59c8fd73aca76edde4c69","unresolved":false,"context_lines":[{"line_number":468,"context_line":"#. If there are misplaced objects deal with them first, as fixing a problem is more important then sharding."},{"line_number":469,"context_line":""},{"line_number":470,"context_line":"#. If there are good candidate subtries, if none, see if this container needs to merge back into patent, if not continue on to next container."},{"line_number":471,"context_line":""},{"line_number":472,"context_line":"#. Split the trie creating a new container, then update the existing one, deleting sharded objects and nodes."},{"line_number":473,"context_line":""},{"line_number":474,"context_line":"And now for a more detailed version, all the work happens in the _one_shard_pass method, which is called by run once, or continually"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a41bdd9_083627e6","line":471,"in_reply_to":"9a41bdd9_19a3f650","updated":"2015-07-14 02:42:53.000000000","message":"Done","commit_id":"f6863e5514cf0e95bfa8efe7ff6756e93ebad28d"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"80a6e37bbb2e3e544c640d8b3fc037af93725465","unresolved":false,"context_lines":[{"line_number":265,"context_line":"Root container short circuiting"},{"line_number":266,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"},{"line_number":267,"context_line":"As discussed at the Vancouver summit, to speed up finding the right place to put an object, the root container now stores a copy of all distributed nodes, this way an object will know right away where is _should_ go and reduce the number of extra container GET\u0027s to find the location."},{"line_number":268,"context_line":""},{"line_number":269,"context_line":"The container-sharder has been changed to update the root container and the trie getting split with the new distributed node when sharding. And will need some work to audit/check if the parent shard container and root container knows about the container, if not, quarantine(?)"},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"CountingTrie and the candidate subtrie algorithm"}],"source_content_type":"text/x-rst","patch_set":2,"id":"9a41bdd9_230d32e8","line":268,"updated":"2015-07-14 14:56:27.000000000","message":"oops - forgot the \"is\" should be \"it\".  pretty small typo, so you could just roll it in if you have another patch for some other update.","commit_id":"96cbe647aa103951540099784886d2a9b6e4b207"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"10c0726a5fe25c66b284353794377683eb42f21e","unresolved":false,"context_lines":[{"line_number":216,"context_line":"  |                                     |   |   X-Container-Sysmeta-Shard-Account: Auth_test |"},{"line_number":217,"context_line":"  |    AUTH_test/cont                   |   |   X-Container-Sysmeta-Shard-Container: cont    |"},{"line_number":218,"context_line":"  |                                     |   |                                                |"},{"line_number":219,"context_line":"  +-------------------------------------+   | Subtrie:                                       |"},{"line_number":220,"context_line":"  | Meta:                               |   |   Root: \u0027obj\u0027                                  |"},{"line_number":221,"context_line":"  |   X-Container-Sysmeta-Sharding: on  |   |                                                |"},{"line_number":222,"context_line":"  |   \u003cany other user or swift meta\u003e    |   +------------------------------------------------+"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da20952f_576c6ccf","line":219,"updated":"2015-09-05 15:08:06.000000000","message":"s/Subtrie/Subtree/ ?","commit_id":"d695007ec13601607b875b73f0f3203f1ef99f1e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"82963981768478a2d1a64532534b05f9fdf6d3b1","unresolved":false,"context_lines":[{"line_number":216,"context_line":"  |                                     |   |   X-Container-Sysmeta-Shard-Account: Auth_test |"},{"line_number":217,"context_line":"  |    AUTH_test/cont                   |   |   X-Container-Sysmeta-Shard-Container: cont    |"},{"line_number":218,"context_line":"  |                                     |   |                                                |"},{"line_number":219,"context_line":"  +-------------------------------------+   | Subtrie:                                       |"},{"line_number":220,"context_line":"  | Meta:                               |   |   Root: \u0027obj\u0027                                  |"},{"line_number":221,"context_line":"  |   X-Container-Sysmeta-Sharding: on  |   |                                                |"},{"line_number":222,"context_line":"  |   \u003cany other user or swift meta\u003e    |   +------------------------------------------------+"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da20952f_7d37d8e2","line":219,"in_reply_to":"da20952f_576c6ccf","updated":"2015-09-08 06:43:01.000000000","message":"Nope, subtrie. Trie is the name of a prefix tree.","commit_id":"d695007ec13601607b875b73f0f3203f1ef99f1e"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"10c0726a5fe25c66b284353794377683eb42f21e","unresolved":false,"context_lines":[{"line_number":273,"context_line":""},{"line_number":274,"context_line":"Growing - CountingTrie and the candidate subtrie algorithm"},{"line_number":275,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"},{"line_number":276,"context_line":"The candidate subtrie algoritm is what is used to find elegable subtries to shard (grow)."},{"line_number":277,"context_line":"The original candidate subtrie algorithm implementation \"worked\" but definitely wasn\u0027t scalable. To properly scan for subtries, all objects had to be loaded into a ShardTrie, but as soon as we got to larger containers (10s of thousands) this tended to use too much RAM. A memory effcient approach was required. Enter the CountingTrie."},{"line_number":278,"context_line":""},{"line_number":279,"context_line":"The candidate rules are still the same::"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da20952f_f716e047","line":276,"updated":"2015-09-05 15:08:06.000000000","message":"s/algoritm/algorithm/","commit_id":"d695007ec13601607b875b73f0f3203f1ef99f1e"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"10c0726a5fe25c66b284353794377683eb42f21e","unresolved":false,"context_lines":[{"line_number":274,"context_line":"Growing - CountingTrie and the candidate subtrie algorithm"},{"line_number":275,"context_line":"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"},{"line_number":276,"context_line":"The candidate subtrie algoritm is what is used to find elegable subtries to shard (grow)."},{"line_number":277,"context_line":"The original candidate subtrie algorithm implementation \"worked\" but definitely wasn\u0027t scalable. To properly scan for subtries, all objects had to be loaded into a ShardTrie, but as soon as we got to larger containers (10s of thousands) this tended to use too much RAM. A memory effcient approach was required. Enter the CountingTrie."},{"line_number":278,"context_line":""},{"line_number":279,"context_line":"The candidate rules are still the same::"},{"line_number":280,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"da20952f_3738d8d2","line":277,"updated":"2015-09-05 15:08:06.000000000","message":"s/effcient/efficient/","commit_id":"d695007ec13601607b875b73f0f3203f1ef99f1e"}]}
