)]}'
{"bin/swift-container-sharder":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"1a1ced50_4b6fd92a","updated":"2017-03-21 23:30:26.000000000","message":"note to self: cool; seems to match other (non-WSGI) daemons pretty well.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"}],"doc/source/overview_container_sharding.rst":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Like the rest of swift\u0027s API, a user can turn on sharding by sending a piece of metadata to"},{"line_number":47,"context_line":"the container in question. This allows a user to choose which containers are sharded at a"},{"line_number":48,"context_line":"container level. To turn on sharding, all a user has to do is send a `X-Container-Sharding: on`"},{"line_number":49,"context_line":"header via either a PUT or POST to the container::"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"  curl -i -X POST -H \u0027X-Container-Sharding: on\u0027 \\"},{"line_number":52,"context_line":"  https://1.2.3.4/v1/\u003caccount\u003e/\u003ccontainer\u003e"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_b98706d6","line":49,"updated":"2017-02-28 23:27:47.000000000","message":"Why is this client-driven? Will it always need to be?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Like the rest of swift\u0027s API, a user can turn on sharding by sending a piece of metadata to"},{"line_number":47,"context_line":"the container in question. This allows a user to choose which containers are sharded at a"},{"line_number":48,"context_line":"container level. To turn on sharding, all a user has to do is send a `X-Container-Sharding: on`"},{"line_number":49,"context_line":"header via either a PUT or POST to the container::"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"  curl -i -X POST -H \u0027X-Container-Sharding: on\u0027 \\"},{"line_number":52,"context_line":"  https://1.2.3.4/v1/\u003caccount\u003e/\u003ccontainer\u003e"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_640778ca","line":49,"in_reply_to":"da36d5c6_b98706d6","updated":"2017-03-03 02:39:40.000000000","message":"I hope not. The plan is:\n\n1. client driven (reseller) as beta.\n2. client driven (reseller or user?) not beta\n3. Once we are happy with it, turned on permanently","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":104,"context_line":"shard that is small enough to merge into a neighbour shard (shrink). "},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"`shard_shrink_point` is what size, as a percentage of `shard_container_size`, the list of objects"},{"line_number":107,"context_line":"should be before being deemed small enough to **attempt** to shrink."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"`shard_shrink_merge_point` is the next step, once a container is small enough, this percentage of"},{"line_number":110,"context_line":"`shard_conatiner_size` tells the sharder if the neighbour + the small container is small enough to"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_75bc7ddc","line":107,"updated":"2017-02-27 03:34:53.000000000","message":"the list would be the entire container list, not the list in the individual shard, right?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":104,"context_line":"shard that is small enough to merge into a neighbour shard (shrink). "},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"`shard_shrink_point` is what size, as a percentage of `shard_container_size`, the list of objects"},{"line_number":107,"context_line":"should be before being deemed small enough to **attempt** to shrink."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"`shard_shrink_merge_point` is the next step, once a container is small enough, this percentage of"},{"line_number":110,"context_line":"`shard_conatiner_size` tells the sharder if the neighbour + the small container is small enough to"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_84daac4a","line":107,"in_reply_to":"da36d5c6_75bc7ddc","updated":"2017-03-03 02:39:40.000000000","message":"Well this is talking about shrinking the current shard into a neighbour shard. So a container-shards object size, although as I am paranoid, I get a majority quorum on what that size really is... maybe we could this quorum check if it\u0027s causing too many requests.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":107,"context_line":"should be before being deemed small enough to **attempt** to shrink."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"`shard_shrink_merge_point` is the next step, once a container is small enough, this percentage of"},{"line_number":110,"context_line":"`shard_conatiner_size` tells the sharder if the neighbour + the small container is small enough to"},{"line_number":111,"context_line":"complete the merge. To get more information, see the shrinking_ section below."},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"-----------------------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_b53fd585","line":110,"updated":"2017-02-27 03:34:53.000000000","message":"s/conatiner/container/","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":115,"context_line":"-----------------------------"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"The basic and simplified idea is rather simple. Firstly, to enable container sharding, pass in a"},{"line_number":118,"context_line":"`X-Container-Sharding: On` X-Header via either PUT or POST::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"  curl -i -H \u0027X-Auth-Token: \u003ctoken\u003e\u0027 -H  \\"},{"line_number":121,"context_line":"  \u0027X-Container-Sharding: On\u0027 \u003curl\u003e/\u003caccount\u003e/\u003ccontainer\u003e -X PUT"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_feb12902","line":118,"updated":"2017-02-28 23:27:47.000000000","message":"I think we want to require that the user be a reseller admin.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":115,"context_line":"-----------------------------"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"The basic and simplified idea is rather simple. Firstly, to enable container sharding, pass in a"},{"line_number":118,"context_line":"`X-Container-Sharding: On` X-Header via either PUT or POST::"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"  curl -i -H \u0027X-Auth-Token: \u003ctoken\u003e\u0027 -H  \\"},{"line_number":121,"context_line":"  \u0027X-Container-Sharding: On\u0027 \u003curl\u003e/\u003caccount\u003e/\u003ccontainer\u003e -X PUT"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_24c74065","line":118,"in_reply_to":"da36d5c6_feb12902","updated":"2017-03-03 02:39:40.000000000","message":"Agreed, that would be a great v1 beta","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":121,"context_line":"  \u0027X-Container-Sharding: On\u0027 \u003curl\u003e/\u003caccount\u003e/\u003ccontainer\u003e -X PUT"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"Once enabled, when a container gets too full, pivot points are found to split the container on. If"},{"line_number":124,"context_line":"it\u0027s a very large container, then it\u0027ll split multiple times. If you have had sharding turned on then"},{"line_number":125,"context_line":"as the container gets big enough, say at 1 million objects, a pivot point is found which will be used"},{"line_number":126,"context_line":"to split the container. In this latter scenario, the container will only need to be split once, meaning"},{"line_number":127,"context_line":"sharding is more efficient the sooner it is activated. "}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_95bd71b9","line":124,"updated":"2017-02-27 03:34:53.000000000","message":"nit: \"have had\" suggests sharding could have been on, but now be off. I guess it should really be \"If you have sharding on...\"","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":121,"context_line":"  \u0027X-Container-Sharding: On\u0027 \u003curl\u003e/\u003caccount\u003e/\u003ccontainer\u003e -X PUT"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"Once enabled, when a container gets too full, pivot points are found to split the container on. If"},{"line_number":124,"context_line":"it\u0027s a very large container, then it\u0027ll split multiple times. If you have had sharding turned on then"},{"line_number":125,"context_line":"as the container gets big enough, say at 1 million objects, a pivot point is found which will be used"},{"line_number":126,"context_line":"to split the container. In this latter scenario, the container will only need to be split once, meaning"},{"line_number":127,"context_line":"sharding is more efficient the sooner it is activated. "}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_24aca08b","line":124,"in_reply_to":"da36d5c6_95bd71b9","updated":"2017-03-03 02:39:40.000000000","message":"that does read better and is more to the point.. it is what I meant, so anything that improves that message is +2 for me ;)","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":160,"context_line":""},{"line_number":161,"context_line":".. image:: images/sharding_snip2.png"},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"Here the sharder has found that object \u0027f\u0027 would be a good first slice. So we mark it. The sharder can"},{"line_number":164,"context_line":"then shard at this point:"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":".. image:: images/sharding_snip3.png"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_752d5dfd","line":163,"range":{"start_line":163,"start_character":54,"end_line":163,"end_character":71},"updated":"2017-02-27 03:34:53.000000000","message":"based on it being at container_shard_size/2?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":160,"context_line":""},{"line_number":161,"context_line":".. image:: images/sharding_snip2.png"},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"Here the sharder has found that object \u0027f\u0027 would be a good first slice. So we mark it. The sharder can"},{"line_number":164,"context_line":"then shard at this point:"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":".. image:: images/sharding_snip3.png"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_e4cc486c","line":163,"range":{"start_line":163,"start_character":54,"end_line":163,"end_character":71},"in_reply_to":"da36d5c6_752d5dfd","updated":"2017-03-03 02:39:40.000000000","message":"Yup","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":163,"context_line":"Here the sharder has found that object \u0027f\u0027 would be a good first slice. So we mark it. The sharder can"},{"line_number":164,"context_line":"then shard at this point:"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":".. image:: images/sharding_snip3.png"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"At this point we\u0027ll pause for a moment, because now you can see that a shard is placed in a hidden"},{"line_number":169,"context_line":"account that maps to the account (.sharded_acct). This allows us to have no namespace collisions but"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_8ab815e7","line":166,"updated":"2017-02-28 23:27:47.000000000","message":"So the container will have some (random?) suffix appended -- will we need to worry about length limits? What if cont was already at or around max_container_name_length?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":163,"context_line":"Here the sharder has found that object \u0027f\u0027 would be a good first slice. So we mark it. The sharder can"},{"line_number":164,"context_line":"then shard at this point:"},{"line_number":165,"context_line":""},{"line_number":166,"context_line":".. image:: images/sharding_snip3.png"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"At this point we\u0027ll pause for a moment, because now you can see that a shard is placed in a hidden"},{"line_number":169,"context_line":"account that maps to the account (.sharded_acct). This allows us to have no namespace collisions but"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_6414d89f","line":166,"in_reply_to":"da36d5c6_8ab815e7","updated":"2017-03-03 02:39:40.000000000","message":"Yup agree, good spotting. We may need to disable the check or only check on incoming requests from outside swift?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":166,"context_line":".. image:: images/sharding_snip3.png"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"At this point we\u0027ll pause for a moment, because now you can see that a shard is placed in a hidden"},{"line_number":169,"context_line":"account that maps to the account (.sharded_acct). This allows us to have no namespace collisions but"},{"line_number":170,"context_line":"also keeps all the shards out of view from users of the account."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Continuing on, we can see that there is now a shard covering a range, and the root container has now"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_1526e114","line":169,"range":{"start_line":169,"start_character":48,"end_line":169,"end_character":50},"updated":"2017-02-27 03:34:53.000000000","message":"nit: and given a unique name","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":200,"context_line":""},{"line_number":201,"context_line":"In the slicing example above, we would find 1 pivot point and then shard it. This works well when"},{"line_number":202,"context_line":"sharding is turned on from the start, as you\u0027ll only ever need to make 1 slice. But on larger"},{"line_number":203,"context_line":"containers where more then 1 slice is required it just wont scale. The rule is we want to get to the"},{"line_number":204,"context_line":"sharded state as quickly as possible, so we need to speed things up, we need to shard in batches."},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"The sharder daemon lives on all the container servers. It will periodically scan all sharded containers"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_35e4c5aa","line":203,"range":{"start_line":203,"start_character":22,"end_line":203,"end_character":26},"updated":"2017-02-27 03:34:53.000000000","message":"s/then/than/","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":203,"context_line":"containers where more then 1 slice is required it just wont scale. The rule is we want to get to the"},{"line_number":204,"context_line":"sharded state as quickly as possible, so we need to speed things up, we need to shard in batches."},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"The sharder daemon lives on all the container servers. It will periodically scan all sharded containers"},{"line_number":207,"context_line":"to see if they need to be sharded. And if so, will try and shard. We wont tie up the sharder and shard"},{"line_number":208,"context_line":"a very large container in one go. Instead it will find pivots and/or shard in batches before moving to the next container. On the next run, it\u0027ll continue where it left off."},{"line_number":209,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_aa7879c5","line":206,"updated":"2017-02-28 23:27:47.000000000","message":"All *unsharded* containers, surely? As well as the shards; just not the sharded root containers.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":203,"context_line":"containers where more then 1 slice is required it just wont scale. The rule is we want to get to the"},{"line_number":204,"context_line":"sharded state as quickly as possible, so we need to speed things up, we need to shard in batches."},{"line_number":205,"context_line":""},{"line_number":206,"context_line":"The sharder daemon lives on all the container servers. It will periodically scan all sharded containers"},{"line_number":207,"context_line":"to see if they need to be sharded. And if so, will try and shard. We wont tie up the sharder and shard"},{"line_number":208,"context_line":"a very large container in one go. Instead it will find pivots and/or shard in batches before moving to the next container. On the next run, it\u0027ll continue where it left off."},{"line_number":209,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_e48b68c1","line":206,"in_reply_to":"da36d5c6_aa7879c5","updated":"2017-03-03 02:39:40.000000000","message":"Well in essence it scans all containers.. maybe we can somehow hook into the container auditor to give us all shards so we done have to loop through the same?\n\nIt only acts upon sharded (and root) containers. The root containers because I want to audit the pivot ranges table, and try and clean up any overlaps.. and figure out what to do if there is a gap :(","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":314,"context_line":""},{"line_number":315,"context_line":"Sharding itself is rather straight forward:"},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"1. Create an empty container database in a hondoff location locally."},{"line_number":318,"context_line":"2. Insert all the stale data from the locked (large) database."},{"line_number":319,"context_line":"3. Check the pivot databases object holding table to see if there is anything there that is related"},{"line_number":320,"context_line":"   and merge it into the handoff database."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_95459a18","line":317,"updated":"2017-02-28 23:27:47.000000000","message":"How do we ensure we don\u0027t revert to the primary location while we\u0027re still inserting?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":314,"context_line":""},{"line_number":315,"context_line":"Sharding itself is rather straight forward:"},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"1. Create an empty container database in a hondoff location locally."},{"line_number":318,"context_line":"2. Insert all the stale data from the locked (large) database."},{"line_number":319,"context_line":"3. Check the pivot databases object holding table to see if there is anything there that is related"},{"line_number":320,"context_line":"   and merge it into the handoff database."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_d5152949","line":317,"range":{"start_line":317,"start_character":43,"end_line":317,"end_character":50},"updated":"2017-02-27 03:34:53.000000000","message":"handoff","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":314,"context_line":""},{"line_number":315,"context_line":"Sharding itself is rather straight forward:"},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"1. Create an empty container database in a hondoff location locally."},{"line_number":318,"context_line":"2. Insert all the stale data from the locked (large) database."},{"line_number":319,"context_line":"3. Check the pivot databases object holding table to see if there is anything there that is related"},{"line_number":320,"context_line":"   and merge it into the handoff database."}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_24e66044","line":317,"in_reply_to":"da36d5c6_95459a18","updated":"2017-03-03 02:39:40.000000000","message":"Shard lock. I describe it in the later section.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24185060289d1257181d1df48b56f4a1e4d898c1","unresolved":false,"context_lines":[{"line_number":323,"context_line":""},{"line_number":324,"context_line":".. image:: images/sharding_lock4.png"},{"line_number":325,"context_line":""},{"line_number":326,"context_line":"This image just demonstrates that that the process continues. The sharder is now sharding the next"},{"line_number":327,"context_line":"pivot. While the scanner node is still searching for pivots. See we have new one \u0027linux\u0027. Now the"},{"line_number":328,"context_line":"root container only needs to deal with requests dealing with objects whose name is greater then linux."},{"line_number":329,"context_line":"So load will continue to diminish."}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_152340a8","line":326,"range":{"start_line":326,"start_character":29,"end_line":326,"end_character":38},"updated":"2017-02-27 03:34:53.000000000","message":"that","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":342,"context_line":"This was an example of sharding a root container. However as container shards grow, the same thing"},{"line_number":343,"context_line":"happens to them except for one small difference. We don\u0027t keep the sharded pivot container around."},{"line_number":344,"context_line":"The references to shards are in the root container only, so a container shard once it hits the SHARDED"},{"line_number":345,"context_line":"state, can be deleted."},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"------------------------"},{"line_number":348,"context_line":"Sharding: Under the hood"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_95ecba90","line":345,"updated":"2017-02-28 23:27:47.000000000","message":"...but the shard that needs sharding isn\u0027t necessarily on the same server as the root container any more -- that seems problematic...","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":342,"context_line":"This was an example of sharding a root container. However as container shards grow, the same thing"},{"line_number":343,"context_line":"happens to them except for one small difference. We don\u0027t keep the sharded pivot container around."},{"line_number":344,"context_line":"The references to shards are in the root container only, so a container shard once it hits the SHARDED"},{"line_number":345,"context_line":"state, can be deleted."},{"line_number":346,"context_line":""},{"line_number":347,"context_line":"------------------------"},{"line_number":348,"context_line":"Sharding: Under the hood"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_c4b9047b","line":345,"in_reply_to":"da36d5c6_95ecba90","updated":"2017-03-03 02:39:40.000000000","message":"True.. we may need to rethink and be careful here. Although we are talking about deleting in the swift sense and not unlinking, so there is a reclaim_age.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":389,"context_line":"      created_at TEXT,"},{"line_number":390,"context_line":"      meta_timestamp TEXT,"},{"line_number":391,"context_line":"      deleted INTEGER DEFAULT 0"},{"line_number":392,"context_line":"  );"},{"line_number":393,"context_line":""},{"line_number":394,"context_line":"You\u0027ll notice there are two timestamps, the created_at will track when it was created or when"},{"line_number":395,"context_line":"something major happens like a smaller shard gets merged into it. The meta_timestamp is updated"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_f06344da","line":392,"updated":"2017-02-28 23:27:47.000000000","message":"Any indexing? Or are we assuming that this should never get big enough to be a problem?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":389,"context_line":"      created_at TEXT,"},{"line_number":390,"context_line":"      meta_timestamp TEXT,"},{"line_number":391,"context_line":"      deleted INTEGER DEFAULT 0"},{"line_number":392,"context_line":"  );"},{"line_number":393,"context_line":""},{"line_number":394,"context_line":"You\u0027ll notice there are two timestamps, the created_at will track when it was created or when"},{"line_number":395,"context_line":"something major happens like a smaller shard gets merged into it. The meta_timestamp is updated"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_04779cd3","line":392,"in_reply_to":"da36d5c6_f06344da","updated":"2017-03-03 02:39:40.000000000","message":"Hmm, Not yet, but your right, it might be smart to index. at least name and delete seeing as we use it in merge_items like we do with the objects table.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":450,"context_line":""},{"line_number":451,"context_line":"The class is pretty basic, it stores the timestamps, stats, lower and upper values. The _contains_,"},{"line_number":452,"context_line":"_lt_, _gt_, iter and _eq_ methods have been overridden so it can do checks against a string or another"},{"line_number":453,"context_line":"PivotRange."},{"line_number":454,"context_line":""},{"line_number":455,"context_line":"The class also contains some extra helper methods:"},{"line_number":456,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_96e664ae","line":453,"updated":"2017-02-28 23:27:47.000000000","message":"How will __gt__ and __lt__ work for overlapping ranges? Does it matter?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":450,"context_line":""},{"line_number":451,"context_line":"The class is pretty basic, it stores the timestamps, stats, lower and upper values. The _contains_,"},{"line_number":452,"context_line":"_lt_, _gt_, iter and _eq_ methods have been overridden so it can do checks against a string or another"},{"line_number":453,"context_line":"PivotRange."},{"line_number":454,"context_line":""},{"line_number":455,"context_line":"The class also contains some extra helper methods:"},{"line_number":456,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_8451cc38","line":453,"in_reply_to":"da36d5c6_96e664ae","updated":"2017-03-03 02:39:40.000000000","message":"can\u0027t remember, I think greater then means lower must be bigger then upper, other wise it\u0027s overlapping, and so not greater, and visa-versa.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":480,"context_line":"easiest and most obvious way is to use a new method in the ContainerBroker `build_pivot_ranges()`."},{"line_number":481,"context_line":""},{"line_number":482,"context_line":"The second is to ask the container for a list of pivot nodes rather than objects. This is done with a"},{"line_number":483,"context_line":"GET to the container server, but with the items\u003dpivot parameter set::"},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"  GET /acc/cont?items\u003dpivot\u0026format\u003djson"},{"line_number":486,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_d6b29c93","line":483,"updated":"2017-02-28 23:27:47.000000000","message":"I feel like this ought to be stripped out of client requests by the proxy\u0027s container controller unless/until we have some use-case in mind.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":480,"context_line":"easiest and most obvious way is to use a new method in the ContainerBroker `build_pivot_ranges()`."},{"line_number":481,"context_line":""},{"line_number":482,"context_line":"The second is to ask the container for a list of pivot nodes rather than objects. This is done with a"},{"line_number":483,"context_line":"GET to the container server, but with the items\u003dpivot parameter set::"},{"line_number":484,"context_line":""},{"line_number":485,"context_line":"  GET /acc/cont?items\u003dpivot\u0026format\u003djson"},{"line_number":486,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_24574027","line":483,"in_reply_to":"da36d5c6_d6b29c93","updated":"2017-03-03 02:39:40.000000000","message":"Good idea, otherwise we are leaking internal swift information.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ee36c8a4f47259f01667a35456455efd5379cd76","unresolved":false,"context_lines":[{"line_number":496,"context_line":"replicated with a sharded one, and being eventually consistent, some of the objects in the unsharded"},{"line_number":497,"context_line":"one might actually exist and need to be merged into a different shard. The sharded container holds all"},{"line_number":498,"context_line":"its objects in the leaves, leaving the root container’s object table as an object holding table. In this"},{"line_number":499,"context_line":"case the objects can be synced into the root container\u0027s holding table, and we can simply let the"},{"line_number":500,"context_line":"sharder pick them up and move them the shards themselves."},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"pending and merge_items"}],"source_content_type":"text/x-rst","patch_set":3,"id":"da36d5c6_9642e466","line":499,"updated":"2017-02-28 23:27:47.000000000","message":"How many objects are we talking here? Like, would we ever have a situation where we sync a giant, predates-sharding container to an already-sharded node and slow everything to a crawl again?","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bedf908fd24473b09e723891ef8857fdffd8a3c1","unresolved":false,"context_lines":[{"line_number":496,"context_line":"replicated with a sharded one, and being eventually consistent, some of the objects in the unsharded"},{"line_number":497,"context_line":"one might actually exist and need to be merged into a different shard. The sharded container holds all"},{"line_number":498,"context_line":"its objects in the leaves, leaving the root container’s object table as an object holding table. In this"},{"line_number":499,"context_line":"case the objects can be synced into the root container\u0027s holding table, and we can simply let the"},{"line_number":500,"context_line":"sharder pick them up and move them the shards themselves."},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"pending and merge_items"}],"source_content_type":"text/x-rst","patch_set":3,"id":"ba2be162_84654cb6","line":499,"in_reply_to":"da36d5c6_9642e466","updated":"2017-03-03 02:39:40.000000000","message":"Yup, I guess thats possible :(\n\nHowever, if were in a SHARDING to an UNSHARDED  state we force usync at least so it wont be everything, only x number of diffs. + we maintain the sync_points so hopefully not the whole thing anyway.","commit_id":"b0f29b2d309d3a33fb54ec37915e15717c56aa38"}],"etc/container-server.conf-sample":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":310,"context_line":"#"},{"line_number":311,"context_line":"# Container sharder specific settings"},{"line_number":312,"context_line":"#"},{"line_number":313,"context_line":"# Shrinking has a few more settings, the first is the shard_shrink_point. This is the"},{"line_number":314,"context_line":"# point at which to start looking to shrink the sharded container. The point is a"},{"line_number":315,"context_line":"# percentage of shard_container_size. 50 means 50% of the shard_container_size."},{"line_number":316,"context_line":"# shard_shrink_point \u003d 50"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_8b2d51aa","line":313,"range":{"start_line":313,"start_character":16,"end_line":313,"end_character":35},"updated":"2017-03-21 23:30:26.000000000","message":"Relative to... sharding, I guess? It\u0027s a little awkward since this is the first thing we see.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":310,"context_line":"#"},{"line_number":311,"context_line":"# Container sharder specific settings"},{"line_number":312,"context_line":"#"},{"line_number":313,"context_line":"# Shrinking has a few more settings, the first is the shard_shrink_point. This is the"},{"line_number":314,"context_line":"# point at which to start looking to shrink the sharded container. The point is a"},{"line_number":315,"context_line":"# percentage of shard_container_size. 50 means 50% of the shard_container_size."},{"line_number":316,"context_line":"# shard_shrink_point \u003d 50"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_f454ddfe","line":313,"range":{"start_line":313,"start_character":16,"end_line":313,"end_character":35},"in_reply_to":"1a1ced50_8b2d51aa","updated":"2017-03-22 01:13:03.000000000","message":"True, the wording should be fixed. at some point there might have been something before it :)","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":312,"context_line":"#"},{"line_number":313,"context_line":"# Shrinking has a few more settings, the first is the shard_shrink_point. This is the"},{"line_number":314,"context_line":"# point at which to start looking to shrink the sharded container. The point is a"},{"line_number":315,"context_line":"# percentage of shard_container_size. 50 means 50% of the shard_container_size."},{"line_number":316,"context_line":"# shard_shrink_point \u003d 50"},{"line_number":317,"context_line":"#"},{"line_number":318,"context_line":"# Shard_shrink_merge_point is the similar to shard_shrink_size, but is the max size"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_66404c35","line":315,"range":{"start_line":315,"start_character":16,"end_line":315,"end_character":36},"updated":"2017-03-21 23:30:26.000000000","message":"It feels a little weird referencing a config value that isn\u0027t actually defined here, but rather in swift.conf. Do we do that for other config values?\n\nShould we maybe call out where this is defined in the help text?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":312,"context_line":"#"},{"line_number":313,"context_line":"# Shrinking has a few more settings, the first is the shard_shrink_point. This is the"},{"line_number":314,"context_line":"# point at which to start looking to shrink the sharded container. The point is a"},{"line_number":315,"context_line":"# percentage of shard_container_size. 50 means 50% of the shard_container_size."},{"line_number":316,"context_line":"# shard_shrink_point \u003d 50"},{"line_number":317,"context_line":"#"},{"line_number":318,"context_line":"# Shard_shrink_merge_point is the similar to shard_shrink_size, but is the max size"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_343cb5c7","line":315,"range":{"start_line":315,"start_character":16,"end_line":315,"end_character":36},"in_reply_to":"1a1ced50_66404c35","updated":"2017-03-22 01:13:03.000000000","message":"I have moved this setting from constraints to the sharder. So we will now talk about it here where it belongs.\n\nThe only places that needs the shard_container_size is the sharder and container backend. The method in the backend is to find the next pivot point, so now it\u0027s a variable passed in.\n\nthis will also make testing eaiser ;)","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":315,"context_line":"# percentage of shard_container_size. 50 means 50% of the shard_container_size."},{"line_number":316,"context_line":"# shard_shrink_point \u003d 50"},{"line_number":317,"context_line":"#"},{"line_number":318,"context_line":"# Shard_shrink_merge_point is the similar to shard_shrink_size, but is the max size"},{"line_number":319,"context_line":"# the sharded container plus the neighbour needs to be to continue with the merge."},{"line_number":320,"context_line":"# Again this is a percentage of shard_container_size. "},{"line_number":321,"context_line":"# shard_shrink_merge_point \u003d 75"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_a65de416","line":318,"range":{"start_line":318,"start_character":27,"end_line":318,"end_character":41},"updated":"2017-03-21 23:30:26.000000000","message":"is similar","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":326,"context_line":"# combined object count \u003c\u003d 750k."},{"line_number":327,"context_line":"#"},{"line_number":328,"context_line":"# When sharding one primary node becomes the scanner and scans for pivot"},{"line_number":329,"context_line":"# points. In one shard pass a scanner will search for up to"},{"line_number":330,"context_line":"# shard_scanner_batch_size pivots before moveing to the next container and"},{"line_number":331,"context_line":"# will find more on the next pass. You can change the number to search for below"},{"line_number":332,"context_line":"# shard_scanner_batch_size \u003d 10"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_cb4bc985","line":329,"range":{"start_line":329,"start_character":17,"end_line":329,"end_character":27},"updated":"2017-03-21 23:30:26.000000000","message":"What is a \"shard pass\"? I guess the sharding analogue of a replication cycle or reconstruction cycle or auditor cycle? Should we maybe call it a \"sharding cycle\" for consistency?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":326,"context_line":"# combined object count \u003c\u003d 750k."},{"line_number":327,"context_line":"#"},{"line_number":328,"context_line":"# When sharding one primary node becomes the scanner and scans for pivot"},{"line_number":329,"context_line":"# points. In one shard pass a scanner will search for up to"},{"line_number":330,"context_line":"# shard_scanner_batch_size pivots before moveing to the next container and"},{"line_number":331,"context_line":"# will find more on the next pass. You can change the number to search for below"},{"line_number":332,"context_line":"# shard_scanner_batch_size \u003d 10"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_14cdf9b6","line":329,"range":{"start_line":329,"start_character":17,"end_line":329,"end_character":27},"in_reply_to":"1a1ced50_cb4bc985","updated":"2017-03-22 01:13:03.000000000","message":"Yeah, maybe the name should change to shard cycle in stead of pass. That way we are using a known term. Will add it to the trello.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":327,"context_line":"#"},{"line_number":328,"context_line":"# When sharding one primary node becomes the scanner and scans for pivot"},{"line_number":329,"context_line":"# points. In one shard pass a scanner will search for up to"},{"line_number":330,"context_line":"# shard_scanner_batch_size pivots before moveing to the next container and"},{"line_number":331,"context_line":"# will find more on the next pass. You can change the number to search for below"},{"line_number":332,"context_line":"# shard_scanner_batch_size \u003d 10"},{"line_number":333,"context_line":"#"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_8be391a3","line":330,"range":{"start_line":330,"start_character":41,"end_line":330,"end_character":48},"updated":"2017-03-21 23:30:26.000000000","message":"moving","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":335,"context_line":"# onto the next."},{"line_number":336,"context_line":"# shard_batch_size \u003d 2"},{"line_number":337,"context_line":"#"},{"line_number":338,"context_line":"# Because the container-sharder is a type of container-replicator, it accepts"},{"line_number":339,"context_line":"# the same configuration."},{"line_number":340,"context_line":"#"},{"line_number":341,"context_line":"# vm_test_mode \u003d no"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_a6ec04f4","line":338,"range":{"start_line":338,"start_character":10,"end_line":338,"end_character":65},"updated":"2017-03-21 23:30:26.000000000","message":"Huh. I need to find out more about that...","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":335,"context_line":"# onto the next."},{"line_number":336,"context_line":"# shard_batch_size \u003d 2"},{"line_number":337,"context_line":"#"},{"line_number":338,"context_line":"# Because the container-sharder is a type of container-replicator, it accepts"},{"line_number":339,"context_line":"# the same configuration."},{"line_number":340,"context_line":"#"},{"line_number":341,"context_line":"# vm_test_mode \u003d no"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_54cdb183","line":338,"range":{"start_line":338,"start_character":10,"end_line":338,"end_character":65},"in_reply_to":"1a1ced50_a6ec04f4","updated":"2017-03-22 01:13:03.000000000","message":"yup, because the way we move data around is via replication, it makes sense that the sharder knows how to do that. and the best way is to make a special container replicator, with shading smarts.\n\nThis way when it want to move something, it can just run it\u0027s local replicate method.\n\nI\u0027ve added a question to the trello in regards if we should always use this replication feature to push pivot ranges to the root container so we can potentially avoid the root containers .pending file as we want the root container\u0027s pivot ranges table as up to date as possible.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":346,"context_line":"# Time in seconds to wait between replication passes"},{"line_number":347,"context_line":"# interval \u003d 30"},{"line_number":348,"context_line":"# run_pause is deprecated, use interval instead"},{"line_number":349,"context_line":"# run_pause \u003d 30"},{"line_number":350,"context_line":"#"},{"line_number":351,"context_line":"# node_timeout \u003d 10"},{"line_number":352,"context_line":"# conn_timeout \u003d 0.5"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_2b47fd77","line":349,"updated":"2017-03-21 23:30:26.000000000","message":"Can we get away with only ever doing interval, if run_pause is already deprecated?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":346,"context_line":"# Time in seconds to wait between replication passes"},{"line_number":347,"context_line":"# interval \u003d 30"},{"line_number":348,"context_line":"# run_pause is deprecated, use interval instead"},{"line_number":349,"context_line":"# run_pause \u003d 30"},{"line_number":350,"context_line":"#"},{"line_number":351,"context_line":"# node_timeout \u003d 10"},{"line_number":352,"context_line":"# conn_timeout \u003d 0.5"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_f4a85d8e","line":349,"in_reply_to":"1a1ced50_2b47fd77","updated":"2017-03-22 01:13:03.000000000","message":"yes, yes we can. As you can see, this config file was pulled from container replicators a year or so ago :)","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":352,"context_line":"# conn_timeout \u003d 0.5"},{"line_number":353,"context_line":"#"},{"line_number":354,"context_line":"# The replicator also performs reclamation"},{"line_number":355,"context_line":"# reclaim_age \u003d 604800"},{"line_number":356,"context_line":"#"},{"line_number":357,"context_line":"# Allow rsync to compress data which is transmitted to destination node"},{"line_number":358,"context_line":"# during sync. However, this is applicable only when destination node is in"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_66d04cf7","line":355,"updated":"2017-03-21 23:30:26.000000000","message":"How do we typically recommend ops decide on a reclaim_age? Something something replication cycle time, maybe? Should that now include considerations of sharder cycle time?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":352,"context_line":"# conn_timeout \u003d 0.5"},{"line_number":353,"context_line":"#"},{"line_number":354,"context_line":"# The replicator also performs reclamation"},{"line_number":355,"context_line":"# reclaim_age \u003d 604800"},{"line_number":356,"context_line":"#"},{"line_number":357,"context_line":"# Allow rsync to compress data which is transmitted to destination node"},{"line_number":358,"context_line":"# during sync. However, this is applicable only when destination node is in"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"1a1ced50_949ac96c","line":355,"in_reply_to":"1a1ced50_66d04cf7","updated":"2017-03-22 01:13:03.000000000","message":"yes? But maybe we dont need to specifically say this here, as reclaim age could be in DEFAULT now right?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":318,"context_line":"#"},{"line_number":319,"context_line":"# Shrinking, when smaller shards merge, have a few settings, the first is the"},{"line_number":320,"context_line":"# shard_shrink_point. This is the point at which to start looking to shrink the"},{"line_number":321,"context_line":"# sharded container. The point is a percentage of shard_container_size. 5 means 50%"},{"line_number":322,"context_line":"# of the shard_container_size."},{"line_number":323,"context_line":"# shard_shrink_point \u003d 50"},{"line_number":324,"context_line":"#"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"ffe62b97_77349693","line":321,"updated":"2017-03-28 20:05:12.000000000","message":"s/5/50/","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":330,"context_line":"# For example, say the shard_container_size is 1 million, shard_shrink_point is 50"},{"line_number":331,"context_line":"# and shard_shrink_merge_point is 75. Meaning start looking to merge with neighbours when"},{"line_number":332,"context_line":"# the current shard container is \u003c\u003d 500k, and only merge if this container and a neighbour\u0027s"},{"line_number":333,"context_line":"# combined object count \u003c\u003d 750k."},{"line_number":334,"context_line":"#"},{"line_number":335,"context_line":"# When sharding one primary node becomes the scanner and scans for pivot"},{"line_number":336,"context_line":"# points. In one shard pass a scanner will search for up to"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"ffe62b97_b8491f8c","line":333,"updated":"2017-03-28 20:05:12.000000000","message":"I think that mostly if not always, comments about options in the sample conf files comes before the option. IDK if this info could be in the docs?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":345,"context_line":"# Because the container-sharder is a type of container-replicator, it accepts"},{"line_number":346,"context_line":"# the same configuration."},{"line_number":347,"context_line":"#"},{"line_number":348,"context_line":"# vm_test_mode \u003d no"},{"line_number":349,"context_line":"# per_diff \u003d 1000"},{"line_number":350,"context_line":"# max_diffs \u003d 100"},{"line_number":351,"context_line":"# concurrency \u003d 8"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"ffe62b97_33ef70b1","line":348,"updated":"2017-03-28 20:05:12.000000000","message":"this is deprecated in replicator conf","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":353,"context_line":"# Time in seconds to wait between replication passes"},{"line_number":354,"context_line":"# interval \u003d 30"},{"line_number":355,"context_line":"# run_pause is deprecated, use interval instead"},{"line_number":356,"context_line":"# run_pause \u003d 30"},{"line_number":357,"context_line":"#"},{"line_number":358,"context_line":"# node_timeout \u003d 10"},{"line_number":359,"context_line":"# conn_timeout \u003d 0.5"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"ffe62b97_b35f0071","line":356,"updated":"2017-03-28 20:05:12.000000000","message":"I think if it is deprecated then we should not allow it for a new daemon","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":356,"context_line":"# run_pause \u003d 30"},{"line_number":357,"context_line":"#"},{"line_number":358,"context_line":"# node_timeout \u003d 10"},{"line_number":359,"context_line":"# conn_timeout \u003d 0.5"},{"line_number":360,"context_line":"#"},{"line_number":361,"context_line":"# The replicator also performs reclamation"},{"line_number":362,"context_line":"# reclaim_age \u003d 604800"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"ffe62b97_f37c68ec","line":359,"updated":"2017-03-28 20:05:12.000000000","message":"the actual default for sharder appears to be 5, but 0.5 is the default for replicator","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":356,"context_line":"# run_pause \u003d 30"},{"line_number":357,"context_line":"#"},{"line_number":358,"context_line":"# node_timeout \u003d 10"},{"line_number":359,"context_line":"# conn_timeout \u003d 0.5"},{"line_number":360,"context_line":"#"},{"line_number":361,"context_line":"# The replicator also performs reclamation"},{"line_number":362,"context_line":"# reclaim_age \u003d 604800"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"ffe62b97_133d3e8e","line":359,"in_reply_to":"ffe62b97_f37c68ec","updated":"2017-03-29 03:59:31.000000000","message":"oh, I don\u0027t remember why, maybe I increased it during some debugging to something.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}],"swift/common/db.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":365,"context_line":""},{"line_number":366,"context_line":"    @property"},{"line_number":367,"context_line":"    def db_file(self):"},{"line_number":368,"context_line":"        return self._db_file"},{"line_number":369,"context_line":""},{"line_number":370,"context_line":"    def _create_connection(self, db_file\u003dNone):"},{"line_number":371,"context_line":"        if not db_file:"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_eca17bf9","line":368,"updated":"2017-03-21 23:30:26.000000000","message":"I\u0027m not *necessarily* opposed, but... why did we (kinda sorta but not-really) hide this?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":365,"context_line":""},{"line_number":366,"context_line":"    @property"},{"line_number":367,"context_line":"    def db_file(self):"},{"line_number":368,"context_line":"        return self._db_file"},{"line_number":369,"context_line":""},{"line_number":370,"context_line":"    def _create_connection(self, db_file\u003dNone):"},{"line_number":371,"context_line":"        if not db_file:"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_74c2ad35","line":368,"in_reply_to":"1a1ced50_eca17bf9","updated":"2017-03-22 01:13:03.000000000","message":"This is because the container backend now needs to know about 2 databases, this common/db.py only needs to know about one. So is a bit of a noop here, but in the container backend, it can be smarter with it.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":373,"context_line":"        try:"},{"line_number":374,"context_line":"            self.conn \u003d get_db_connection(db_file, self.timeout)"},{"line_number":375,"context_line":"        except (sqlite3.DatabaseError, DatabaseConnectionError):"},{"line_number":376,"context_line":"            self.possibly_quarantine(*sys.exc_info())"},{"line_number":377,"context_line":""},{"line_number":378,"context_line":"    @contextmanager"},{"line_number":379,"context_line":"    def get(self):"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_20d0f71e","line":376,"updated":"2017-03-21 23:30:26.000000000","message":"I feel like these two helpers might benefit from some in-memory checking? Like,\n\n def _db_exists(self):\n     return self.db_file \u003d\u003d \u0027:memory:\u0027 or os.path.exists(self.db_file)\n\n def _create_connection(self, db_file\u003dNone):\n     if not db_file:\n         db_file \u003d self.db_file\n     if self.db_file \u003d\u003d \u0027:memory:\u0027:\n         return\n     ...","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":404,"context_line":"        \"\"\"Use with the \"with\" statement; locks a database.\"\"\""},{"line_number":405,"context_line":"        if not self.conn:"},{"line_number":406,"context_line":"            if self.db_file !\u003d \u0027:memory:\u0027 and self._db_exists():"},{"line_number":407,"context_line":"                self.conn \u003d get_db_connection(self.db_file, self.timeout)"},{"line_number":408,"context_line":"            else:"},{"line_number":409,"context_line":"                raise DatabaseConnectionError(self.db_file, \"DB doesn\u0027t exist\")"},{"line_number":410,"context_line":"        conn \u003d self.conn"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_36011d39","line":407,"updated":"2017-03-21 23:30:26.000000000","message":"off-topic: it feels weird that this doesn\u0027t do the possibly_quarantine() thing...","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":381,"context_line":"    def get(self):"},{"line_number":382,"context_line":"        \"\"\"Use with the \"with\" statement; returns a database connection.\"\"\""},{"line_number":383,"context_line":"        if not self.conn:"},{"line_number":384,"context_line":"            if self.db_file !\u003d \u0027:memory:\u0027 and self._db_exists():"},{"line_number":385,"context_line":"                self._create_connection()"},{"line_number":386,"context_line":"            else:"},{"line_number":387,"context_line":"                raise DatabaseConnectionError(self.db_file, \"DB doesn\u0027t exist\")"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_980381be","line":384,"updated":"2017-03-25 02:28:22.000000000","message":"So *now* we can just say\n\n if self._db_exists():\n     self._create_connection()\n else:\n     raise ...\n\nright?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":468,"context_line":""},{"line_number":469,"context_line":"        :returns: True if the DB is considered to be deleted, False otherwise"},{"line_number":470,"context_line":"        \"\"\""},{"line_number":471,"context_line":"        if self.db_file !\u003d \u0027:memory:\u0027 and not self._db_exists():"},{"line_number":472,"context_line":"            return True"},{"line_number":473,"context_line":"        self._commit_puts_stale_ok()"},{"line_number":474,"context_line":"        with self.get() as conn:"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_f801d56b","line":471,"updated":"2017-03-25 02:28:22.000000000","message":"Similarly, this can just be\n\n if not self._db_exists():\n\nnow.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":919,"context_line":"            \u0027 WHERE status_changed_at \u003c ?\u0027 % self.db_type,"},{"line_number":920,"context_line":"            (timestamp, timestamp))"},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"    def get_brokers(self):"},{"line_number":923,"context_line":"        return [self]"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c7e74cac","line":922,"updated":"2017-03-28 00:55:04.000000000","message":"So this will only be used by container processes, no account processes...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":919,"context_line":"            \u0027 WHERE status_changed_at \u003c ?\u0027 % self.db_type,"},{"line_number":920,"context_line":"            (timestamp, timestamp))"},{"line_number":921,"context_line":""},{"line_number":922,"context_line":"    def get_brokers(self):"},{"line_number":923,"context_line":"        return [self]"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_d35b56f3","line":922,"in_reply_to":"ffe62b97_c7e74cac","updated":"2017-03-29 03:59:31.000000000","message":"Yeah, but also in db_replication, so it needs to exist in both, but only return the single value in the account sense.\n\nSo created here, and overriden in container backend.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}],"swift/common/db_replicator.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":107,"context_line":"                    if os.path.exists(object_file):"},{"line_number":108,"context_line":"                        yield (partition, object_file, node_id)"},{"line_number":109,"context_line":"                    elif os.path.exists(piv_object_file):"},{"line_number":110,"context_line":"                        yield (partition, piv_object_file, node_id)"},{"line_number":111,"context_line":"                    else:"},{"line_number":112,"context_line":"                        try:"},{"line_number":113,"context_line":"                            os.rmdir(hash_dir)"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_56a85920","line":110,"updated":"2017-03-21 23:30:26.000000000","message":"note to self: so while we\u0027re in the sharding state, we may have a pivot db but it won\u0027t be yielded here...","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":233,"context_line":"            self.rcache, self.logger)"},{"line_number":234,"context_line":"        self.logger.info(\u0027 \u0027.join([\u0027%s:%s\u0027 % item for item in"},{"line_number":235,"context_line":"                                   self.stats.items() if item[0] in"},{"line_number":236,"context_line":"                                   (\u0027no_change\u0027, \u0027hashmatch\u0027, \u0027rsync\u0027, \u0027diff\u0027, \u0027ts_repl\u0027,"},{"line_number":237,"context_line":"                                    \u0027empty\u0027, \u0027diff_capped\u0027)]))"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"    def _add_failure_stats(self, failure_devs_info):"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_b6698dfb","line":236,"updated":"2017-03-21 23:30:26.000000000","message":"*grumble, grumble* pep8 *grumble*","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":233,"context_line":"            self.rcache, self.logger)"},{"line_number":234,"context_line":"        self.logger.info(\u0027 \u0027.join([\u0027%s:%s\u0027 % item for item in"},{"line_number":235,"context_line":"                                   self.stats.items() if item[0] in"},{"line_number":236,"context_line":"                                   (\u0027no_change\u0027, \u0027hashmatch\u0027, \u0027rsync\u0027, \u0027diff\u0027, \u0027ts_repl\u0027,"},{"line_number":237,"context_line":"                                    \u0027empty\u0027, \u0027diff_capped\u0027)]))"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"    def _add_failure_stats(self, failure_devs_info):"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_941ba9c9","line":236,"in_reply_to":"1a1ced50_b6698dfb","updated":"2017-03-22 01:13:03.000000000","message":"Done","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":249,"context_line":"        Sync a single file using rsync. Used by _rsync_db to handle syncing."},{"line_number":250,"context_line":""},{"line_number":251,"context_line":"        :param db_file: file or list of files to be synced"},{"line_number":252,"context_line":"        :param remote_file: remote location to sync the DB file to"},{"line_number":253,"context_line":"        :param whole-file: if True, uses rsync\u0027s --whole-file flag"},{"line_number":254,"context_line":"        :param different_region: if True, the destination node is in a"},{"line_number":255,"context_line":"                                 different region"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_9632d10e","line":252,"range":{"start_line":252,"start_character":15,"end_line":252,"end_character":26},"updated":"2017-03-21 23:30:26.000000000","message":"I take it this is a bit mis-named? Seems like we use it as though it were remote_dir (and notably expect there to be a trailing slash).","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":272,"context_line":""},{"line_number":273,"context_line":"        for db_f in db_file:"},{"line_number":274,"context_line":"            args \u003d list(popen_args)"},{"line_number":275,"context_line":"            rfile \u003d \"%s%s\" % (remote_file, os.path.basename(db_f))"},{"line_number":276,"context_line":"            args.extend([db_f, rfile])"},{"line_number":277,"context_line":"            proc \u003d subprocess.Popen(args)"},{"line_number":278,"context_line":"            proc.communicate()"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_f63dc5dd","line":275,"updated":"2017-03-21 23:30:26.000000000","message":"nit: I\u0027d be partial toward ordinary string concatenation at this point.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":355,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":356,"context_line":"            objects \u003d broker.get_items_since(point, self.per_diff)"},{"line_number":357,"context_line":"        if objects:"},{"line_number":358,"context_line":"            self._sync_other_items(broker, local_id, http)"},{"line_number":359,"context_line":"            self.logger.debug("},{"line_number":360,"context_line":"                \u0027Synchronization for %s has fallen more than \u0027"},{"line_number":361,"context_line":"                \u0027%s rows behind; moving on and will try again next pass.\u0027,"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_7918e675","line":358,"updated":"2017-03-21 23:30:26.000000000","message":"Worth hoisting this up \u0026 out of the if, since both branches start with it?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":355,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":356,"context_line":"            objects \u003d broker.get_items_since(point, self.per_diff)"},{"line_number":357,"context_line":"        if objects:"},{"line_number":358,"context_line":"            self._sync_other_items(broker, local_id, http)"},{"line_number":359,"context_line":"            self.logger.debug("},{"line_number":360,"context_line":"                \u0027Synchronization for %s has fallen more than \u0027"},{"line_number":361,"context_line":"                \u0027%s rows behind; moving on and will try again next pass.\u0027,"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_5453f16a","line":358,"in_reply_to":"1a1ced50_7918e675","updated":"2017-03-22 01:13:03.000000000","message":"Good call, at some point I wasn\u0027t always syncing pivots (many POCs ago). Then obviously made it during debugging (which is what we want).\n\ndone.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":391,"context_line":"                                          local_id)"},{"line_number":392,"context_line":"            if not response or response.status \u003e\u003d 300 \\"},{"line_number":393,"context_line":"                    or response.status \u003c 200:"},{"line_number":394,"context_line":"                if response:"},{"line_number":395,"context_line":"                    self.logger.error(_(\u0027ERROR Bad response %(status)s \u0027"},{"line_number":396,"context_line":"                                        \u0027from %(host)s\u0027),"},{"line_number":397,"context_line":"                                      {\u0027status\u0027: response.status,"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_994e826d","line":394,"updated":"2017-03-21 23:30:26.000000000","message":"What\u0027s the else: side of this look like? Should we be logging something for that, too?\n\nDoes that indicate a timeout or socket error or something?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":391,"context_line":"                                          local_id)"},{"line_number":392,"context_line":"            if not response or response.status \u003e\u003d 300 \\"},{"line_number":393,"context_line":"                    or response.status \u003c 200:"},{"line_number":394,"context_line":"                if response:"},{"line_number":395,"context_line":"                    self.logger.error(_(\u0027ERROR Bad response %(status)s \u0027"},{"line_number":396,"context_line":"                                        \u0027from %(host)s\u0027),"},{"line_number":397,"context_line":"                                      {\u0027status\u0027: response.status,"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_74e38da4","line":394,"in_reply_to":"1a1ced50_994e826d","updated":"2017-03-22 01:13:03.000000000","message":"hmm yeah, it\u0027ll mean replication of the other items failed.\n\nAt the moment it\u0027s just mirroring what happens in usync_db, except we don\u0027t seem to pass back the False that this generates. I guess the question is, if other_items fails is it considered a failed replication... I guess so.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":354,"context_line":"            # order with no gaps"},{"line_number":355,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":356,"context_line":"            objects \u003d broker.get_items_since(point, self.per_diff)"},{"line_number":357,"context_line":"            self._sync_other_items(broker, local_id, http)"},{"line_number":358,"context_line":"        if objects:"},{"line_number":359,"context_line":"            self.logger.debug("},{"line_number":360,"context_line":"                \u0027Synchronization for %s has fallen more than \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_f80f1586","line":357,"updated":"2017-03-25 02:28:22.000000000","message":"Whoops! De-dent.\n\nInteresting that tests didn\u0027t seem to notice...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":354,"context_line":"            # order with no gaps"},{"line_number":355,"context_line":"            point \u003d objects[-1][\u0027ROWID\u0027]"},{"line_number":356,"context_line":"            objects \u003d broker.get_items_since(point, self.per_diff)"},{"line_number":357,"context_line":"            self._sync_other_items(broker, local_id, http)"},{"line_number":358,"context_line":"        if objects:"},{"line_number":359,"context_line":"            self.logger.debug("},{"line_number":360,"context_line":"                \u0027Synchronization for %s has fallen more than \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_731a4a17","line":357,"in_reply_to":"1a1ced50_f80f1586","updated":"2017-03-29 03:59:31.000000000","message":"Cause I think the tests haven\u0027t been updated to be other items aware. Many tests are missing. Many of this replication stuff was added during debugging to get it working. Cause I\u0027m a bad man.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"70e4b07cbce40f051d9435210d6f5700bbfa101c","unresolved":false,"context_lines":[{"line_number":900,"context_line":"        filenames \u003d []"},{"line_number":901,"context_line":"        if args:"},{"line_number":902,"context_line":"            filenames \u003d args.pop(0)"},{"line_number":903,"context_line":"            filename \u003d \u0027%s%s\u0027 % (local_id, filenames[0])"},{"line_number":904,"context_line":"        old_filename \u003d os.path.join(self.root, drive, \u0027tmp\u0027, filename)"},{"line_number":905,"context_line":"        if not os.path.exists(db_file) or not os.path.exists(old_filename):"},{"line_number":906,"context_line":"            return HTTPNotFound()"}],"source_content_type":"text/x-python","patch_set":11,"id":"9f436f4f_e3bd0369","line":903,"updated":"2017-08-08 20:51:13.000000000","message":"Hmm... https://github.com/openstack/swift/commit/843184f added a test that supplies an extra argument... though I\u0027m not sure why... and it causes us to 404 because we added the first letter of \u0027arg2\u0027 to the end of our db name.\n\nDo we actually *document* these RPCs anywhere?","commit_id":"519c522263891485dcf0ceab58603d5aaad483c7"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f6594e2ab7d6783d7aee39a095c1eee263b045af","unresolved":false,"context_lines":[{"line_number":231,"context_line":"    def _add_failure_stats(self, failure_devs_info):"},{"line_number":232,"context_line":"        for node, dev in failure_devs_info:"},{"line_number":233,"context_line":"            self.stats[\u0027failure\u0027] +\u003d 1"},{"line_number":234,"context_line":"            failure_devs \u003d self.stats[\u0027failure_nodes\u0027].setdefault(node, {})"},{"line_number":235,"context_line":"            failure_devs.setdefault(dev, 0)"},{"line_number":236,"context_line":"            failure_devs[dev] +\u003d 1"},{"line_number":237,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_f10e9be3","line":234,"updated":"2017-09-06 00:55:01.000000000","message":"What? self.stats[\u0027failure_nodes\u0027] is now an int??\n\n Sep  6 00:03:33 saio container-6031: message repeated 6 times: [ Syncing chunks with 127.0.0.1, starting at -1]\n Sep  6 00:03:33 saio container-6031: STDERR: Traceback (most recent call last):\n Sep  6 00:03:33 saio container-6031: STDERR: File \"/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py\", line 114, in wait\n Sep  6 00:03:33 saio container-6031: STDERR: listener.cb(fileno)\n Sep  6 00:03:33 saio container-6031: STDERR: File \"/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py\", line 218, in main\n Sep  6 00:03:33 saio container-6031: STDERR: result \u003d function(*args, **kwargs)\n Sep  6 00:03:33 saio container-6031: STDERR: File \"/vagrant/swift/swift/common/utils.py\", line 2774, in _run_func\n Sep  6 00:03:33 saio container-6031: STDERR: self._responses.put(func(*args, **kwargs))\n Sep  6 00:03:33 saio container-6031: STDERR: File \"/vagrant/swift/swift/common/db_replicator.py\", line 637, in _replicate_object\n Sep  6 00:03:33 saio container-6031: STDERR: self._add_failure_stats(failure_devs_info)\n Sep  6 00:03:33 saio container-6031: STDERR: File \"/vagrant/swift/swift/common/db_replicator.py\", line 234, in _add_failure_stats\n Sep  6 00:03:33 saio container-6031: STDERR: failure_devs \u003d self.stats[\u0027failure_nodes\u0027].setdefault(node, {})\n Sep  6 00:03:33 saio container-6031: STDERR: AttributeError: \u0027int\u0027 object has no attribute \u0027setdefault\u0027\n\n... followed immediately by\n\n Sep  6 00:03:33 saio container-6031: STDERR: Removing descriptor: 33\n\n... which I have *no* idea about.","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"}],"swift/common/direct_client.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"839ee5a76b2bf900536141d677f9060c3490b4e6","unresolved":false,"context_lines":[{"line_number":103,"context_line":"    if \u0027nodes\u0027 in kargs:"},{"line_number":104,"context_line":"        qs +\u003d \u0027\u0026nodes\u003d%s\u0027 % quote(kargs.get(\u0027nodes\u0027, \u0027\u0027))"},{"line_number":105,"context_line":"    if \u0027noshard\u0027 in kargs:"},{"line_number":106,"context_line":"        qs +\u003d \u0027\u0026noshard\u003d%s\u0027 % quote(kargs.get(\u0027noshard\u0027, \u0027\u0027))"},{"line_number":107,"context_line":"    with Timeout(conn_timeout):"},{"line_number":108,"context_line":"        conn \u003d http_connect(node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027], part,"},{"line_number":109,"context_line":"                            \u0027GET\u0027, path, query_string\u003dqs,"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a3905b3_5ea6c653","line":106,"range":{"start_line":106,"start_character":47,"end_line":106,"end_character":54},"updated":"2017-01-23 05:40:28.000000000","message":"May not need this anymore, this might be cruft that isn\u0027t required anymore.","commit_id":"e7f5932e086ea82d64c68d526ac753628d2122ac"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":81,"context_line":"                                  marker\u003dNone, limit\u003dNone,"},{"line_number":82,"context_line":"                                  prefix\u003dNone, delimiter\u003dNone,"},{"line_number":83,"context_line":"                                  conn_timeout\u003d5, response_timeout\u003d15,"},{"line_number":84,"context_line":"                                  end_marker\u003dNone, reverse\u003dNone, **kargs):"},{"line_number":85,"context_line":"    \"\"\"Base class for get direct account and container."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"    Do not use directly use the get_direct_account or"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_f8eb9576","line":84,"range":{"start_line":84,"start_character":67,"end_line":84,"end_character":72},"updated":"2017-03-25 02:28:22.000000000","message":"nit: Isn\u0027t kwargs more standard? This reminds me how I always get tripped up by the parm keyword arg in swift_test_client (I always want to call it \u0027param\u0027)...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":101,"context_line":"    if reverse:"},{"line_number":102,"context_line":"        qs +\u003d \u0027\u0026reverse\u003d%s\u0027 % quote(reverse)"},{"line_number":103,"context_line":"    if \u0027nodes\u0027 in kargs:"},{"line_number":104,"context_line":"        qs +\u003d \u0027\u0026nodes\u003d%s\u0027 % quote(kargs.get(\u0027nodes\u0027, \u0027\u0027))"},{"line_number":105,"context_line":"    if \u0027noshard\u0027 in kargs:"},{"line_number":106,"context_line":"        qs +\u003d \u0027\u0026noshard\u003d%s\u0027 % quote(kargs.get(\u0027noshard\u0027, \u0027\u0027))"},{"line_number":107,"context_line":"    with Timeout(conn_timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_58fbc947","line":104,"range":{"start_line":104,"start_character":34,"end_line":104,"end_character":56},"updated":"2017-03-25 02:28:22.000000000","message":"nit: Since we know \u0027nodes\u0027 is in kargs, this can just be kargs[\u0027nodes\u0027]","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":101,"context_line":"    if reverse:"},{"line_number":102,"context_line":"        qs +\u003d \u0027\u0026reverse\u003d%s\u0027 % quote(reverse)"},{"line_number":103,"context_line":"    if \u0027nodes\u0027 in kargs:"},{"line_number":104,"context_line":"        qs +\u003d \u0027\u0026nodes\u003d%s\u0027 % quote(kargs.get(\u0027nodes\u0027, \u0027\u0027))"},{"line_number":105,"context_line":"    if \u0027noshard\u0027 in kargs:"},{"line_number":106,"context_line":"        qs +\u003d \u0027\u0026noshard\u003d%s\u0027 % quote(kargs.get(\u0027noshard\u0027, \u0027\u0027))"},{"line_number":107,"context_line":"    with Timeout(conn_timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_33bf22d2","line":104,"range":{"start_line":104,"start_character":34,"end_line":104,"end_character":56},"in_reply_to":"1a1ced50_58fbc947","updated":"2017-03-29 03:59:31.000000000","message":"yes, yes it should","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":102,"context_line":"    if \u0027nodes\u0027 in kargs:"},{"line_number":103,"context_line":"        qs +\u003d \u0027\u0026nodes\u003d%s\u0027 % quote(kargs.get(\u0027nodes\u0027, \u0027\u0027))"},{"line_number":104,"context_line":"    if \u0027noshard\u0027 in kargs:"},{"line_number":105,"context_line":"        qs +\u003d \u0027\u0026noshard\u003d%s\u0027 % quote(kargs.get(\u0027noshard\u0027, \u0027\u0027))"},{"line_number":106,"context_line":"    with Timeout(conn_timeout):"},{"line_number":107,"context_line":"        conn \u003d http_connect(node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027], part,"},{"line_number":108,"context_line":"                            \u0027GET\u0027, path, query_string\u003dqs,"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_846a5ab0","line":105,"updated":"2017-09-07 00:17:46.000000000","message":"I thought we\u0027d added `items` and `include_end_marker`, not `nodes` and `noshard`...\n\nShould we raise some sort of error if the new params are sent to an account server?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"}],"swift/common/exceptions.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":274,"context_line":"    pass"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"class RangeAnalyserException(SwiftException):"},{"line_number":278,"context_line":"    pass"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_78e1659a","line":277,"range":{"start_line":277,"start_character":16,"end_line":277,"end_character":17},"updated":"2017-03-25 02:28:22.000000000","message":"You and your non-American English :P","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":274,"context_line":"    pass"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"class RangeAnalyserException(SwiftException):"},{"line_number":278,"context_line":"    pass"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_53c3c657","line":277,"range":{"start_line":277,"start_character":16,"end_line":277,"end_character":17},"in_reply_to":"1a1ced50_78e1659a","updated":"2017-03-29 03:59:31.000000000","message":"lol, yeah, i can\u0027t help it. There probably is a policy that things should be in US english or something, but until I see it, I\u0027m going to continue writing things my way.. also, I\u0027m blind to it :)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}],"swift/common/manager.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":31,"context_line":"RUN_DIR \u003d \u0027/var/run/swift\u0027"},{"line_number":32,"context_line":"PROC_DIR \u003d \u0027/proc\u0027"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"# auth-server has been removed from ALL_SERVERS, start it explicitly"},{"line_number":35,"context_line":"ALL_SERVERS \u003d [\u0027account-auditor\u0027, \u0027account-server\u0027, \u0027container-auditor\u0027,"},{"line_number":36,"context_line":"               \u0027container-replicator\u0027, \u0027container-reconciler\u0027,"},{"line_number":37,"context_line":"               \u0027container-server\u0027, \u0027container-sharder\u0027, \u0027container-sync\u0027,"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_c285075a","line":34,"updated":"2017-03-25 02:28:22.000000000","message":"off-topic: I was going to ask about this, but decided to just submit https://review.openstack.org/449270 instead (so none of us would have to think about it again, at least until we refactor test_manager.py)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":31,"context_line":"RUN_DIR \u003d \u0027/var/run/swift\u0027"},{"line_number":32,"context_line":"PROC_DIR \u003d \u0027/proc\u0027"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"# auth-server has been removed from ALL_SERVERS, start it explicitly"},{"line_number":35,"context_line":"ALL_SERVERS \u003d [\u0027account-auditor\u0027, \u0027account-server\u0027, \u0027container-auditor\u0027,"},{"line_number":36,"context_line":"               \u0027container-replicator\u0027, \u0027container-reconciler\u0027,"},{"line_number":37,"context_line":"               \u0027container-server\u0027, \u0027container-sharder\u0027, \u0027container-sync\u0027,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_93926e56","line":34,"in_reply_to":"1a1ced50_c285075a","updated":"2017-03-29 03:59:31.000000000","message":"lol, nice","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}],"swift/common/request_helpers.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":752,"context_line":"            if not container_list:"},{"line_number":753,"context_line":"                return HTTPNoContent(request\u003dreq, headers\u003dresp_headers)"},{"line_number":754,"context_line":"            if isinstance(container_list[0], dict):"},{"line_number":755,"context_line":"                index \u003d \u0027name\u0027"},{"line_number":756,"context_line":"            else:"},{"line_number":757,"context_line":"                index \u003d 0"},{"line_number":758,"context_line":"            ret.body \u003d \u0027\\n\u0027.join(rec[index] for rec in container_list) + \u0027\\n\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_27e2c3d6","line":755,"updated":"2017-03-25 02:28:22.000000000","message":"Does this play well with delimiter requests? I can\u0027t really see how it would...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":754,"context_line":"            if isinstance(container_list[0], dict):"},{"line_number":755,"context_line":"                index \u003d \u0027name\u0027"},{"line_number":756,"context_line":"            else:"},{"line_number":757,"context_line":"                index \u003d 0"},{"line_number":758,"context_line":"            ret.body \u003d \u0027\\n\u0027.join(rec[index] for rec in container_list) + \u0027\\n\u0027"},{"line_number":759,"context_line":"        ret.last_modified \u003d math.ceil(float(resp_headers[\u0027X-PUT-Timestamp\u0027]))"},{"line_number":760,"context_line":"        return ret"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_651c5d6b","line":757,"updated":"2017-03-25 02:28:22.000000000","message":"So we\u0027ve either got a list of dicts (like the json response), or a ... list of lists? Where do we use the latter? Ah, and the non-text/plain out_content_type possibilities use update_container_data_record to get to the dict format.\n\nFrom my playing around with https://review.openstack.org/#/c/449394/ I guess the lists-of-lists come from raw broker responses?\n\nDoes this play well with PivotRanges?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":754,"context_line":"            if isinstance(container_list[0], dict):"},{"line_number":755,"context_line":"                index \u003d \u0027name\u0027"},{"line_number":756,"context_line":"            else:"},{"line_number":757,"context_line":"                index \u003d 0"},{"line_number":758,"context_line":"            ret.body \u003d \u0027\\n\u0027.join(rec[index] for rec in container_list) + \u0027\\n\u0027"},{"line_number":759,"context_line":"        ret.last_modified \u003d math.ceil(float(resp_headers[\u0027X-PUT-Timestamp\u0027]))"},{"line_number":760,"context_line":"        return ret"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_d39a3660","line":757,"in_reply_to":"1a1ced50_651c5d6b","updated":"2017-03-29 03:59:31.000000000","message":"I think it does, because pivot_ranges always comes from the root container. And pivot ranges has a \u0027name\u0027 and it happens to sit in index 0","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":756,"context_line":"            else:"},{"line_number":757,"context_line":"                index \u003d 0"},{"line_number":758,"context_line":"            ret.body \u003d \u0027\\n\u0027.join(rec[index] for rec in container_list) + \u0027\\n\u0027"},{"line_number":759,"context_line":"        ret.last_modified \u003d math.ceil(float(resp_headers[\u0027X-PUT-Timestamp\u0027]))"},{"line_number":760,"context_line":"        return ret"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a1ced50_22941b43","line":759,"updated":"2017-03-25 02:28:22.000000000","message":"Between this and https://review.openstack.org/#/c/198633/16/swift/container/server.py I\u0027m starting to wonder if that math.ceil call should be in swob.py...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":755,"context_line":"                index \u003d \u0027name\u0027"},{"line_number":756,"context_line":"            else:"},{"line_number":757,"context_line":"                index \u003d 0"},{"line_number":758,"context_line":"            ret.body \u003d \u0027\\n\u0027.join(rec[index] for rec in container_list) + \u0027\\n\u0027"},{"line_number":759,"context_line":"        ret.last_modified \u003d math.ceil(float(resp_headers[\u0027X-PUT-Timestamp\u0027]))"},{"line_number":760,"context_line":"        return ret"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_99f63cd7","line":758,"updated":"2017-09-07 00:17:46.000000000","message":"This can trigger\n\n TypeError: \u0027PivotRange\u0027 object does not support indexing","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f6594e2ab7d6783d7aee39a095c1eee263b045af","unresolved":false,"context_lines":[{"line_number":756,"context_line":"            else:"},{"line_number":757,"context_line":"                index \u003d 0"},{"line_number":758,"context_line":"            ret.body \u003d \u0027\\n\u0027.join(rec[index] for rec in container_list) + \u0027\\n\u0027"},{"line_number":759,"context_line":"        ret.last_modified \u003d math.ceil(float(resp_headers[\u0027X-PUT-Timestamp\u0027]))"},{"line_number":760,"context_line":"        return ret"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_3f1d0f73","line":759,"range":{"start_line":759,"start_character":58,"end_line":759,"end_character":73},"updated":"2017-09-06 00:55:01.000000000","message":"This isn\u0027t always present...","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":756,"context_line":"            else:"},{"line_number":757,"context_line":"                index \u003d 0"},{"line_number":758,"context_line":"            ret.body \u003d \u0027\\n\u0027.join(rec[index] for rec in container_list) + \u0027\\n\u0027"},{"line_number":759,"context_line":"        ret.last_modified \u003d math.ceil(float(resp_headers[\u0027X-PUT-Timestamp\u0027]))"},{"line_number":760,"context_line":"        return ret"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_28325411","line":759,"range":{"start_line":759,"start_character":58,"end_line":759,"end_character":73},"in_reply_to":"7f287b81_3f1d0f73","updated":"2017-09-07 00:17:46.000000000","message":"Ah. In the proxy controller, we\u0027ve got X-Backend-Put-Timestamp but no X-PUT-Timestamp.","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"}],"swift/common/utils.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f0fcfe79c41d2a821064f754f5ef3cb135669d4","unresolved":false,"context_lines":[{"line_number":4134,"context_line":"class PivotRange(object):"},{"line_number":4135,"context_line":"    def __init__(self, name\u003dNone, timestamp\u003dNone, lower\u003dNone, upper\u003dNone,"},{"line_number":4136,"context_line":"                 obj_count\u003d0, bytes_used\u003d0, meta_timestamp\u003dNone):"},{"line_number":4137,"context_line":"        self._name \u003d name"},{"line_number":4138,"context_line":"        self._lower \u003d lower"},{"line_number":4139,"context_line":"        self._upper \u003d upper"},{"line_number":4140,"context_line":"        self._timestamp \u003d timestamp"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a92c9e4a","line":4137,"updated":"2017-03-30 16:24:02.000000000","message":"if there is going to be a setter, shouldn\u0027t it be used here so that is ever any behaviour is added to the setter then it gets picked up here? That\u0027s not an issue if the setter will never do any different than self._name \u003d name, but (better example) what if the timestamp setter became\n\n    @timestamp.setter\n    def timestamp(self, ts):\n        self._timestamp \u003d Timestamp(ts)\n\nthen it would be good idea to use the setter here.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4136,"context_line":"                 obj_count\u003d0, bytes_used\u003d0, meta_timestamp\u003dNone):"},{"line_number":4137,"context_line":"        self._name \u003d name"},{"line_number":4138,"context_line":"        self._lower \u003d lower"},{"line_number":4139,"context_line":"        self._upper \u003d upper"},{"line_number":4140,"context_line":"        self._timestamp \u003d timestamp"},{"line_number":4141,"context_line":"        self._meta_timestamp \u003d meta_timestamp if meta_timestamp else timestamp"},{"line_number":4142,"context_line":"        self._count \u003d obj_count"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_86d7a34b","line":4139,"updated":"2017-03-29 23:15:54.000000000","message":"Should we have something like\n\n if upper and upper \u003c lower:\n     raise ValueError\n\n? I feel like requiring that is reasonable, and will help us keep our sanity when we\u0027re thinking about this...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4134,"context_line":"class PivotRange(object):"},{"line_number":4135,"context_line":"    def __init__(self, name\u003dNone, timestamp\u003dNone, lower\u003dNone, upper\u003dNone,"},{"line_number":4136,"context_line":"                 obj_count\u003d0, bytes_used\u003d0, meta_timestamp\u003dNone):"},{"line_number":4137,"context_line":"        self._name \u003d name"},{"line_number":4138,"context_line":"        self._lower \u003d lower"},{"line_number":4139,"context_line":"        self._upper \u003d upper"},{"line_number":4140,"context_line":"        self._timestamp \u003d timestamp"},{"line_number":4141,"context_line":"        self._meta_timestamp \u003d meta_timestamp if meta_timestamp else timestamp"},{"line_number":4142,"context_line":"        self._count \u003d obj_count"},{"line_number":4143,"context_line":"        self._bytes \u003d bytes_used"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_87486fcc","line":4140,"range":{"start_line":4137,"start_character":8,"end_line":4140,"end_character":35},"updated":"2017-03-25 02:28:22.000000000","message":"...can we just use self.name, etc. until we actually need to go with getters/setters?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4134,"context_line":"class PivotRange(object):"},{"line_number":4135,"context_line":"    def __init__(self, name\u003dNone, timestamp\u003dNone, lower\u003dNone, upper\u003dNone,"},{"line_number":4136,"context_line":"                 obj_count\u003d0, bytes_used\u003d0, meta_timestamp\u003dNone):"},{"line_number":4137,"context_line":"        self._name \u003d name"},{"line_number":4138,"context_line":"        self._lower \u003d lower"},{"line_number":4139,"context_line":"        self._upper \u003d upper"},{"line_number":4140,"context_line":"        self._timestamp \u003d timestamp"},{"line_number":4141,"context_line":"        self._meta_timestamp \u003d meta_timestamp if meta_timestamp else timestamp"},{"line_number":4142,"context_line":"        self._count \u003d obj_count"},{"line_number":4143,"context_line":"        self._bytes \u003d bytes_used"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_33e042cd","line":4140,"range":{"start_line":4137,"start_character":8,"end_line":4140,"end_character":35},"in_reply_to":"ffe62b97_87486fcc","updated":"2017-03-29 03:59:31.000000000","message":"Sure, I don\u0027t see why not. It was for completeness sake ;)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f0fcfe79c41d2a821064f754f5ef3cb135669d4","unresolved":false,"context_lines":[{"line_number":4138,"context_line":"        self._lower \u003d lower"},{"line_number":4139,"context_line":"        self._upper \u003d upper"},{"line_number":4140,"context_line":"        self._timestamp \u003d timestamp"},{"line_number":4141,"context_line":"        self._meta_timestamp \u003d meta_timestamp if meta_timestamp else timestamp"},{"line_number":4142,"context_line":"        self._count \u003d obj_count"},{"line_number":4143,"context_line":"        self._bytes \u003d bytes_used"},{"line_number":4144,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c9c48262","line":4141,"updated":"2017-03-30 16:24:02.000000000","message":"I suggest that we either require the timestamps to be of type Timestamp or coerce them when set so that we know we have Timestamps. We have tried to be fairly rigorous about things called \u0027timestamp\u0027 being Timestamps.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4180,"context_line":""},{"line_number":4181,"context_line":"    @obj_count.setter"},{"line_number":4182,"context_line":"    def obj_count(self, count):"},{"line_number":4183,"context_line":"        self.meta_timestamp \u003d Timestamp(time.time()).internal"},{"line_number":4184,"context_line":"        self._count \u003d count"},{"line_number":4185,"context_line":""},{"line_number":4186,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4714a7a0","line":4183,"updated":"2017-03-25 02:28:22.000000000","message":"This seems like a curious side-effect; why have it for obj_count and bytes_used but nothing else?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4180,"context_line":""},{"line_number":4181,"context_line":"    @obj_count.setter"},{"line_number":4182,"context_line":"    def obj_count(self, count):"},{"line_number":4183,"context_line":"        self.meta_timestamp \u003d Timestamp(time.time()).internal"},{"line_number":4184,"context_line":"        self._count \u003d count"},{"line_number":4185,"context_line":""},{"line_number":4186,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_73912af1","line":4183,"in_reply_to":"ffe62b97_4714a7a0","updated":"2017-03-29 03:59:31.000000000","message":"It meant, that the metadata timestamps would automatically be updated when pulling updating the latest counts in the sharder. i.e. See what the shard stats are, place them into pivotrange class and pass it up to the root container.\n\nLet\u0027s take a step back...\nSo there are 2 timestamps we need to track, created_timestamp and meta_timestamp. A major event (a shard or shrink) needs to take precedence, this is what the created_timestamp tracks. The meta_timestamp is only used for stat updates.\n\nProblem being, if we have a bunch of requests in pending items at a container that include a bunch of stats updates to the same pivot range we need the newest to win (meta_timestamp). However, When a shrink happens, then we need to update a pivotRange, but we don\u0027t want a newer request with only a newer stat from some unshrunk primary to win (and thereby undoing what we\u0027ve done) so we need 2. So TL;DR 2 timestamps, one tracks major changes (new pivots and shrinks) and 1 to track the frequent stats updates.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4180,"context_line":""},{"line_number":4181,"context_line":"    @obj_count.setter"},{"line_number":4182,"context_line":"    def obj_count(self, count):"},{"line_number":4183,"context_line":"        self.meta_timestamp \u003d Timestamp(time.time()).internal"},{"line_number":4184,"context_line":"        self._count \u003d count"},{"line_number":4185,"context_line":""},{"line_number":4186,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_06927357","line":4183,"in_reply_to":"ffe62b97_73912af1","updated":"2017-03-29 23:15:54.000000000","message":"Thanks for the two-timestamp explanation, makes sense.\n\nI\u0027m still wondering why we\u0027ve got the meta_timestamp updates in our class though -- I was kinda expecting that to all happen closer to the DB updates? That\u0027s the canonical source of truth for the current state, right? Or are there long-lived PivotRange objects in memory in the container server/replicator/sharder such that this makes more sense than updating the DB and then creating a new PivotRange?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4200,"context_line":"    def meta_timestamp(self, ts):"},{"line_number":4201,"context_line":"        self._meta_timestamp \u003d ts"},{"line_number":4202,"context_line":""},{"line_number":4203,"context_line":"    def __contains__(self, item):"},{"line_number":4204,"context_line":"        if not self._lower and not self._upper:"},{"line_number":4205,"context_line":"            # No limits so must match"},{"line_number":4206,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_8667a34d","line":4203,"updated":"2017-03-29 23:15:54.000000000","message":"Neat!\n\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027) in PivotRange(lower\u003dNone, upper\u003dNone)\n True\n \u003e\u003e\u003e PivotRange(lower\u003dNone, upper\u003dNone) in PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027)\n False\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027) in PivotRange(lower\u003d\u0027m\u0027, upper\u003dNone)\n False\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027) in PivotRange(lower\u003d\u0027g\u0027, upper\u003d\u0027m\u0027)\n False\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027g\u0027, upper\u003d\u0027m\u0027) in PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027)\n True\n \u003e\u003e\u003e PivotRange(lower\u003dNone, upper\u003dNone) in PivotRange(lower\u003dNone, upper\u003dNone)\n True\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027) in PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027)\n True\n\nPretty sure this is still cool:\n\n \u003e\u003e\u003e PivotRange(lower\u003dNone, upper\u003d\u0027m\u0027) in PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027)\n False\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027) in PivotRange(lower\u003d\u0027m\u0027, upper\u003dNone)\n False\n\nLess cool:\n\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027m\u0027, upper\u003dNone) in PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027)\n True\n \u003e\u003e\u003e PivotRange(lower\u003d\u0027a\u0027, upper\u003d\u0027z\u0027) in PivotRange(lower\u003dNone, upper\u003d\u0027m\u0027)\n True","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4201,"context_line":"        self._meta_timestamp \u003d ts"},{"line_number":4202,"context_line":""},{"line_number":4203,"context_line":"    def __contains__(self, item):"},{"line_number":4204,"context_line":"        if not self._lower and not self._upper:"},{"line_number":4205,"context_line":"            # No limits so must match"},{"line_number":4206,"context_line":"            return True"},{"line_number":4207,"context_line":"        elif not self._lower:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_474ca79d","line":4204,"updated":"2017-03-25 02:28:22.000000000","message":"This is almost (but not quite) equivalent to\n\n if self.entire_namespace():\n\nCan upper or lower be the empty string? Would that mean roughly the same sort of thing as it does for marker and end_marker?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4201,"context_line":"        self._meta_timestamp \u003d ts"},{"line_number":4202,"context_line":""},{"line_number":4203,"context_line":"    def __contains__(self, item):"},{"line_number":4204,"context_line":"        if not self._lower and not self._upper:"},{"line_number":4205,"context_line":"            # No limits so must match"},{"line_number":4206,"context_line":"            return True"},{"line_number":4207,"context_line":"        elif not self._lower:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_53a50646","line":4204,"in_reply_to":"ffe62b97_474ca79d","updated":"2017-03-29 03:59:31.000000000","message":"hmm yeah maybe it is.. you can tell what came first :P\n\nI\u0027ve been using None to track the start and end. We can\u0027t have an object with no name (\u0027\u0027).","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4201,"context_line":"        self._meta_timestamp \u003d ts"},{"line_number":4202,"context_line":""},{"line_number":4203,"context_line":"    def __contains__(self, item):"},{"line_number":4204,"context_line":"        if not self._lower and not self._upper:"},{"line_number":4205,"context_line":"            # No limits so must match"},{"line_number":4206,"context_line":"            return True"},{"line_number":4207,"context_line":"        elif not self._lower:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_837e75e1","line":4204,"in_reply_to":"ffe62b97_53a50646","updated":"2017-03-29 23:15:54.000000000","message":"\u003e We can\u0027t have an object with no name (\u0027\u0027).\n\nThat\u0027s part of what makes it appealing as a special marker -- that and knowing that whenever you\u0027re dealing with upper/lower values, they\u0027re always strings.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4218,"context_line":"            if other.lower is None:"},{"line_number":4219,"context_line":"                return False"},{"line_number":4220,"context_line":"            lower \u003d self._lower or \u0027\u0027"},{"line_number":4221,"context_line":"            upper \u003d self._upper or other.lower + \u0027a\u0027"},{"line_number":4222,"context_line":"            return upper \u003c other.lower or lower \u003c other.lower"},{"line_number":4223,"context_line":"        elif other is None:"},{"line_number":4224,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_87762f06","line":4221,"range":{"start_line":4221,"start_character":50,"end_line":4221,"end_character":51},"updated":"2017-03-25 02:28:22.000000000","message":"Probably should be \\x00?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4218,"context_line":"            if other.lower is None:"},{"line_number":4219,"context_line":"                return False"},{"line_number":4220,"context_line":"            lower \u003d self._lower or \u0027\u0027"},{"line_number":4221,"context_line":"            upper \u003d self._upper or other.lower + \u0027a\u0027"},{"line_number":4222,"context_line":"            return upper \u003c other.lower or lower \u003c other.lower"},{"line_number":4223,"context_line":"        elif other is None:"},{"line_number":4224,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_612369a8","line":4221,"range":{"start_line":4221,"start_character":50,"end_line":4221,"end_character":51},"in_reply_to":"ffe62b97_1352de3b","updated":"2017-03-29 23:15:54.000000000","message":"Wait... shouldn\u0027t self._upper always be truthy? Is that `or ...` necessary?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4218,"context_line":"            if other.lower is None:"},{"line_number":4219,"context_line":"                return False"},{"line_number":4220,"context_line":"            lower \u003d self._lower or \u0027\u0027"},{"line_number":4221,"context_line":"            upper \u003d self._upper or other.lower + \u0027a\u0027"},{"line_number":4222,"context_line":"            return upper \u003c other.lower or lower \u003c other.lower"},{"line_number":4223,"context_line":"        elif other is None:"},{"line_number":4224,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_1352de3b","line":4221,"range":{"start_line":4221,"start_character":50,"end_line":4221,"end_character":51},"in_reply_to":"ffe62b97_87762f06","updated":"2017-03-29 03:59:31.000000000","message":"yeah good call.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4231,"context_line":"        if isinstance(other, PivotRange):"},{"line_number":4232,"context_line":"            if other.upper is None:"},{"line_number":4233,"context_line":"                return False"},{"line_number":4234,"context_line":"            lower \u003d self._lower or \u0027\u0027"},{"line_number":4235,"context_line":"            upper \u003d self._upper or other.upper + \u0027a\u0027"},{"line_number":4236,"context_line":"            return lower \u003e\u003d other.upper or upper \u003e other.upper"},{"line_number":4237,"context_line":"        elif other is None:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_617fe965","line":4234,"updated":"2017-03-29 23:15:54.000000000","message":"We\u0027ve already checked that self._lower is not None, right?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4235,"context_line":"            upper \u003d self._upper or other.upper + \u0027a\u0027"},{"line_number":4236,"context_line":"            return lower \u003e\u003d other.upper or upper \u003e other.upper"},{"line_number":4237,"context_line":"        elif other is None:"},{"line_number":4238,"context_line":"            return True"},{"line_number":4239,"context_line":"        else:"},{"line_number":4240,"context_line":"            return self._lower \u003e\u003d other"},{"line_number":4241,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9d2dadff","line":4238,"updated":"2017-03-29 23:15:54.000000000","message":"This seems odd.\n\n None \u003e x\n\nis False for most x:\n\n \u003e\u003e\u003e any(None \u003e x for x in (0, \u0027\u0027, 0.0, float(\u0027-inf\u0027), [], (), {}, None))\n False\n\nIn fact, I can\u0027t think of a counter-example... closest I can come up with is\n\n \u003e\u003e\u003e None \u003e\u003d None\n True","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4239,"context_line":"        else:"},{"line_number":4240,"context_line":"            return self._lower \u003e\u003d other"},{"line_number":4241,"context_line":""},{"line_number":4242,"context_line":"    def __eq__(self, other):"},{"line_number":4243,"context_line":"        if not isinstance(other, PivotRange):"},{"line_number":4244,"context_line":"            return False"},{"line_number":4245,"context_line":"        return self._lower \u003d\u003d other.lower and self._upper \u003d\u003d other.upper"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_e7a3bb4e","line":4242,"updated":"2017-03-25 02:28:22.000000000","message":"Maybe also worth defining __ne__ -- I remember this coming up with https://review.openstack.org/#/c/337428/\n\nDo we want __le__ and __ge__? Is it worth looking at https://docs.python.org/2/library/functools.html#functools.total_ordering ?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":4245,"context_line":"        return self._lower \u003d\u003d other.lower and self._upper \u003d\u003d other.upper"},{"line_number":4246,"context_line":""},{"line_number":4247,"context_line":"    def __repr__(self):"},{"line_number":4248,"context_line":"        if isinstance(self.timestamp, Timestamp):"},{"line_number":4249,"context_line":"            ts \u003d self.timestamp.internal"},{"line_number":4250,"context_line":"        else:"},{"line_number":4251,"context_line":"            ts \u003d self.timestamp"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_6de389a5","line":4248,"updated":"2017-03-29 01:29:08.000000000","message":"timestamp, not meta_timestamp? I\u0027m still a little confused about when you\u0027d use one vs. the other...\n\nShould we force self.timestamp to always be a Timestamp, so we can avoid the type-checking?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4245,"context_line":"        return self._lower \u003d\u003d other.lower and self._upper \u003d\u003d other.upper"},{"line_number":4246,"context_line":""},{"line_number":4247,"context_line":"    def __repr__(self):"},{"line_number":4248,"context_line":"        if isinstance(self.timestamp, Timestamp):"},{"line_number":4249,"context_line":"            ts \u003d self.timestamp.internal"},{"line_number":4250,"context_line":"        else:"},{"line_number":4251,"context_line":"            ts \u003d self.timestamp"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3334c2fb","line":4248,"in_reply_to":"ffe62b97_6de389a5","updated":"2017-03-29 03:59:31.000000000","message":"yeah, meta_timestamp in a newer addition (see my comment above). The created_timestamp (or timestamp) is the major event one, whereas the meta_ is only for stats updates. We could add both.\n\nAlso good call (forcing timestamp to be a Timestamp)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":4249,"context_line":"            ts \u003d self.timestamp.internal"},{"line_number":4250,"context_line":"        else:"},{"line_number":4251,"context_line":"            ts \u003d self.timestamp"},{"line_number":4252,"context_line":"        return \u0027(%s to %s as of %s)\u0027 % (self.lower, self.upper, ts)"},{"line_number":4253,"context_line":""},{"line_number":4254,"context_line":"    def entire_namespace(self):"},{"line_number":4255,"context_line":"        return self._lower is None and self._upper is None"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_ed81d9c4","line":4252,"updated":"2017-03-29 01:29:08.000000000","message":"Since it\u0027s a repr, should this include some mention of the class name? Like \u0027PivotRange(%s to %s as of %s)\u0027, say?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4249,"context_line":"            ts \u003d self.timestamp.internal"},{"line_number":4250,"context_line":"        else:"},{"line_number":4251,"context_line":"            ts \u003d self.timestamp"},{"line_number":4252,"context_line":"        return \u0027(%s to %s as of %s)\u0027 % (self.lower, self.upper, ts)"},{"line_number":4253,"context_line":""},{"line_number":4254,"context_line":"    def entire_namespace(self):"},{"line_number":4255,"context_line":"        return self._lower is None and self._upper is None"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_331de273","line":4252,"in_reply_to":"ffe62b97_ed81d9c4","updated":"2017-03-29 03:59:31.000000000","message":"+1","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4275,"context_line":"               other.upper \u003e self._lower \u003e other.lower or \\"},{"line_number":4276,"context_line":"               self._upper \u003e other.upper \u003e self._lower or \\"},{"line_number":4277,"context_line":"               self._upper \u003e other.lower \u003e self._lower or \\"},{"line_number":4278,"context_line":"               (self._upper \u003e other.upper and self._lower \u003c other.lower) or \\"},{"line_number":4279,"context_line":"               (other.lower \u003c self._lower and other.upper \u003e self._upper)"},{"line_number":4280,"context_line":""},{"line_number":4281,"context_line":"    def newer(self, other):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c6b12b46","line":4278,"range":{"start_line":4278,"start_character":16,"end_line":4278,"end_character":71},"updated":"2017-03-29 23:15:54.000000000","message":"Isn\u0027t this equivalent to the two above? Or is there some way in which we can arrive at\n\n other.lower \u003e other.upper\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4276,"context_line":"               self._upper \u003e other.upper \u003e self._lower or \\"},{"line_number":4277,"context_line":"               self._upper \u003e other.lower \u003e self._lower or \\"},{"line_number":4278,"context_line":"               (self._upper \u003e other.upper and self._lower \u003c other.lower) or \\"},{"line_number":4279,"context_line":"               (other.lower \u003c self._lower and other.upper \u003e self._upper)"},{"line_number":4280,"context_line":""},{"line_number":4281,"context_line":"    def newer(self, other):"},{"line_number":4282,"context_line":"        if self._timestamp:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_e73f3bd4","line":4279,"updated":"2017-03-25 02:28:22.000000000","message":"...that\u0027s a lot of comparisons. I\u0027m\u0027a need to look at this some more...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4276,"context_line":"               self._upper \u003e other.upper \u003e self._lower or \\"},{"line_number":4277,"context_line":"               self._upper \u003e other.lower \u003e self._lower or \\"},{"line_number":4278,"context_line":"               (self._upper \u003e other.upper and self._lower \u003c other.lower) or \\"},{"line_number":4279,"context_line":"               (other.lower \u003c self._lower and other.upper \u003e self._upper)"},{"line_number":4280,"context_line":""},{"line_number":4281,"context_line":"    def newer(self, other):"},{"line_number":4282,"context_line":"        if self._timestamp:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_1e2e6591","line":4279,"in_reply_to":"ffe62b97_e73f3bd4","updated":"2017-03-29 03:59:31.000000000","message":"Yes, yes it is. Problem has been when debugging finding or conflicts.. there surely is a better way.. but this is POC cruft from many debugging edge case problems. Dirty but just getting it working.\n\nI wonder if we can get away with making it behave like a set or something (just a random thought that popped into my head)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4278,"context_line":"               (self._upper \u003e other.upper and self._lower \u003c other.lower) or \\"},{"line_number":4279,"context_line":"               (other.lower \u003c self._lower and other.upper \u003e self._upper)"},{"line_number":4280,"context_line":""},{"line_number":4281,"context_line":"    def newer(self, other):"},{"line_number":4282,"context_line":"        if self._timestamp:"},{"line_number":4283,"context_line":"            return self._timestamp \u003e other.timestamp"},{"line_number":4284,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_87074f1d","line":4281,"updated":"2017-03-25 02:28:22.000000000","message":"So there\u0027s another greater-than / less-than axis to this class.\n\nAnd if self \u003d\u003d other, we call other \"newer\".","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4278,"context_line":"               (self._upper \u003e other.upper and self._lower \u003c other.lower) or \\"},{"line_number":4279,"context_line":"               (other.lower \u003c self._lower and other.upper \u003e self._upper)"},{"line_number":4280,"context_line":""},{"line_number":4281,"context_line":"    def newer(self, other):"},{"line_number":4282,"context_line":"        if self._timestamp:"},{"line_number":4283,"context_line":"            return self._timestamp \u003e other.timestamp"},{"line_number":4284,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3e50090b","line":4281,"in_reply_to":"ffe62b97_87074f1d","updated":"2017-03-29 03:59:31.000000000","message":"I\u0027d have to go see where this was used.. or if it still is.\n\nthe PivotRange class has grown alot of cruft over the last few POCs. This is why fresh eyes are important.\n\nMulti-axis FTW :P","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f0fcfe79c41d2a821064f754f5ef3cb135669d4","unresolved":false,"context_lines":[{"line_number":4280,"context_line":""},{"line_number":4281,"context_line":"    def newer(self, other):"},{"line_number":4282,"context_line":"        if self._timestamp:"},{"line_number":4283,"context_line":"            return self._timestamp \u003e other.timestamp"},{"line_number":4284,"context_line":"        else:"},{"line_number":4285,"context_line":"            return False"},{"line_number":4286,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_29c40e92","line":4283,"updated":"2017-03-30 16:24:02.000000000","message":"in __repr__ self.timestamp may or may not be a Timestamp, so does this work for all combinations of self and other timestamps being Timestamps or other type.\n\nI\u0027d vote for coercing self.timestamp to Timestamp whenever it is set so we know what type we have.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4298,"context_line":"    \"\"\""},{"line_number":4299,"context_line":""},{"line_number":4300,"context_line":"    :param item: The item that needs to be placed"},{"line_number":4301,"context_line":"    :param ranges: Byte order sorted list of ranges."},{"line_number":4302,"context_line":"    :return:"},{"line_number":4303,"context_line":"    \"\"\""},{"line_number":4304,"context_line":"    lower_bound \u003d 0"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_1d763d71","line":4301,"updated":"2017-03-29 23:15:54.000000000","message":"Should we validate this assumption? Something like\n\n if not all (x \u003c y for x, y in zip(ranges[:-1], ranges[1:])):\n     raise ValueError\n\n? Is that still good with overlapping ranges?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":4315,"context_line":"            if last_index \u003d\u003d index:"},{"line_number":4316,"context_line":"                return None"},{"line_number":4317,"context_line":""},{"line_number":4318,"context_line":"        return ranges[index]"},{"line_number":4319,"context_line":"    except Exception:"},{"line_number":4320,"context_line":"        return None"},{"line_number":4321,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4aed4bb5","line":4318,"updated":"2017-03-29 01:29:08.000000000","message":"So how does this work if there are overlapping ranges?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4315,"context_line":"            if last_index \u003d\u003d index:"},{"line_number":4316,"context_line":"                return None"},{"line_number":4317,"context_line":""},{"line_number":4318,"context_line":"        return ranges[index]"},{"line_number":4319,"context_line":"    except Exception:"},{"line_number":4320,"context_line":"        return None"},{"line_number":4321,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3e9ec9b8","line":4318,"in_reply_to":"ffe62b97_4aed4bb5","updated":"2017-03-29 03:59:31.000000000","message":"This is used by the proxy when finding where something should go. Not by the sharder which will try and fix things. So inline requests, if we get a place for it, lets use it, and if it\u0027s overlapping and doesn\u0027t end up being the correct one, then let misplaced items deal with it.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4317,"context_line":""},{"line_number":4318,"context_line":"        return ranges[index]"},{"line_number":4319,"context_line":"    except Exception:"},{"line_number":4320,"context_line":"        return None"},{"line_number":4321,"context_line":""},{"line_number":4322,"context_line":""},{"line_number":4323,"context_line":"def pivot_to_pivot_container(account, container, pivot\u003dNone):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_e20de937","line":4320,"updated":"2017-03-25 02:28:22.000000000","message":"https://docs.python.org/2/library/bisect.html might be useful?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4317,"context_line":""},{"line_number":4318,"context_line":"        return ranges[index]"},{"line_number":4319,"context_line":"    except Exception:"},{"line_number":4320,"context_line":"        return None"},{"line_number":4321,"context_line":""},{"line_number":4322,"context_line":""},{"line_number":4323,"context_line":"def pivot_to_pivot_container(account, container, pivot\u003dNone):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_7e49f15d","line":4320,"in_reply_to":"ffe62b97_e20de937","updated":"2017-03-29 03:59:31.000000000","message":"oh nice!","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4327,"context_line":""},{"line_number":4328,"context_line":"        Given something like ``acc, cont, orange`` it will return:"},{"line_number":4329,"context_line":""},{"line_number":4330,"context_line":"            .sharded_acc cont_orange"},{"line_number":4331,"context_line":""},{"line_number":4332,"context_line":"        :param account: The root container\u0027s account"},{"line_number":4333,"context_line":"        :param container: The root container"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_02220da5","line":4330,"updated":"2017-03-25 02:28:22.000000000","message":"Is this the actual naming scheme? IIRC it\u0027s not, but I haven\u0027t gotten back to the docs since ATL...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4327,"context_line":""},{"line_number":4328,"context_line":"        Given something like ``acc, cont, orange`` it will return:"},{"line_number":4329,"context_line":""},{"line_number":4330,"context_line":"            .sharded_acc cont_orange"},{"line_number":4331,"context_line":""},{"line_number":4332,"context_line":"        :param account: The root container\u0027s account"},{"line_number":4333,"context_line":"        :param container: The root container"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_5eb2ad4a","line":4330,"in_reply_to":"ffe62b97_02220da5","updated":"2017-03-29 03:59:31.000000000","message":"nope, it\u0027s changed, and this method now does a lot less, so we should see if it\u0027s actually used or needs to be used anymore.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":4361,"context_line":"    if not isinstance(value, string_types):"},{"line_number":4362,"context_line":"        return False"},{"line_number":4363,"context_line":"    try:"},{"line_number":4364,"context_line":"        int(value)"},{"line_number":4365,"context_line":"    except ValueError:"},{"line_number":4366,"context_line":"        return False"},{"line_number":4367,"context_line":"    return True"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_22a071f4","line":4364,"updated":"2017-03-25 02:28:22.000000000","message":"Huh.\n\n \u003e\u003e\u003e int(\u0027-   110\u0027)\n -110\n \u003e\u003e\u003e int(\u0027+   110\u0027)\n 110\n \u003e\u003e\u003e int(\u0027 110  \u0027)\n 110\n\nTIL. But I suppose the important thing is that values like \u00270x10\u0027 and \u00271e10\u0027 raise ValueErrors.\n\n*If we really wanted* we could tighten it up a bit with something like\n\n if not isinstance(value, string_types):\n     return False\n if value[0] in \u0027+-\u0027:\n     value \u003d value[1:]\n return value.isnumeric()\n\nbut I don\u0027t know that it\u0027s really necessary.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":4361,"context_line":"    if not isinstance(value, string_types):"},{"line_number":4362,"context_line":"        return False"},{"line_number":4363,"context_line":"    try:"},{"line_number":4364,"context_line":"        int(value)"},{"line_number":4365,"context_line":"    except ValueError:"},{"line_number":4366,"context_line":"        return False"},{"line_number":4367,"context_line":"    return True"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_7e90d195","line":4364,"in_reply_to":"ffe62b97_22a071f4","updated":"2017-03-29 03:59:31.000000000","message":"yeah, I don\u0027t think this is used, and I think it\u0027s changed and never removed.. I have no idea what this is doing.\n\nIt obviously used to check the header, then lost something and now does nothing useful.... I\u0027ve got nothing!","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":4361,"context_line":"    if not isinstance(value, string_types):"},{"line_number":4362,"context_line":"        return False"},{"line_number":4363,"context_line":"    try:"},{"line_number":4364,"context_line":"        int(value)"},{"line_number":4365,"context_line":"    except ValueError:"},{"line_number":4366,"context_line":"        return False"},{"line_number":4367,"context_line":"    return True"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c38b5dbd","line":4364,"in_reply_to":"ffe62b97_7e90d195","updated":"2017-03-29 23:15:54.000000000","message":"Oh, sure enough... I hadn\u0027t even noticed! The function definition is literally the only place we have the string \"verify_pivot_usage_header\".","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":366,"context_line":"        val \u003d float(value)"},{"line_number":367,"context_line":"        if minimum and val \u003c minimum:"},{"line_number":368,"context_line":"            raise ValueError()"},{"line_number":369,"context_line":"        if maximum and val \u003e maximum:"},{"line_number":370,"context_line":"            raise ValueError()"},{"line_number":371,"context_line":"        return val"},{"line_number":372,"context_line":"    except (TypeError, ValueError):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_76a4dbb9","line":369,"updated":"2017-04-28 01:38:47.000000000","message":"0.0 is falsey, so we want `is not None` checks I think.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"}],"swift/container/backend.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"839ee5a76b2bf900536141d677f9060c3490b4e6","unresolved":false,"context_lines":[{"line_number":43,"context_line":"DB_STATE_UNSHARDED \u003d 1"},{"line_number":44,"context_line":"DB_STATE_SHARDING \u003d 2"},{"line_number":45,"context_line":"DB_STATE_SHARDED \u003d 3"},{"line_number":46,"context_line":"DB_STATE \u003d ["},{"line_number":47,"context_line":"    \u0027notfound\u0027,"},{"line_number":48,"context_line":"    \u0027unsharded\u0027,"},{"line_number":49,"context_line":"    \u0027sharding\u0027,"},{"line_number":50,"context_line":"    \u0027sharded\u0027]"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"POLICY_STAT_TABLE_CREATE \u003d \u0027\u0027\u0027"},{"line_number":53,"context_line":"    CREATE TABLE policy_stat ("}],"source_content_type":"text/x-python","patch_set":1,"id":"5a3905b3_1e77ded3","line":50,"range":{"start_line":46,"start_character":0,"end_line":50,"end_character":14},"updated":"2017-01-23 05:40:28.000000000","message":"I don\u0027t think these are used.. so more cruft (Added them because I\u0027m sure they made sense at the time).","commit_id":"e7f5932e086ea82d64c68d526ac753628d2122ac"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"839ee5a76b2bf900536141d677f9060c3490b4e6","unresolved":false,"context_lines":[{"line_number":268,"context_line":"        return existing, None"},{"line_number":269,"context_line":"    else:"},{"line_number":270,"context_line":"        # Both missing"},{"line_number":271,"context_line":"        return \u0027\u0027, True"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":""},{"line_number":274,"context_line":"def merge_pivots(item, existing):"}],"source_content_type":"text/x-python","patch_set":1,"id":"5a3905b3_79eb9c13","line":271,"updated":"2017-01-23 05:40:28.000000000","message":"This has code that can send stats to the root container (obj_count, bytes_used) prefixed with - and +.. the thought here is you can turn on more requests and have the numbers more up todate.. however the current code currently doesn\u0027t use it.. and maybe we\u0027ll decide to never to. In which case these can be greatly simplified.","commit_id":"e7f5932e086ea82d64c68d526ac753628d2122ac"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"839ee5a76b2bf900536141d677f9060c3490b4e6","unresolved":false,"context_lines":[{"line_number":313,"context_line":"        # The auditor will create a backend using the pivot_db as the db_file."},{"line_number":314,"context_line":"        if db_file.endswith(\"_pivot.db\"):"},{"line_number":315,"context_line":"            self._pivot_db_file \u003d db_file"},{"line_number":316,"context_line":"            # self._db_file \u003d \u0027\u0027"},{"line_number":317,"context_line":"        else:"},{"line_number":318,"context_line":"            self._pivot_db_file \u003d db_file[:0 - len(\u0027.db\u0027)] + \"_pivot.db\""},{"line_number":319,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"5a3905b3_39e67444","line":316,"updated":"2017-01-23 05:40:28.000000000","message":"commented out.. so now not sure if it\u0027s needed or not.. but lets assume not.","commit_id":"e7f5932e086ea82d64c68d526ac753628d2122ac"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":236,"context_line":"    return any(newer_than_existing)"},{"line_number":237,"context_line":""},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"def merge_data(item, existing):"},{"line_number":240,"context_line":"    # The Parameters match one of the following:"},{"line_number":241,"context_line":"    #  1. Prefix + item, existing \u003d existing + item"},{"line_number":242,"context_line":"    #  2. Prefix + item, prefix + existing \u003d prefix + exiting + item"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a4ae9195","line":239,"updated":"2017-03-28 00:55:04.000000000","message":"A docstring would be handy","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":237,"context_line":""},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"def merge_data(item, existing):"},{"line_number":240,"context_line":"    # The Parameters match one of the following:"},{"line_number":241,"context_line":"    #  1. Prefix + item, existing \u003d existing + item"},{"line_number":242,"context_line":"    #  2. Prefix + item, prefix + existing \u003d prefix + exiting + item"},{"line_number":243,"context_line":"    #  3. item, existing \u003d item"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_0163ef43","line":240,"range":{"start_line":240,"start_character":10,"end_line":240,"end_character":20},"updated":"2017-03-28 00:55:04.000000000","message":"Which params? I\u0027m not sure I understand the notation below...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":237,"context_line":""},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"def merge_data(item, existing):"},{"line_number":240,"context_line":"    # The Parameters match one of the following:"},{"line_number":241,"context_line":"    #  1. Prefix + item, existing \u003d existing + item"},{"line_number":242,"context_line":"    #  2. Prefix + item, prefix + existing \u003d prefix + exiting + item"},{"line_number":243,"context_line":"    #  3. item, existing \u003d item"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_bee9b9e6","line":240,"range":{"start_line":240,"start_character":10,"end_line":240,"end_character":20},"in_reply_to":"ffe62b97_0163ef43","updated":"2017-03-29 03:59:31.000000000","message":"ahh yeah, that was a note to help trying to cover all use cases while I was writing it.\n\nI\u0027ll update the inline documentation. But from memory this is used when we find an another item (pivot range) in pending items. Usually we just pick the newest one. But this supports a -/+ prefix.\n\nThe rule is, if an item comes in without a prefix, it should be the new value, however if it has a prefix, it needs to change whatever the existing value is. If the existing item (older value found in pending) also has a pending then then these need to be merged. ie. -1, -1 \u003d -2.\n\nSo:\n\n [+,-]item, existing \u003d existing + [+,-]item\n [+,-]item, [+,-]existing \u003d [+,-]existing + [+,-]item\n item, existing \u003d item\n item, [+,-]existing \u003d item\n\nSo it\u0027s if some uses the \u0027-\u0027, \u0027+\u0027 api that we aren\u0027t acutally using. I mention this api briefly in the docs (I think).","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":260,"context_line":"            # Can\u0027t be -\u0027ve"},{"line_number":261,"context_line":"            if item \u003c 0:"},{"line_number":262,"context_line":"                item \u003d 0"},{"line_number":263,"context_line":"        return (\"%s%d\" % (prefix_str, item), prefix)"},{"line_number":264,"context_line":"    elif item:"},{"line_number":265,"context_line":"        return item, None"},{"line_number":266,"context_line":"    elif existing:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9ec9555a","line":263,"updated":"2017-03-29 03:59:31.000000000","message":"Because we don\u0027t use this currently and now looking at it, I don\u0027t this would work. As we only need to add a prefix of \u0027+\u0027 if the existing and item were both prefixed. As:\n\n  \"%s%d\" % (\u0027-\u0027, -1) \u003d\u003d \u0027--1\u0027\n\nBut\n\n  \u0027%s%d\u0027 % (\u0027+\u0027, 1) \u003d\u003d \u0027+1\u0027\n\nHonestly I think I\u0027d need to go back over this prefixed api code to see if it actually works, or still works. It might just be easier to remove it all.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":270,"context_line":"        return \u0027\u0027, True"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"def merge_pivots(item, existing):"},{"line_number":274,"context_line":"    if not existing:"},{"line_number":275,"context_line":"        return True"},{"line_number":276,"context_line":"    if existing[\u0027created_at\u0027] \u003c item[\u0027created_at\u0027]:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c4dd1543","line":273,"updated":"2017-03-28 00:55:04.000000000","message":"Here, too","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":270,"context_line":"        return \u0027\u0027, True"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"def merge_pivots(item, existing):"},{"line_number":274,"context_line":"    if not existing:"},{"line_number":275,"context_line":"        return True"},{"line_number":276,"context_line":"    if existing[\u0027created_at\u0027] \u003c item[\u0027created_at\u0027]:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_fefe2169","line":273,"in_reply_to":"ffe62b97_c4dd1543","updated":"2017-03-29 03:59:31.000000000","message":"Yup, this is the code that checks dates and then tries to merge.. again. I think I need to sit down and refamilarise myself with it... and see if it can be simplied (and doc stringed).\n\nAgain probably much simpler if we remove the prefixed api.. which seemed like a good idea at the time :P","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":299,"context_line":""},{"line_number":300,"context_line":""},{"line_number":301,"context_line":"class ContainerBroker(DatabaseBroker):"},{"line_number":302,"context_line":"    \"\"\"Encapsulates working with a container database.\"\"\""},{"line_number":303,"context_line":"    db_type \u003d \u0027container\u0027"},{"line_number":304,"context_line":"    db_contains_type \u003d \u0027object\u0027"},{"line_number":305,"context_line":"    db_reclaim_timestamp \u003d \u0027created_at\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3f928e9f","line":302,"updated":"2017-03-28 00:55:04.000000000","message":"So does this encapsulate a single (on-disk) container DB, or a set of on-disk DBs corresponding to a single container?\n\nReading a bit more, I guess \"both\"?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":299,"context_line":""},{"line_number":300,"context_line":""},{"line_number":301,"context_line":"class ContainerBroker(DatabaseBroker):"},{"line_number":302,"context_line":"    \"\"\"Encapsulates working with a container database.\"\"\""},{"line_number":303,"context_line":"    db_type \u003d \u0027container\u0027"},{"line_number":304,"context_line":"    db_contains_type \u003d \u0027object\u0027"},{"line_number":305,"context_line":"    db_reclaim_timestamp \u003d \u0027created_at\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_fe33c160","line":302,"in_reply_to":"ffe62b97_3f928e9f","updated":"2017-03-29 03:59:31.000000000","message":"yeah see, it\u0027s not complicated at all :P\n\nHonestly I just went in and made it understand 2 dbs.. but looking back we could take a look at being smarter about it.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":312,"context_line":"        # The auditor will create a backend using the pivot_db as the db_file."},{"line_number":313,"context_line":"        if db_file.endswith(\"_pivot.db\"):"},{"line_number":314,"context_line":"            self._pivot_db_file \u003d db_file"},{"line_number":315,"context_line":"            # self._db_file \u003d \u0027\u0027"},{"line_number":316,"context_line":"        else:"},{"line_number":317,"context_line":"            self._pivot_db_file \u003d db_file[:0 - len(\u0027.db\u0027)] + \"_pivot.db\""},{"line_number":318,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a438d13c","line":315,"updated":"2017-03-28 00:55:04.000000000","message":"Should we delete this?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":312,"context_line":"        # The auditor will create a backend using the pivot_db as the db_file."},{"line_number":313,"context_line":"        if db_file.endswith(\"_pivot.db\"):"},{"line_number":314,"context_line":"            self._pivot_db_file \u003d db_file"},{"line_number":315,"context_line":"            # self._db_file \u003d \u0027\u0027"},{"line_number":316,"context_line":"        else:"},{"line_number":317,"context_line":"            self._pivot_db_file \u003d db_file[:0 - len(\u0027.db\u0027)] + \"_pivot.db\""},{"line_number":318,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_be41b9ab","line":315,"in_reply_to":"ffe62b97_a438d13c","updated":"2017-03-29 03:59:31.000000000","message":"yup, obviously some debugging.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":314,"context_line":"            self._pivot_db_file \u003d db_file"},{"line_number":315,"context_line":"            # self._db_file \u003d \u0027\u0027"},{"line_number":316,"context_line":"        else:"},{"line_number":317,"context_line":"            self._pivot_db_file \u003d db_file[:0 - len(\u0027.db\u0027)] + \"_pivot.db\""},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"    def get_db_state(self):"},{"line_number":320,"context_line":"        db_exists \u003d os.path.exists(self._db_file) and \\"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_8456ed5b","line":317,"updated":"2017-03-28 00:55:04.000000000","message":"Should we have some validation that db_file.endswith(\u0027.db\u0027)? Otherwise, I think in-memory DBs will get a pivot db file like \u0027:memo_pivot.db\u0027 which seems... strange...\n\nnit: can just be db_file[:-len(\u0027.db\u0027)]","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":316,"context_line":"        else:"},{"line_number":317,"context_line":"            self._pivot_db_file \u003d db_file[:0 - len(\u0027.db\u0027)] + \"_pivot.db\""},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"    def get_db_state(self):"},{"line_number":320,"context_line":"        db_exists \u003d os.path.exists(self._db_file) and \\"},{"line_number":321,"context_line":"            self._db_file !\u003d self._pivot_db_file"},{"line_number":322,"context_line":"        pivot_exists \u003d os.path.exists(self._pivot_db_file)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_10fe97ad","line":319,"updated":"2017-03-28 00:55:04.000000000","message":"How does this work for in-memory DBs?\n\nShould we have a\n\n if self._db_file \u003d\u003d \u0027:memory:\u0027:\n     return DB_STATE_UNSHARDED\n\nor something?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":317,"context_line":"            self._pivot_db_file \u003d db_file[:0 - len(\u0027.db\u0027)] + \"_pivot.db\""},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"    def get_db_state(self):"},{"line_number":320,"context_line":"        db_exists \u003d os.path.exists(self._db_file) and \\"},{"line_number":321,"context_line":"            self._db_file !\u003d self._pivot_db_file"},{"line_number":322,"context_line":"        pivot_exists \u003d os.path.exists(self._pivot_db_file)"},{"line_number":323,"context_line":"        if db_exists and not pivot_exists:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_dfbf22ce","line":320,"range":{"start_line":320,"start_character":20,"end_line":320,"end_character":34},"updated":"2017-03-28 00:55:04.000000000","message":"os.path.exists, or os.path.isfile? Does it matter much?\n\nIf we ever find ourselves with directories where we expected to find files (fs corruption, or some botched copy), will somebody (presumably the auditor) eventually clean that up?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":318,"context_line":""},{"line_number":319,"context_line":"    def get_db_state(self):"},{"line_number":320,"context_line":"        db_exists \u003d os.path.exists(self._db_file) and \\"},{"line_number":321,"context_line":"            self._db_file !\u003d self._pivot_db_file"},{"line_number":322,"context_line":"        pivot_exists \u003d os.path.exists(self._pivot_db_file)"},{"line_number":323,"context_line":"        if db_exists and not pivot_exists:"},{"line_number":324,"context_line":"            return DB_STATE_UNSHARDED"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9f0cfa3b","line":321,"updated":"2017-03-28 00:55:04.000000000","message":"So if we have a ContainerBroker for a pivot DB... _db_file will always be the same as _pivot_db_file, and this will always be False, right?\n\nSo, it\u0027ll only ever claim to be in the SHARDED or NOTFOUND state?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":330,"context_line":"            # Both Pivot db and db doesn\u0027t doesn\u0027t exist."},{"line_number":331,"context_line":"            return DB_STATE_NOTFOUND"},{"line_number":332,"context_line":""},{"line_number":333,"context_line":"    @property"},{"line_number":334,"context_line":"    def db_file(self):"},{"line_number":335,"context_line":"        db_state \u003d self.get_db_state()"},{"line_number":336,"context_line":"        if db_state in (DB_STATE_NOTFOUND, DB_STATE_UNSHARDED):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_bf5c5e7e","line":333,"updated":"2017-03-28 00:55:04.000000000","message":"I was debating about whether get_db_state ought to be a property... then thought \"maybe it was intentional that we made it a function; it makes a couple syscalls to check existence on disk, those aren\u0027t exactly trivial... maybe we intentionally made it a function to imply that it\u0027s heavier-weight than you might assume from a property\"\n\nBut the first thing we do here is go get the DB state :-/","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":339,"context_line":"            return self._pivot_db_file"},{"line_number":340,"context_line":""},{"line_number":341,"context_line":"    def _db_exists(self):"},{"line_number":342,"context_line":"        return not self.get_db_state() \u003d\u003d DB_STATE_NOTFOUND"},{"line_number":343,"context_line":""},{"line_number":344,"context_line":"    @property"},{"line_number":345,"context_line":"    def storage_policy_index(self):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9f251aae","line":342,"updated":"2017-03-28 00:55:04.000000000","message":"nit: Maybe just\n\n self.get_db_state() !\u003d DB_STATE_NOTFOUND\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":364,"context_line":"        self.create_policy_stat_table(conn, storage_policy_index)"},{"line_number":365,"context_line":"        self.create_container_info_table(conn, put_timestamp,"},{"line_number":366,"context_line":"                                         storage_policy_index)"},{"line_number":367,"context_line":"        self.create_pivot_ranges_table(conn)"},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"    def create_object_table(self, conn):"},{"line_number":370,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_5a232006","line":367,"updated":"2017-03-28 00:55:04.000000000","message":"So all DBs always have a pivot table, and whether or not it\u0027s populated is a separate question...\n\nAnd we migrate during get_pivot_ranges and get_pivot_usage. What about merge_items or update_pivot_usage?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":364,"context_line":"        self.create_policy_stat_table(conn, storage_policy_index)"},{"line_number":365,"context_line":"        self.create_container_info_table(conn, put_timestamp,"},{"line_number":366,"context_line":"                                         storage_policy_index)"},{"line_number":367,"context_line":"        self.create_pivot_ranges_table(conn)"},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"    def create_object_table(self, conn):"},{"line_number":370,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_1e7f85e0","line":367,"in_reply_to":"ffe62b97_5a232006","updated":"2017-03-29 03:59:31.000000000","message":"Well in older POCs there was no such thing as a locked DB. so this was a new table all containers had.\n\nNow, a root container in the UNSHARDED state may store some pivotranges, if it happens to not be the scanner node, and replicates with the scanner after the scanner has found some pivots. So the answer is still yes.. but maybe we don\u0027t _need_ to create it at initalize anymore.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":464,"context_line":"                    created_at TEXT,"},{"line_number":465,"context_line":"                    meta_timestamp TEXT,"},{"line_number":466,"context_line":"                    deleted INTEGER DEFAULT 0"},{"line_number":467,"context_line":"                );"},{"line_number":468,"context_line":"            \"\"\")"},{"line_number":469,"context_line":"        except Exception:"},{"line_number":470,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_f08a1319","line":467,"updated":"2017-03-28 00:55:04.000000000","message":"Should this include a column for storage policy?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":464,"context_line":"                    created_at TEXT,"},{"line_number":465,"context_line":"                    meta_timestamp TEXT,"},{"line_number":466,"context_line":"                    deleted INTEGER DEFAULT 0"},{"line_number":467,"context_line":"                );"},{"line_number":468,"context_line":"            \"\"\")"},{"line_number":469,"context_line":"        except Exception:"},{"line_number":470,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_1ecd4530","line":467,"in_reply_to":"ffe62b97_f08a1319","updated":"2017-03-29 03:59:31.000000000","message":"when we pickle we need to have place for that data, but only as a place holder, it\u0027s always 0. Thanks to pending pickles to only old item data. And I\u0027m now making it hold more. This is why you see it trying to deal with storage policies in the code.\n\nBut a pivot range has no concept or care about storage policies. So no.\n\nOnce this gets in, item pickles can happy grow, because we can append, as we\u0027ll have the record_type index of the pickle to tell us what type it is.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":467,"context_line":"                );"},{"line_number":468,"context_line":"            \"\"\")"},{"line_number":469,"context_line":"        except Exception:"},{"line_number":470,"context_line":"            pass"},{"line_number":471,"context_line":""},{"line_number":472,"context_line":"    def get_db_version(self, conn):"},{"line_number":473,"context_line":"        if self._db_version \u003d\u003d -1:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_daca9018","line":470,"updated":"2017-03-28 00:55:04.000000000","message":"We just swallow exceptions? Why?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":467,"context_line":"                );"},{"line_number":468,"context_line":"            \"\"\")"},{"line_number":469,"context_line":"        except Exception:"},{"line_number":470,"context_line":"            pass"},{"line_number":471,"context_line":""},{"line_number":472,"context_line":"    def get_db_version(self, conn):"},{"line_number":473,"context_line":"        if self._db_version \u003d\u003d -1:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9e6a7517","line":470,"in_reply_to":"ffe62b97_daca9018","updated":"2017-03-29 03:59:31.000000000","message":"We shouldn\u0027t.. but through the magic of POCs I obviously just want to push forward to the sharding and never came back (the pass here was more of a place holder).","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":502,"context_line":"        \"\"\"See :func:`swift.common.db.DatabaseBroker._commit_puts_load`\"\"\""},{"line_number":503,"context_line":"        data \u003d pickle.loads(entry.decode(\u0027base64\u0027))"},{"line_number":504,"context_line":"        record_type \u003d data[9] if len(data) \u003e 9 else RECORD_TYPE_OBJECT"},{"line_number":505,"context_line":"        if record_type and record_type \u003d\u003d RECORD_TYPE_PIVOT_NODE:"},{"line_number":506,"context_line":"            (name, timestamp, lower, upper, object_count, bytes_used,"},{"line_number":507,"context_line":"             meta_timestamp, deleted) \u003d data[:8]"},{"line_number":508,"context_line":"            item \u003d {"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_b55a3df0","line":505,"updated":"2017-03-28 00:55:04.000000000","message":"Maybe just\n\n if record_type \u003d\u003d RECORD_TYPE_PIVOT_NODE:\n\nIs it weird that RECORD_TYPE_OBJECT is falsey?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":502,"context_line":"        \"\"\"See :func:`swift.common.db.DatabaseBroker._commit_puts_load`\"\"\""},{"line_number":503,"context_line":"        data \u003d pickle.loads(entry.decode(\u0027base64\u0027))"},{"line_number":504,"context_line":"        record_type \u003d data[9] if len(data) \u003e 9 else RECORD_TYPE_OBJECT"},{"line_number":505,"context_line":"        if record_type and record_type \u003d\u003d RECORD_TYPE_PIVOT_NODE:"},{"line_number":506,"context_line":"            (name, timestamp, lower, upper, object_count, bytes_used,"},{"line_number":507,"context_line":"             meta_timestamp, deleted) \u003d data[:8]"},{"line_number":508,"context_line":"            item \u003d {"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_1ef66564","line":505,"in_reply_to":"ffe62b97_b55a3df0","updated":"2017-03-29 03:59:31.000000000","message":"yup nice, seeing as record_type is definitely being defined the line before :)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":504,"context_line":"        record_type \u003d data[9] if len(data) \u003e 9 else RECORD_TYPE_OBJECT"},{"line_number":505,"context_line":"        if record_type and record_type \u003d\u003d RECORD_TYPE_PIVOT_NODE:"},{"line_number":506,"context_line":"            (name, timestamp, lower, upper, object_count, bytes_used,"},{"line_number":507,"context_line":"             meta_timestamp, deleted) \u003d data[:8]"},{"line_number":508,"context_line":"            item \u003d {"},{"line_number":509,"context_line":"                \u0027name\u0027: name,"},{"line_number":510,"context_line":"                \u0027created_at\u0027: timestamp,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_1518c972","line":507,"updated":"2017-03-28 00:55:04.000000000","message":"So I guess we had to pad by one","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":504,"context_line":"        record_type \u003d data[9] if len(data) \u003e 9 else RECORD_TYPE_OBJECT"},{"line_number":505,"context_line":"        if record_type and record_type \u003d\u003d RECORD_TYPE_PIVOT_NODE:"},{"line_number":506,"context_line":"            (name, timestamp, lower, upper, object_count, bytes_used,"},{"line_number":507,"context_line":"             meta_timestamp, deleted) \u003d data[:8]"},{"line_number":508,"context_line":"            item \u003d {"},{"line_number":509,"context_line":"                \u0027name\u0027: name,"},{"line_number":510,"context_line":"                \u0027created_at\u0027: timestamp,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_dea0fd44","line":507,"in_reply_to":"ffe62b97_1518c972","updated":"2017-03-29 03:59:31.000000000","message":"Yeah :(\n\nsooner we get it in the less padding we may need to add.. or we think up a new way of pickling (or having 2 pickles).","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":559,"context_line":"                    \u0027SELECT object_count from container_stat\u0027).fetchone()"},{"line_number":560,"context_line":"            return (row[0] \u003d\u003d 0)"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"    def delete_object(self, name, timestamp, storage_policy_index\u003d0,"},{"line_number":563,"context_line":"                      lower\u003dNone, upper\u003dNone, object_count\u003d0, bytes_used\u003d0,"},{"line_number":564,"context_line":"                      meta_timestamp\u003dNone, record_type\u003dRECORD_TYPE_OBJECT):"},{"line_number":565,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3522ad01","line":562,"updated":"2017-03-28 00:55:04.000000000","message":"So delete_object can also delete pivot ranges? I wonder whether we\u0027re starting to stretch our abstractions to breaking...\n\nWe\u0027ve already got a more-general put_record... would it be viable to have put_pivot_range/delete_pivot_range and not muddy put_object/delete_object?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":559,"context_line":"                    \u0027SELECT object_count from container_stat\u0027).fetchone()"},{"line_number":560,"context_line":"            return (row[0] \u003d\u003d 0)"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"    def delete_object(self, name, timestamp, storage_policy_index\u003d0,"},{"line_number":563,"context_line":"                      lower\u003dNone, upper\u003dNone, object_count\u003d0, bytes_used\u003d0,"},{"line_number":564,"context_line":"                      meta_timestamp\u003dNone, record_type\u003dRECORD_TYPE_OBJECT):"},{"line_number":565,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9eaf1529","line":562,"in_reply_to":"ffe62b97_3522ad01","updated":"2017-03-29 03:59:31.000000000","message":"Yeah, good call. I dunno. At first we weren\u0027t pushing it so much, only now 5 POC later have we got here.\n\nI agree, we should just say, look it\u0027s a pivot so lets add/delete pivot!\n\n\\o/ more cruft that grew to be teh crap.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0287f58fc2df6a8b7b215eff62467c0f1d38b64b","unresolved":false,"context_lines":[{"line_number":559,"context_line":"                    \u0027SELECT object_count from container_stat\u0027).fetchone()"},{"line_number":560,"context_line":"            return (row[0] \u003d\u003d 0)"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"    def delete_object(self, name, timestamp, storage_policy_index\u003d0,"},{"line_number":563,"context_line":"                      lower\u003dNone, upper\u003dNone, object_count\u003d0, bytes_used\u003d0,"},{"line_number":564,"context_line":"                      meta_timestamp\u003dNone, record_type\u003dRECORD_TYPE_OBJECT):"},{"line_number":565,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_64283e96","line":562,"in_reply_to":"ffe62b97_9eaf1529","updated":"2017-03-29 05:55:05.000000000","message":"https://review.openstack.org/451197","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":560,"context_line":"            return (row[0] \u003d\u003d 0)"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"    def delete_object(self, name, timestamp, storage_policy_index\u003d0,"},{"line_number":563,"context_line":"                      lower\u003dNone, upper\u003dNone, object_count\u003d0, bytes_used\u003d0,"},{"line_number":564,"context_line":"                      meta_timestamp\u003dNone, record_type\u003dRECORD_TYPE_OBJECT):"},{"line_number":565,"context_line":"        \"\"\""},{"line_number":566,"context_line":"        Mark an object deleted."},{"line_number":567,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_7525d5c1","line":564,"range":{"start_line":563,"start_character":46,"end_line":564,"end_character":41},"updated":"2017-03-28 00:55:04.000000000","message":"What does meta_timestamp mean for a deleted pivot range? It weirds me out a little that we pass this around for deleted pivot ranges but not deleted objects.\n\nWon\u0027t object_count and bytes_used always be 0?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":560,"context_line":"            return (row[0] \u003d\u003d 0)"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"    def delete_object(self, name, timestamp, storage_policy_index\u003d0,"},{"line_number":563,"context_line":"                      lower\u003dNone, upper\u003dNone, object_count\u003d0, bytes_used\u003d0,"},{"line_number":564,"context_line":"                      meta_timestamp\u003dNone, record_type\u003dRECORD_TYPE_OBJECT):"},{"line_number":565,"context_line":"        \"\"\""},{"line_number":566,"context_line":"        Mark an object deleted."},{"line_number":567,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_fe6bc1ed","line":564,"range":{"start_line":563,"start_character":46,"end_line":564,"end_character":41},"in_reply_to":"ffe62b97_7525d5c1","updated":"2017-03-29 03:59:31.000000000","message":"hmm yeah, you have a point","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":563,"context_line":"                      lower\u003dNone, upper\u003dNone, object_count\u003d0, bytes_used\u003d0,"},{"line_number":564,"context_line":"                      meta_timestamp\u003dNone, record_type\u003dRECORD_TYPE_OBJECT):"},{"line_number":565,"context_line":"        \"\"\""},{"line_number":566,"context_line":"        Mark an object deleted."},{"line_number":567,"context_line":""},{"line_number":568,"context_line":"        :param name: object name to be deleted"},{"line_number":569,"context_line":"        :param timestamp: timestamp when the object was marked as deleted"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_d57ca1ec","line":566,"updated":"2017-03-28 00:55:04.000000000","message":"Needs to be updated.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":585,"context_line":"            return (record[\u0027name\u0027], record[\u0027created_at\u0027], record[\u0027lower\u0027],"},{"line_number":586,"context_line":"                    record[\u0027upper\u0027], record[\u0027object_count\u0027],"},{"line_number":587,"context_line":"                    record[\u0027bytes_used\u0027], record[\u0027meta_timestamp\u0027],"},{"line_number":588,"context_line":"                    record[\u0027deleted\u0027], 0, record[\u0027record_type\u0027])"},{"line_number":589,"context_line":"        return (record[\u0027name\u0027], record[\u0027created_at\u0027], record[\u0027size\u0027],"},{"line_number":590,"context_line":"                record[\u0027content_type\u0027], record[\u0027etag\u0027], record[\u0027deleted\u0027],"},{"line_number":591,"context_line":"                record[\u0027storage_policy_index\u0027],"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_95edf98c","line":588,"range":{"start_line":588,"start_character":39,"end_line":588,"end_character":40},"updated":"2017-03-28 00:55:04.000000000","message":"Got it; there\u0027s our padding.\n\nWorth adding a comment in here about how record_type *must* be at index 9?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":585,"context_line":"            return (record[\u0027name\u0027], record[\u0027created_at\u0027], record[\u0027lower\u0027],"},{"line_number":586,"context_line":"                    record[\u0027upper\u0027], record[\u0027object_count\u0027],"},{"line_number":587,"context_line":"                    record[\u0027bytes_used\u0027], record[\u0027meta_timestamp\u0027],"},{"line_number":588,"context_line":"                    record[\u0027deleted\u0027], 0, record[\u0027record_type\u0027])"},{"line_number":589,"context_line":"        return (record[\u0027name\u0027], record[\u0027created_at\u0027], record[\u0027size\u0027],"},{"line_number":590,"context_line":"                record[\u0027content_type\u0027], record[\u0027etag\u0027], record[\u0027deleted\u0027],"},{"line_number":591,"context_line":"                record[\u0027storage_policy_index\u0027],"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_be79b9c8","line":588,"range":{"start_line":588,"start_character":39,"end_line":588,"end_character":40},"in_reply_to":"ffe62b97_95edf98c","updated":"2017-03-29 03:59:31.000000000","message":"+1","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":598,"context_line":"                   meta_timestamp\u003dNone, lower\u003dNone, upper\u003dNone, object_count\u003d0,"},{"line_number":599,"context_line":"                   bytes_used\u003d0, record_type\u003dRECORD_TYPE_OBJECT):"},{"line_number":600,"context_line":"        \"\"\""},{"line_number":601,"context_line":"        Creates an object in the DB with its metadata."},{"line_number":602,"context_line":""},{"line_number":603,"context_line":"        :param name: object name to be created"},{"line_number":604,"context_line":"        :param timestamp: timestamp of when the object was created"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_30b99b2d","line":601,"updated":"2017-03-28 00:55:04.000000000","message":"Needs updating","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":615,"context_line":"                            object data or a pivot point)"},{"line_number":616,"context_line":"        \"\"\""},{"line_number":617,"context_line":"        if record_type \u003d\u003d RECORD_TYPE_PIVOT_NODE:"},{"line_number":618,"context_line":"            if lower \u003d\u003d \u0027None\u0027:"},{"line_number":619,"context_line":"                lower \u003d None"},{"line_number":620,"context_line":"            if upper \u003d\u003d \u0027None\u0027:"},{"line_number":621,"context_line":"                upper \u003d None"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_304bdb59","line":618,"updated":"2017-03-28 00:55:04.000000000","message":"Where might it get str()ed? Can we just, y\u0027know, not do that?\n\nWhat if we actually *do* want a pivot to start or end at \u0027None\u0027?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cf4ece22fe772a8a43ba3374bdc13554fa5d0e27","unresolved":false,"context_lines":[{"line_number":615,"context_line":"                            object data or a pivot point)"},{"line_number":616,"context_line":"        \"\"\""},{"line_number":617,"context_line":"        if record_type \u003d\u003d RECORD_TYPE_PIVOT_NODE:"},{"line_number":618,"context_line":"            if lower \u003d\u003d \u0027None\u0027:"},{"line_number":619,"context_line":"                lower \u003d None"},{"line_number":620,"context_line":"            if upper \u003d\u003d \u0027None\u0027:"},{"line_number":621,"context_line":"                upper \u003d None"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9e5dd548","line":618,"in_reply_to":"ffe62b97_304bdb59","updated":"2017-03-29 03:59:31.000000000","message":"Honestly, I\u0027d have to go check, can\u0027t remember. And yes, great points.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":669,"context_line":"                  returned by get_info and is_deleted is a boolean."},{"line_number":670,"context_line":"        \"\"\""},{"line_number":671,"context_line":"        if self._db_file !\u003d \u0027:memory:\u0027 and \\"},{"line_number":672,"context_line":"                self.get_db_state() \u003d\u003d DB_STATE_NOTFOUND:"},{"line_number":673,"context_line":"            return {}, True"},{"line_number":674,"context_line":"        info \u003d self.get_info()"},{"line_number":675,"context_line":"        return info, self._is_deleted_info(**info)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_30f4fb85","line":672,"updated":"2017-03-28 00:55:04.000000000","message":"It seems like we\u0027ve been moving toward using self._db_exists() for these sorts of checks?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":696,"context_line":"                                    reported_object_count, reported_bytes_used, hash,"},{"line_number":697,"context_line":"                                    id, %s, %s"},{"line_number":698,"context_line":"                                    FROM container_stat"},{"line_number":699,"context_line":"                            \u0027\u0027\u0027) % (trailing_sync, trailing_pol)).fetchone()"},{"line_number":700,"context_line":"                except sqlite3.OperationalError as err:"},{"line_number":701,"context_line":"                    err_msg \u003d str(err)"},{"line_number":702,"context_line":"                    if err_msg in errors:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3022bb01","line":699,"updated":"2017-03-28 00:55:04.000000000","message":"Dedent? Should clear up some more flake8 failures","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":717,"context_line":"            self.container \u003d data[\u0027container\u0027]"},{"line_number":718,"context_line":""},{"line_number":719,"context_line":"            db_state \u003d self.get_db_state()"},{"line_number":720,"context_line":"            data[\u0027db_state\u0027] \u003d db_state"},{"line_number":721,"context_line":""},{"line_number":722,"context_line":"            return data"},{"line_number":723,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_f0c7b311","line":720,"updated":"2017-03-28 00:55:04.000000000","message":"nit: can just be\n\n data[\u0027db_state\u0027] \u003d self.get_db_state()","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":735,"context_line":"        \"\"\""},{"line_number":736,"context_line":"        data \u003d self._get_info()"},{"line_number":737,"context_line":""},{"line_number":738,"context_line":"        db_state \u003d self.get_db_state()"},{"line_number":739,"context_line":"        if db_state \u003d\u003d DB_STATE_SHARDING:"},{"line_number":740,"context_line":"            # grab the obj_count, bytes used from locked DB. We need"},{"line_number":741,"context_line":"            # obj_count for sharding."}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_7056035f","line":738,"updated":"2017-03-28 00:55:04.000000000","message":"nit: pull this out of data[\u0027db_state\u0027]","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":738,"context_line":"        db_state \u003d self.get_db_state()"},{"line_number":739,"context_line":"        if db_state \u003d\u003d DB_STATE_SHARDING:"},{"line_number":740,"context_line":"            # grab the obj_count, bytes used from locked DB. We need"},{"line_number":741,"context_line":"            # obj_count for sharding."},{"line_number":742,"context_line":"            other_info \u003d self.get_brokers()[0]._get_info()"},{"line_number":743,"context_line":"            data.update({\u0027object_count\u0027: other_info.get(\u0027object_count\u0027, 0),"},{"line_number":744,"context_line":"                         \u0027bytes_used\u0027: other_info.get(\u0027bytes_used\u0027, 0)})"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_d066ef29","line":741,"updated":"2017-03-28 00:55:04.000000000","message":"What do these look like in container_stat while we\u0027re sharding?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":887,"context_line":"                     the path"},{"line_number":888,"context_line":"        :param storage_policy_index: storage policy index for query"},{"line_number":889,"context_line":"        :param reverse: reverse the result order."},{"line_number":890,"context_line":"        :param include_end_marker: Include the item at end_marker in results"},{"line_number":891,"context_line":"        :param include_deleted: Include items that have the delete marker set"},{"line_number":892,"context_line":""},{"line_number":893,"context_line":"        :returns: list of tuples of (name, created_at, size, content_type,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_124fbc34","line":890,"updated":"2017-03-28 00:55:04.000000000","message":"I can\u0027t put my finger on it just yet, but it feels weird to me that we need this...\n\nAt the very least, it seems strange that we would need it for end_marker but not for marker :-/","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":942,"context_line":"                    query +\u003d \u0027 name \u003e\u003d ? AND\u0027"},{"line_number":943,"context_line":"                    query_args.append(prefix)"},{"line_number":944,"context_line":"                if include_deleted and query.endswith(\u0027 AND\u0027):"},{"line_number":945,"context_line":"                    query \u003d query[:-4]"},{"line_number":946,"context_line":"                else:"},{"line_number":947,"context_line":"                    if self.get_db_version(conn) \u003c 1:"},{"line_number":948,"context_line":"                        query +\u003d \u0027 +deleted \u003d 0\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_92b4ecd2","line":945,"updated":"2017-03-28 00:55:04.000000000","message":"Have we checked that we still use ix_object_deleted_name here? We may need something like\n\n if self.get_db_version(conn) \u003c 1:\n     query \u003d query[:-4]\n else:\n     query +\u003d \u0027 deleted in (0, 1)\u0027\n\nhere instead... somewhat similar to what we do down in merge_items.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":943,"context_line":"                    query_args.append(prefix)"},{"line_number":944,"context_line":"                if include_deleted and query.endswith(\u0027 AND\u0027):"},{"line_number":945,"context_line":"                    query \u003d query[:-4]"},{"line_number":946,"context_line":"                else:"},{"line_number":947,"context_line":"                    if self.get_db_version(conn) \u003c 1:"},{"line_number":948,"context_line":"                        query +\u003d \u0027 +deleted \u003d 0\u0027"},{"line_number":949,"context_line":"                    else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_35aaaa70","line":946,"updated":"2017-03-28 00:55:04.000000000","message":"So even in the\n\n include_deleted and not query.endswith(\u0027 AND\u0027)\n\ncase we check for deleted \u003d 0? That seems wrong...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1044,"context_line":""},{"line_number":1045,"context_line":"    def merge_items(self, item_list, source\u003dNone):"},{"line_number":1046,"context_line":"        \"\"\""},{"line_number":1047,"context_line":"        Merge items into the object table."},{"line_number":1048,"context_line":""},{"line_number":1049,"context_line":"        :param item_list: list of dictionaries of {\u0027name\u0027, \u0027created_at\u0027,"},{"line_number":1050,"context_line":"                          \u0027size\u0027, \u0027content_type\u0027, \u0027etag\u0027, \u0027deleted\u0027,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_f215789d","line":1047,"updated":"2017-03-28 00:55:04.000000000","message":"Needs updating, now that it can merge pivot ranges","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1055,"context_line":"        for item in item_list:"},{"line_number":1056,"context_line":"            item.setdefault(\u0027record_type\u0027, RECORD_TYPE_OBJECT)"},{"line_number":1057,"context_line":"            cols \u003d [\u0027name\u0027]"},{"line_number":1058,"context_line":"            if item.get(\u0027lower\u0027):"},{"line_number":1059,"context_line":"                cols +\u003d [\u0027lower\u0027, \u0027upper\u0027]"},{"line_number":1060,"context_line":"            for col in cols:"},{"line_number":1061,"context_line":"                if isinstance(item[col], six.text_type):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_d2cdd431","line":1058,"updated":"2017-03-28 00:55:04.000000000","message":"Maybe better to switch off item[\u0027record_type\u0027]?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1065,"context_line":"                            if item[\u0027record_type\u0027] \u003d\u003d RECORD_TYPE_PIVOT_NODE]"},{"line_number":1066,"context_line":""},{"line_number":1067,"context_line":"        item_list \u003d [item for item in item_list"},{"line_number":1068,"context_line":"                     if item[\u0027record_type\u0027] \u003d\u003d RECORD_TYPE_OBJECT]"},{"line_number":1069,"context_line":""},{"line_number":1070,"context_line":"        def get_records_object(curs, offset, rec_list, query_mod):"},{"line_number":1071,"context_line":"            chunk \u003d [record[\u0027name\u0027] for record"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_d29b9415","line":1068,"updated":"2017-03-28 00:55:04.000000000","message":"Should we raise some sort of error if\n\n any(item for item in item_list\n     if item[\u0027record_type\u0027] not in (RECORD_TYPE_OBJECT,\n                                    RECORD_TYPE_PIVOT_NODE))\n\n?\n\nAm I right to think that we should generally only have objects or only have pivot nodes? Under what circumstances would we have both? Replicating object holding tables, I guess?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1081,"context_line":"        def get_records_pivot(curs, offset, rec_list, query_mod):"},{"line_number":1082,"context_line":"                chunk \u003d [record[\u0027name\u0027] for record"},{"line_number":1083,"context_line":"                         in rec_list[offset:offset + SQLITE_ARG_LIMIT]]"},{"line_number":1084,"context_line":"                # None/NULL must be checked with the IS operator"},{"line_number":1085,"context_line":"                sql \u003d ((\u0027SELECT name, created_at, lower, upper, object_count, \u0027"},{"line_number":1086,"context_line":"                        \u0027bytes_used, meta_timestamp, deleted \u0027"},{"line_number":1087,"context_line":"                        \u0027FROM pivot_ranges \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_95485687","line":1084,"updated":"2017-03-28 00:55:04.000000000","message":"This seems like useful information, but I don\u0027t quite see how it\u0027s useful here.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1119,"context_line":"            # Get created_at times for objects in rec_list that already exist."},{"line_number":1120,"context_line":"            # We must chunk it up to avoid sqlite\u0027s limit of 999 args."},{"line_number":1121,"context_line":"            records \u003d {}"},{"line_number":1122,"context_line":"            for offset in range(0, len(rec_list), SQLITE_ARG_LIMIT):"},{"line_number":1123,"context_line":"                records.update("},{"line_number":1124,"context_line":"                    get_records(curs, offset, rec_list, query_mod))"},{"line_number":1125,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_d82e1591","line":1122,"updated":"2017-03-28 00:55:04.000000000","message":"nit: Seems like\n\n chunk \u003d [record[\u0027name\u0027] for record\n          in rec_list[offset:offset + SQLITE_ARG_LIMIT]]\n\nought to go here, and then we can pass curs, chunk, and query_mod to get_records.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1151,"context_line":"                    self.update_pivot_usage(item)"},{"line_number":1152,"context_line":""},{"line_number":1153,"context_line":"            if to_delete:"},{"line_number":1154,"context_line":"                filters \u003d \u0027 AND \u0027.join([\u0027%s IS ?\u0027 % key for key in del_keys])"},{"line_number":1155,"context_line":"                sql \u003d \u0027DELETE FROM %s WHERE %s%s\u0027 % (table, query_mod, filters)"},{"line_number":1156,"context_line":"                del_generator \u003d (tuple([record[key] for key in del_keys])"},{"line_number":1157,"context_line":"                                 for record in to_delete.values())"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_98376d4e","line":1154,"updated":"2017-03-28 00:55:04.000000000","message":"Can name or storage policy actually be NULL?\n\nWas this a holdover from a time when del_keys included lower/upper?\n\nI\u0027m wondering again, why *are* lower \u0026 upper allowed to be NULL? Why not use the empty string for \"to end of range\" similar to how we use it for marker/end_marker?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1164,"context_line":"                                 for record in to_add.values())"},{"line_number":1165,"context_line":"                curs.executemany(sql, add_generator)"},{"line_number":1166,"context_line":""},{"line_number":1167,"context_line":"        def _really_merge_items(conn):"},{"line_number":1168,"context_line":"            curs \u003d conn.cursor()"},{"line_number":1169,"context_line":"            curs.execute(\u0027BEGIN IMMEDIATE\u0027)"},{"line_number":1170,"context_line":"            if item_list:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_b87b5179","line":1167,"updated":"2017-03-28 00:55:04.000000000","message":"So we\u0027ve now got\n\n* a property called merge_items which has\n* local variables called _merge_items and _really_merge_items where\n* _merge_items is the one that does most of the merging\n* by calling out to one of two global functions update_new_item_from_existing or merge_pivots, one of which in turn calls another\n* global function merge_data?\n\nNaming is hard :-(","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1171,"context_line":"                _merge_items(curs, item_list)"},{"line_number":1172,"context_line":"            if pivot_range_list:"},{"line_number":1173,"context_line":"                _merge_items(curs, pivot_range_list, False)"},{"line_number":1174,"context_line":"            if source and item_list:"},{"line_number":1175,"context_line":"                # for replication we rely on the remote end sending merges in"},{"line_number":1176,"context_line":"                # order with no gaps to increment sync_points"},{"line_number":1177,"context_line":"                sync_point \u003d item_list[-1][\u0027ROWID\u0027]"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_06badabf","line":1174,"updated":"2017-03-28 00:55:04.000000000","message":"Why only if there were objects to sync? Do we not track sync points in sharded containers?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1359,"context_line":"                data.row_factory \u003d None"},{"line_number":1360,"context_line":"                return [row for row in data]"},{"line_number":1361,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1362,"context_line":"                if \u0027no such table: pivot_ranges\u0027 in str(err):"},{"line_number":1363,"context_line":"                    self.create_pivot_ranges_table(conn)"},{"line_number":1364,"context_line":"            return []"},{"line_number":1365,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_46f10295","line":1362,"updated":"2017-03-28 00:55:04.000000000","message":"else: raise?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1383,"context_line":"                if new_data \u003c 0:"},{"line_number":1384,"context_line":"                    new_data \u003d 0"},{"line_number":1385,"context_line":""},{"line_number":1386,"context_line":"            return new_data"},{"line_number":1387,"context_line":""},{"line_number":1388,"context_line":"        with self.get() as conn:"},{"line_number":1389,"context_line":"            # we need the current value."}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a6856ed1","line":1386,"updated":"2017-03-28 00:55:04.000000000","message":"So this sometimes returns a string and sometimes an integer?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1392,"context_line":"                SELECT object_count, bytes_used"},{"line_number":1393,"context_line":"                FROM pivot_ranges"},{"line_number":1394,"context_line":"                WHERE deleted\u003d0 AND"},{"line_number":1395,"context_line":"                created_at \u003c ? AND"},{"line_number":1396,"context_line":"                name \u003d\u003d ?;"},{"line_number":1397,"context_line":"                \u0027\u0027\u0027"},{"line_number":1398,"context_line":"                data \u003d conn.execute(sql, item[\u0027created_at\u0027], item[\u0027name\u0027])"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_86b38a98","line":1395,"updated":"2017-03-28 00:55:04.000000000","message":"What happens when we have a newer pivot_range? I guess we just assume it already includes the data from item?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1398,"context_line":"                data \u003d conn.execute(sql, item[\u0027created_at\u0027], item[\u0027name\u0027])"},{"line_number":1399,"context_line":"                row \u003d data.fetchone()"},{"line_number":1400,"context_line":"                if not row:"},{"line_number":1401,"context_line":"                    row \u003d 0"},{"line_number":1402,"context_line":"            except sqlite3.OperationalError:"},{"line_number":1403,"context_line":"                row \u003d 0"},{"line_number":1404,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_26f83e47","line":1401,"updated":"2017-03-28 00:55:04.000000000","message":"We mean something like\n\n row \u003d [0, 0]\n\nright? Otherwise we\u0027ll hit an IndexError below.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1415,"context_line":"                WHERE deleted\u003d0;"},{"line_number":1416,"context_line":"                \u0027\u0027\u0027"},{"line_number":1417,"context_line":"                data \u003d conn.execute(sql)"},{"line_number":1418,"context_line":"                data.row_factory \u003d None"},{"line_number":1419,"context_line":"                row \u003d data.fetchone()"},{"line_number":1420,"context_line":"                object_count \u003d row[0]"},{"line_number":1421,"context_line":"                bytes_used \u003d row[1]"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_26a6fe54","line":1418,"updated":"2017-03-28 00:55:04.000000000","message":"?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1422,"context_line":"                return {\u0027bytes_used\u0027: bytes_used,"},{"line_number":1423,"context_line":"                        \u0027object_count\u0027: object_count}"},{"line_number":1424,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1425,"context_line":"                if \u0027no such table: pivot_ranges\u0027 in str(err):"},{"line_number":1426,"context_line":"                    self.create_pivot_ranges_table(conn)"},{"line_number":1427,"context_line":"            return {\u0027bytes_used\u0027: 0, \u0027object_count\u0027: 0}"},{"line_number":1428,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a66faefa","line":1425,"updated":"2017-03-28 00:55:04.000000000","message":"else: raise?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1456,"context_line":"                    \u0027record_type\u0027: RECORD_TYPE_PIVOT_NODE})"},{"line_number":1457,"context_line":"                result.append(obj)"},{"line_number":1458,"context_line":"            except Exception:"},{"line_number":1459,"context_line":"                continue"},{"line_number":1460,"context_line":"        return result"},{"line_number":1461,"context_line":""},{"line_number":1462,"context_line":"    def build_pivot_ranges(self):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_064e5a96","line":1459,"updated":"2017-03-28 00:55:04.000000000","message":"Oh man, I\u0027ve not done VB in a while! Nothing like a good old\n\n On Error Resume Next\n\nFWIW, no tests fail if I apply http://paste.openstack.org/show/604386/ to drop the try...except","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1470,"context_line":"        try:"},{"line_number":1471,"context_line":"            offset \u003d int(shard_container_size) // 2"},{"line_number":1472,"context_line":""},{"line_number":1473,"context_line":"            sql \u003d \u0027SELECT name FROM object WHERE deleted\u003d0 \u0027"},{"line_number":1474,"context_line":"            if last_upper:"},{"line_number":1475,"context_line":"                sql +\u003d \"AND name \u003e \u0027%s\u0027 \" % last_upper"},{"line_number":1476,"context_line":"            sql +\u003d \"ORDER BY name LIMIT 1 OFFSET %d\" % offset"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c11f3c59","line":1473,"updated":"2017-03-28 00:55:04.000000000","message":"Do we need to check the DB version and sometimes use\n\n WHERE +deleted\u003d0\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1472,"context_line":""},{"line_number":1473,"context_line":"            sql \u003d \u0027SELECT name FROM object WHERE deleted\u003d0 \u0027"},{"line_number":1474,"context_line":"            if last_upper:"},{"line_number":1475,"context_line":"                sql +\u003d \"AND name \u003e \u0027%s\u0027 \" % last_upper"},{"line_number":1476,"context_line":"            sql +\u003d \"ORDER BY name LIMIT 1 OFFSET %d\" % offset"},{"line_number":1477,"context_line":"            data \u003d conn.execute(sql)"},{"line_number":1478,"context_line":"            if data:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a1ce78b1","line":1475,"range":{"start_line":1475,"start_character":35,"end_line":1475,"end_character":39},"updated":"2017-03-28 00:55:04.000000000","message":"Nope, use a ?\n\nIf last_upper happens to have a quote in it, we\u0027re in for some trouble.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1475,"context_line":"                sql +\u003d \"AND name \u003e \u0027%s\u0027 \" % last_upper"},{"line_number":1476,"context_line":"            sql +\u003d \"ORDER BY name LIMIT 1 OFFSET %d\" % offset"},{"line_number":1477,"context_line":"            data \u003d conn.execute(sql)"},{"line_number":1478,"context_line":"            if data:"},{"line_number":1479,"context_line":"                data \u003d data.fetchone()"},{"line_number":1480,"context_line":"                if not data:"},{"line_number":1481,"context_line":"                    return None"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_e131205c","line":1478,"updated":"2017-03-28 00:55:04.000000000","message":"Can execute() *not* return a cursor? I expected that it would always either give you a cursor or raise some flavor of sqlite3.Error...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1478,"context_line":"            if data:"},{"line_number":1479,"context_line":"                data \u003d data.fetchone()"},{"line_number":1480,"context_line":"                if not data:"},{"line_number":1481,"context_line":"                    return None"},{"line_number":1482,"context_line":"                return data[\u0027name\u0027]"},{"line_number":1483,"context_line":"            else:"},{"line_number":1484,"context_line":"                return \u0027\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_61263018","line":1481,"updated":"2017-03-28 00:55:04.000000000","message":"So we use None to indicate we\u0027ve walked the whole DB...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1483,"context_line":"            else:"},{"line_number":1484,"context_line":"                return \u0027\u0027"},{"line_number":1485,"context_line":"        except Exception:"},{"line_number":1486,"context_line":"            return \u0027\u0027"},{"line_number":1487,"context_line":""},{"line_number":1488,"context_line":"    def get_next_pivot_point(self, shard_container_size, last_upper\u003dNone,"},{"line_number":1489,"context_line":"                             connection\u003dNone):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_816ed42c","line":1486,"updated":"2017-03-28 00:55:04.000000000","message":"...and \u0027\u0027 to indicate an error? Why not raise an exception?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1489,"context_line":"                             connection\u003dNone):"},{"line_number":1490,"context_line":"        \"\"\""},{"line_number":1491,"context_line":"        Finds the middle entry of the table that could be used as a pivot"},{"line_number":1492,"context_line":"        point when sharding. It finds the middle object and returns it."},{"line_number":1493,"context_line":""},{"line_number":1494,"context_line":"        If there is an error or no objects in the container it will return an"},{"line_number":1495,"context_line":"        emply string (\u0027\u0027)."}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4157ac27","line":1492,"updated":"2017-03-28 00:55:04.000000000","message":"Not really... maybe something more like\n\n\u003e Finds the next entry in the objects table to use as a pivot point when sharding.\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1491,"context_line":"        Finds the middle entry of the table that could be used as a pivot"},{"line_number":1492,"context_line":"        point when sharding. It finds the middle object and returns it."},{"line_number":1493,"context_line":""},{"line_number":1494,"context_line":"        If there is an error or no objects in the container it will return an"},{"line_number":1495,"context_line":"        emply string (\u0027\u0027)."},{"line_number":1496,"context_line":""},{"line_number":1497,"context_line":"        :return: The middle object in the container."}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_81b37498","line":1494,"range":{"start_line":1494,"start_character":29,"end_line":1494,"end_character":59},"updated":"2017-03-28 00:55:04.000000000","message":"Wait, really? My read on the above made me think this would have us return None...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1509,"context_line":"                    return self._get_next_pivot_point(shard_container_size,"},{"line_number":1510,"context_line":"                                                      last_upper, conn)"},{"line_number":1511,"context_line":"            finally:"},{"line_number":1512,"context_line":"                self._create_connection()"},{"line_number":1513,"context_line":""},{"line_number":1514,"context_line":"    def is_shrinking(self):"},{"line_number":1515,"context_line":"        return self.metadata.get(\u0027X-Container-Sysmeta-Shard-Merge\u0027) or \\"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a15e7873","line":1512,"updated":"2017-03-28 00:55:04.000000000","message":"That looks a little funny...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1528,"context_line":"                res[\u0027merge\u0027] \u003d shard_merge[0]"},{"line_number":1529,"context_line":"        return res"},{"line_number":1530,"context_line":""},{"line_number":1531,"context_line":"    def is_root_container(self):"},{"line_number":1532,"context_line":"        \"\"\" Is this container a root container"},{"line_number":1533,"context_line":""},{"line_number":1534,"context_line":"        Any container that doesn\u0027t have a root container associated with"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_8c296be7","line":1531,"updated":"2017-03-28 00:55:04.000000000","message":"This is (seems?) cheap -- worth a @property?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1541,"context_line":"    def sharding_lock(self):"},{"line_number":1542,"context_line":"        lockpath \u003d \u0027%s/.sharding\u0027 % self.db_dir"},{"line_number":1543,"context_line":"        try:"},{"line_number":1544,"context_line":"            fd \u003d os.open(lockpath, os.O_WRONLY | os.O_CREAT)"},{"line_number":1545,"context_line":"            yield fd"},{"line_number":1546,"context_line":"        finally:"},{"line_number":1547,"context_line":"            os.unlink(lockpath)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c7062c1d","line":1544,"updated":"2017-03-28 00:55:04.000000000","message":"No os.O_EXCL? So we can do something like\n\n with broker.sharding_lock():\n     some_stuff()\n     with broker.sharding_lock():\n         some_other_stuff()\n     yet_more_stuff()\n\n? Is that intentional and/or necessary?\n\n\u0027Cause I\u0027m a bit concerned that the .sharding file will have been unlinked before we get to yet_more_stuff :-/\n\nShould we be using fcntl.flock like we do in lock_path and lock_file? Come to think of it, should this be something more like\n\n def sharding_lock(self):\n     try:\n         with swift.common.utils.lock_file(\u0027%s/.sharding\u0027 % self.db_dir):\n             self.has_sharding_lock \u003d True\n             yield\n     finally:\n         self.has_sharding_lock \u003d False\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1544,"context_line":"            fd \u003d os.open(lockpath, os.O_WRONLY | os.O_CREAT)"},{"line_number":1545,"context_line":"            yield fd"},{"line_number":1546,"context_line":"        finally:"},{"line_number":1547,"context_line":"            os.unlink(lockpath)"},{"line_number":1548,"context_line":""},{"line_number":1549,"context_line":"    def has_sharding_lock(self):"},{"line_number":1550,"context_line":"        lockpath \u003d \u0027%s/.sharding\u0027 % self.db_dir"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_6c8747b7","line":1547,"updated":"2017-03-28 00:55:04.000000000","message":"Assuming we *don\u0027t* move to lock_file, should this also include a\n\n os.close(fd)\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1548,"context_line":""},{"line_number":1549,"context_line":"    def has_sharding_lock(self):"},{"line_number":1550,"context_line":"        lockpath \u003d \u0027%s/.sharding\u0027 % self.db_dir"},{"line_number":1551,"context_line":"        return os.path.exists(lockpath)"},{"line_number":1552,"context_line":""},{"line_number":1553,"context_line":"    def set_sharding_state(self):"},{"line_number":1554,"context_line":"        db_state \u003d self.get_db_state()"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_2745d843","line":1551,"updated":"2017-03-28 00:55:04.000000000","message":"So this applies to the *path* regardless of *process* or *thread*? Would we ever need to worry about multiple processes or threads requesting the sharding lock at the same time?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1562,"context_line":"        # playing with the wrong database while setting up the pivot database"},{"line_number":1563,"context_line":"        if not self.conn:"},{"line_number":1564,"context_line":"            with self.get():"},{"line_number":1565,"context_line":"                pass"},{"line_number":1566,"context_line":""},{"line_number":1567,"context_line":"        # For this initial version, we\u0027ll create a new container along side."},{"line_number":1568,"context_line":"        # Later we will remove parts so the pivot DB only has what it really"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_07ebf412","line":1565,"updated":"2017-03-28 00:55:04.000000000","message":"Thanks for the comment! Otherwise I\u0027d be so confused about what was going on here...\n\nKinda curious that we don\u0027t just do\n\n self._create_connection(self._db_file)\n\n...like we do in get_items_since, though.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1576,"context_line":""},{"line_number":1577,"context_line":"        # If there are pivot_ranges defined.. which can happen when the scanner"},{"line_number":1578,"context_line":"        # node finds the first pivot then replicates out to the others who"},{"line_number":1579,"context_line":"        # are still in the UNSHARDED state."},{"line_number":1580,"context_line":"        pivot_ranges \u003d self.get_pivot_ranges()"},{"line_number":1581,"context_line":"        if pivot_ranges:"},{"line_number":1582,"context_line":"            pivot_ranges \u003d \\"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_87dee46a","line":1579,"updated":"2017-03-28 00:55:04.000000000","message":"Hey, that sounds like a probe test!","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1600,"context_line":"                      \"(ROWID, name, created_at, size, content_type, etag) \" \\"},{"line_number":1601,"context_line":"                    \"values (?, \u0027pivted_remove\u0027, ?, 0, \u0027\u0027, ?)\""},{"line_number":1602,"context_line":"                conn.execute(sql, (max_row, Timestamp(time.time()).internal,"},{"line_number":1603,"context_line":"                                   \u002768b329da9893e34099c7d8ad5cb9c940\u0027))"},{"line_number":1604,"context_line":"                conn.execute(\u0027DELETE FROM object WHERE ROWID \u003d %d\u0027 % max_row)"},{"line_number":1605,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1606,"context_line":"                self.logger.error(_(\u0027Failed to set the ROWID of the pivot \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_47521ce0","line":1603,"range":{"start_line":1603,"start_character":36,"end_line":1603,"end_character":68},"updated":"2017-03-28 00:55:04.000000000","message":"nit: md5(\u0027\\n\u0027)? Not md5(\u0027\u0027)? *shrug*","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1601,"context_line":"                    \"values (?, \u0027pivted_remove\u0027, ?, 0, \u0027\u0027, ?)\""},{"line_number":1602,"context_line":"                conn.execute(sql, (max_row, Timestamp(time.time()).internal,"},{"line_number":1603,"context_line":"                                   \u002768b329da9893e34099c7d8ad5cb9c940\u0027))"},{"line_number":1604,"context_line":"                conn.execute(\u0027DELETE FROM object WHERE ROWID \u003d %d\u0027 % max_row)"},{"line_number":1605,"context_line":"            except sqlite3.OperationalError as err:"},{"line_number":1606,"context_line":"                self.logger.error(_(\u0027Failed to set the ROWID of the pivot \u0027"},{"line_number":1607,"context_line":"                                    \u0027database for %s/%s: %s\u0027), self.account,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a72a0829","line":1604,"updated":"2017-03-28 00:55:04.000000000","message":"Should there be a conn.commit()? I was kinda expecting one... especially since the context manager does a rollback() when we end the with-block...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1611,"context_line":"        # will be in use."},{"line_number":1612,"context_line":"        self.conn \u003d None"},{"line_number":1613,"context_line":""},{"line_number":1614,"context_line":"        return True"},{"line_number":1615,"context_line":""},{"line_number":1616,"context_line":"    def set_sharded_state(self):"},{"line_number":1617,"context_line":"        db_state \u003d self.get_db_state()"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_271cb8bf","line":1614,"updated":"2017-03-28 00:55:04.000000000","message":"I\u0027m nervous about all the operations going on here -- what happens when some other process gets a broker midway through all this setup? I guess they see we\u0027re in the sharding state, but they won\u0027t necessarily be able to get the right metadata from the pivot db?\n\nWhat happens when the process dies partway through? Will that setup ever get completed?\n\nWould it be worth doing a\n\n with self.sharding_lock():\n\nonce we\u0027ve verified the db state? Or maybe getting a temp file, using *that* to do all the setup, then doing an atomic rename to the pivot db location? Maybe both?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1621,"context_line":"                             self.container, DB_STATE[db_state])"},{"line_number":1622,"context_line":"            return False"},{"line_number":1623,"context_line":""},{"line_number":1624,"context_line":"        # TODO add some checks to see if we are ready to unlink the old db"},{"line_number":1625,"context_line":"        os.unlink(self._db_file)"},{"line_number":1626,"context_line":""},{"line_number":1627,"context_line":"        # now reset the connection so next time the coeect database will be used"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_07d5341c","line":1624,"updated":"2017-03-28 00:55:04.000000000","message":"What should these checks entail?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1622,"context_line":"            return False"},{"line_number":1623,"context_line":""},{"line_number":1624,"context_line":"        # TODO add some checks to see if we are ready to unlink the old db"},{"line_number":1625,"context_line":"        os.unlink(self._db_file)"},{"line_number":1626,"context_line":""},{"line_number":1627,"context_line":"        # now reset the connection so next time the coeect database will be used"},{"line_number":1628,"context_line":"        self.conn \u003d None"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c7b3cc09","line":1625,"updated":"2017-03-28 00:55:04.000000000","message":"Should this tolerate an ENOENT?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1624,"context_line":"        # TODO add some checks to see if we are ready to unlink the old db"},{"line_number":1625,"context_line":"        os.unlink(self._db_file)"},{"line_number":1626,"context_line":""},{"line_number":1627,"context_line":"        # now reset the connection so next time the coeect database will be used"},{"line_number":1628,"context_line":"        self.conn \u003d None"},{"line_number":1629,"context_line":""},{"line_number":1630,"context_line":"    def get_brokers(self):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a7dfa838","line":1627,"range":{"start_line":1627,"start_character":52,"end_line":1627,"end_character":58},"updated":"2017-03-28 00:55:04.000000000","message":"nit: correct","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1625,"context_line":"        os.unlink(self._db_file)"},{"line_number":1626,"context_line":""},{"line_number":1627,"context_line":"        # now reset the connection so next time the coeect database will be used"},{"line_number":1628,"context_line":"        self.conn \u003d None"},{"line_number":1629,"context_line":""},{"line_number":1630,"context_line":"    def get_brokers(self):"},{"line_number":1631,"context_line":"        brokers \u003d []"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_2716f8f2","line":1628,"updated":"2017-03-28 00:55:04.000000000","message":"No\n\n return True\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1642,"context_line":"            brokers.append(ContainerBroker("},{"line_number":1643,"context_line":"                self._pivot_db_file, self.timeout, self.logger, self.account,"},{"line_number":1644,"context_line":"                self.container, self.pending_timeout, self.stale_reads_ok))"},{"line_number":1645,"context_line":"        return brokers"},{"line_number":1646,"context_line":""},{"line_number":1647,"context_line":"    def get_items_since(self, start, count):"},{"line_number":1648,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_07c9b424","line":1645,"updated":"2017-03-28 00:55:04.000000000","message":"I feel like one of these ought to just be self, but I\u0027m not sure which :-/","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1661,"context_line":"            if old_max_row \u003e start:"},{"line_number":1662,"context_line":"                self._create_connection(self._pivot_db_file)"},{"line_number":1663,"context_line":"                return \\"},{"line_number":1664,"context_line":"                    super(ContainerBroker, self).get_items_since(start, count)"},{"line_number":1665,"context_line":""},{"line_number":1666,"context_line":"            objs \u003d super(ContainerBroker, self).get_items_since(start, count)"},{"line_number":1667,"context_line":"            if objs[-1][\u0027ROWID\u0027] \u003c old_max_row or len(objs) \u003d\u003d count:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_074e5496","line":1664,"updated":"2017-03-28 00:55:04.000000000","message":"Wait, if old_max_row is *greater* than start, can we just return rows from the pivot db? I thought that meant there *are* items in the read-only db that we have to merge in...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1664,"context_line":"                    super(ContainerBroker, self).get_items_since(start, count)"},{"line_number":1665,"context_line":""},{"line_number":1666,"context_line":"            objs \u003d super(ContainerBroker, self).get_items_since(start, count)"},{"line_number":1667,"context_line":"            if objs[-1][\u0027ROWID\u0027] \u003c old_max_row or len(objs) \u003d\u003d count:"},{"line_number":1668,"context_line":"                return objs"},{"line_number":1669,"context_line":""},{"line_number":1670,"context_line":"            self._create_connection(self._pivot_db_file)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_675e4045","line":1667,"range":{"start_line":1667,"start_character":50,"end_line":1667,"end_character":68},"updated":"2017-03-28 00:55:04.000000000","message":"Is this check necessary? I was expecting the other half to be sufficient.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"811bef594da5eb350d2dc1fe45507c7c57faadbb","unresolved":false,"context_lines":[{"line_number":1664,"context_line":"                    super(ContainerBroker, self).get_items_since(start, count)"},{"line_number":1665,"context_line":""},{"line_number":1666,"context_line":"            objs \u003d super(ContainerBroker, self).get_items_since(start, count)"},{"line_number":1667,"context_line":"            if objs[-1][\u0027ROWID\u0027] \u003c old_max_row or len(objs) \u003d\u003d count:"},{"line_number":1668,"context_line":"                return objs"},{"line_number":1669,"context_line":""},{"line_number":1670,"context_line":"            self._create_connection(self._pivot_db_file)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_25a92623","line":1667,"range":{"start_line":1667,"start_character":50,"end_line":1667,"end_character":68},"in_reply_to":"ffe62b97_675e4045","updated":"2017-03-28 18:32:08.000000000","message":"Nope! I was wrong. The *other* check isn\u0027t necessary, and *this* one is all we need. Or maybe they\u0027re equivalent?\n\nWhat else might cause objs[-1][\u0027ROWID\u0027] \u003c old_max_row ... having old_row_max get reclaimed would do it -- I guess that would need to happen *before* the pivot DB gets created. In that case it seems like we ought to go to the pivot db to try to get up to count though, yeah?\n\nAt any rate, the first check can cause an IndexError if start \u003d\u003d old_max_row. (At least, it does after my swapping to less-than above -- I suppose that\u0027s a sign it should be less-than-or-equal-to.)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":1668,"context_line":"                return objs"},{"line_number":1669,"context_line":""},{"line_number":1670,"context_line":"            self._create_connection(self._pivot_db_file)"},{"line_number":1671,"context_line":"            objs.extend(super(ContainerBroker, self).get_items_since("},{"line_number":1672,"context_line":"                old_max_row, count - len(objs)))"},{"line_number":1673,"context_line":"            return objs"},{"line_number":1674,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_071c1476","line":1671,"updated":"2017-03-28 00:55:04.000000000","message":"Right, there shouldn\u0027t be any overlap, so we\u0027ve got a nice, simple extend().","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":328,"context_line":"        elif not db_exists and pivot_exists:"},{"line_number":329,"context_line":"            return DB_STATE_SHARDED"},{"line_number":330,"context_line":"        else:"},{"line_number":331,"context_line":"            # Both Pivot db and db doesn\u0027t doesn\u0027t exist."},{"line_number":332,"context_line":"            return DB_STATE_NOTFOUND"},{"line_number":333,"context_line":""},{"line_number":334,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_49b3300b","line":331,"updated":"2017-04-28 01:38:47.000000000","message":"\u003e Neither db nor pivot db exists","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":447,"context_line":"            VALUES (?)"},{"line_number":448,"context_line":"        \"\"\", (storage_policy_index,))"},{"line_number":449,"context_line":""},{"line_number":450,"context_line":"    def create_pivot_ranges_table(self, conn):"},{"line_number":451,"context_line":"        \"\"\""},{"line_number":452,"context_line":"        Create the pivot_ranges table which is specific to the container DB."},{"line_number":453,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_56b2ffe7","line":450,"updated":"2017-04-28 01:38:47.000000000","message":"Still kinda thinking we\u0027ll want (or at least ought to think about) an index. deleted, name seems reasonable? Or even deleted, lower or something -- how many shards are we expecting again? 100s-1,000s, but not 1,000,000s?\n\nStill might be better to have some way to get pivots in order without having a table scan, though.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a49daff90fbb259e5bbeb82b105ecd87e5580f3c","unresolved":false,"context_lines":[{"line_number":942,"context_line":"                elif prefix:"},{"line_number":943,"context_line":"                    query +\u003d \u0027 name \u003e\u003d ? AND\u0027"},{"line_number":944,"context_line":"                    query_args.append(prefix)"},{"line_number":945,"context_line":"                if include_deleted and query.endswith(\u0027 AND\u0027):"},{"line_number":946,"context_line":"                    query \u003d query[:-4]"},{"line_number":947,"context_line":"                else:"},{"line_number":948,"context_line":"                    if self.get_db_version(conn) \u003c 1:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_01fe3e6a","line":945,"range":{"start_line":945,"start_character":35,"end_line":945,"end_character":61},"updated":"2017-05-11 19:51:11.000000000","message":"This part\u0027s always true, yeah?\n\nThat explains why we can ignore my previous concern about the\n\n include_deleted and not query.endswith(\u0027 AND\u0027)\n\ncase; it *can\u0027t* come up.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"1509be99dce9f6bc16346d41a0a2b71fc9c9fc4c","unresolved":false,"context_lines":[{"line_number":1079,"context_line":"        for item in item_list:"},{"line_number":1080,"context_line":"            item.setdefault(\u0027record_type\u0027, RECORD_TYPE_OBJECT)"},{"line_number":1081,"context_line":"            cols \u003d [\u0027name\u0027]"},{"line_number":1082,"context_line":"            if item.get(\u0027lower\u0027):"},{"line_number":1083,"context_line":"                cols +\u003d [\u0027lower\u0027, \u0027upper\u0027]"},{"line_number":1084,"context_line":"            for col in cols:"},{"line_number":1085,"context_line":"                if isinstance(item[col], six.text_type):"},{"line_number":1086,"context_line":"                    item[col] \u003d item[col].encode(\u0027utf-8\u0027)"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_4f448247","line":1083,"range":{"start_line":1082,"start_character":12,"end_line":1083,"end_character":42},"updated":"2017-09-07 01:42:44.000000000","message":"I noticed a potential for a bug here, if lower is None or \u0027\u0027 (so the first pivot) this wont check that the upper is utf-8 encoded so:\n\n  if item.get(\u0027lower\u0027):   \n      cols.append(\u0027lower\u0027)\n  if item.get(\u0027upper\u0027):   \n      cols.append(\u0027upper\u0027)","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"}],"swift/container/replicator.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":268,"context_line":""},{"line_number":269,"context_line":"    def _in_sync(self, rinfo, info, broker, local_sync):"},{"line_number":270,"context_line":"        if len(broker.get_pivot_ranges()) \u003e 0:"},{"line_number":271,"context_line":"            return False"},{"line_number":272,"context_line":""},{"line_number":273,"context_line":"        return super(ContainerReplicator, self)._in_sync("},{"line_number":274,"context_line":"            rinfo, info, broker, local_sync)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_62ff99aa","line":271,"updated":"2017-03-25 02:28:22.000000000","message":"So we always assume that sharding/sharded DBs are not in-sync? Probably a safe assumption... we *need* to talk and get each other\u0027s actually pivot ranges before we can decide, I guess?\n\nHow do we decide in the unsharded state? There must be some lightweight check we do rather than comparing all objects... can we emulate that for pivot ranges? Or should we just punt on that work until we actually have pivot DBs large enough for it to be a problem?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":281,"context_line":""},{"line_number":282,"context_line":"    def _can_push(self, info, rinfo):"},{"line_number":283,"context_line":"        def is_sharded(inf):"},{"line_number":284,"context_line":"            return inf.get(\u0027db_state\u0027) and inf.get(\u0027db_state\u0027) \u003d\u003d \\"},{"line_number":285,"context_line":"                    DB_STATE_SHARDED"},{"line_number":286,"context_line":""},{"line_number":287,"context_line":"        def is_sharding(inf):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_821bbd19","line":284,"updated":"2017-03-25 02:28:22.000000000","message":"Maybe cleaner as\n\n return info.get(\u0027db_state\u0027, DB_STATE_UNSHARDED) \u003d\u003d DB_STATE_SHARDED\n\n?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8ae18dec75f7bf21b917a55585ffc8805b600035","unresolved":false,"context_lines":[{"line_number":289,"context_line":""},{"line_number":290,"context_line":"        if is_sharded(rinfo) and not is_sharded(info):"},{"line_number":291,"context_line":"            return False"},{"line_number":292,"context_line":"        if is_sharding(info) or is_sharding(rinfo):"},{"line_number":293,"context_line":"            return False"},{"line_number":294,"context_line":"        return True"},{"line_number":295,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"5f2577a7_baf00d69","line":292,"updated":"2017-09-12 19:55:40.000000000","message":"...\n\n or is_sharded(rinfo)\n\nMaybe better as\n\n if is_sharded(info) and is_sharded(rinfo):\n     return True\n if is_unsharded(info) and is_unsharded(rinfo):\n     return True\n return False\n\n?","commit_id":"ddd4465614e6310c500f840533621a6fab00c25d"}],"swift/container/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":102,"context_line":"            os.path.join(swift_dir, \u0027container-sync-realms.conf\u0027),"},{"line_number":103,"context_line":"            self.logger)"},{"line_number":104,"context_line":"        self.ips \u003d whataremyips()"},{"line_number":105,"context_line":"        self.port \u003d int(conf.get(\u0027bind_port\u0027, 6201))"},{"line_number":106,"context_line":"        #: The list of hosts we\u0027re allowed to send syncs to. This can be"},{"line_number":107,"context_line":"        #: overridden by data in self.realms_conf"},{"line_number":108,"context_line":"        self.allowed_sync_hosts \u003d ["}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_62ed59ee","line":105,"updated":"2017-03-25 02:28:22.000000000","message":"Huh... these weren\u0027t here before? How did we... I guess that was all handled for us in swift/common/wsgi.py?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":283,"context_line":"                       \u0027X-Redirect-Timestamp\u0027:"},{"line_number":284,"context_line":"                           containing_range.timestamp}"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"            return HTTPMovedPermanently(headers\u003dheaders)"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"        except Exception:"},{"line_number":289,"context_line":"            return HTTPInternalServerError()"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_889d69bd","line":286,"range":{"start_line":286,"start_character":19,"end_line":286,"end_character":39},"updated":"2017-03-30 00:23:49.000000000","message":"Maybe better as a 308? https://tools.ietf.org/html/rfc7538\n\nI guess \"permanent\" is fairly appropriate? It should take a *while* to have that shard no longer be the correct location for it, right? Just debating about 307... but I suppose it doesn\u0027t really matter much since it\u0027s all internal to the cluster anyway. And there\u0027s certainly an argument to have us be able to see 307 and think \"symlinks\" or 308 and think \"sharding\".\n\noff-topic: I have no idea where this came from https://github.com/openstack/swift/blob/master/swift/common/http.py#L105","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":286,"context_line":"            return HTTPMovedPermanently(headers\u003dheaders)"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"        except Exception:"},{"line_number":289,"context_line":"            return HTTPInternalServerError()"},{"line_number":290,"context_line":""},{"line_number":291,"context_line":"    @public"},{"line_number":292,"context_line":"    @timing_stats()"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a2912155","line":289,"updated":"2017-03-25 02:28:22.000000000","message":"Should we at least do some sort of logging?\n\nDo we have any idea what sort of errors might occur?\n\nI don\u0027t especially like that if I\u0027m changing this and typo\n\n if borker._is_root_cotnainer():\n\nthis will 500 with no tracebacks :-/","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a91c4c1e898a9e8c56b1862a120e86b1e6196953","unresolved":false,"context_lines":[{"line_number":334,"context_line":""},{"line_number":335,"context_line":"            elif len(broker.get_pivot_ranges()) \u003e 0:"},{"line_number":336,"context_line":"                # cannot put to a root shard container, find actual container"},{"line_number":337,"context_line":"                return self._find_shard_location(req, broker, obj)"},{"line_number":338,"context_line":"            else:"},{"line_number":339,"context_line":"                broker.delete_object(obj, req.headers.get(\u0027x-timestamp\u0027),"},{"line_number":340,"context_line":"                                     obj_policy_index)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_82acdd8c","line":337,"updated":"2017-03-25 02:28:22.000000000","message":"Can this return None? We\u0027ve got a guard for that sort of thing below in PUT...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":431,"context_line":"            elif len(broker.get_pivot_ranges()) \u003e 0:"},{"line_number":432,"context_line":"                # cannot put to a root shard container, find actual container"},{"line_number":433,"context_line":"                res \u003d self._find_shard_location(req, broker, obj)"},{"line_number":434,"context_line":"                if res:"},{"line_number":435,"context_line":"                    return res"},{"line_number":436,"context_line":"            else:"},{"line_number":437,"context_line":"                broker.put_object(obj, req_timestamp.internal,"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_a42316a7","line":434,"range":{"start_line":434,"start_character":16,"end_line":434,"end_character":23},"updated":"2017-09-07 00:17:46.000000000","message":"else...?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":437,"context_line":"                broker.put_object(obj, req_timestamp.internal,"},{"line_number":438,"context_line":"                                  int(req.headers[\u0027x-size\u0027]),"},{"line_number":439,"context_line":"                                  req.headers.get(\u0027x-content-type\u0027),"},{"line_number":440,"context_line":"                                  req.headers.get(\u0027x-etag\u0027), 0,"},{"line_number":441,"context_line":"                                  obj_policy_index,"},{"line_number":442,"context_line":"                                  req.headers.get(\u0027x-content-type-timestamp\u0027),"},{"line_number":443,"context_line":"                                  req.headers.get(\u0027x-meta-timestamp\u0027))"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_44191278","line":440,"range":{"start_line":440,"start_character":46,"end_line":440,"end_character":49},"updated":"2017-09-07 00:17:46.000000000","message":"Can these ever actually be missing?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f6594e2ab7d6783d7aee39a095c1eee263b045af","unresolved":false,"context_lines":[{"line_number":547,"context_line":"        old_b, pivot_b \u003d broker.get_brokers()"},{"line_number":548,"context_line":"        info, is_deleted \u003d old_b.get_info_is_deleted()"},{"line_number":549,"context_line":"        headers.update({\u0027X-Container-Object-Count\u0027: info[\u0027object_count\u0027],"},{"line_number":550,"context_line":"                        \u0027X-Container-Bytes-Used\u0027: info[\u0027bytes_used\u0027]})"},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"        old_items \u003d old_b.list_objects_iter("},{"line_number":553,"context_line":"            limit, marker, end_marker, prefix, delimiter, path,"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_91c0777e","line":550,"updated":"2017-09-06 00:55:01.000000000","message":"Once we\u0027ve sharded (and therefor unlinked the non-pivot db), headers is always an empty dict coming into this function, yeah? So we only ever get these two items...","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0e3ee9cb7be1642fb46fad96a411e5208eb6cbdf","unresolved":false,"context_lines":[{"line_number":547,"context_line":"        old_b, pivot_b \u003d broker.get_brokers()"},{"line_number":548,"context_line":"        info, is_deleted \u003d old_b.get_info_is_deleted()"},{"line_number":549,"context_line":"        headers.update({\u0027X-Container-Object-Count\u0027: info[\u0027object_count\u0027],"},{"line_number":550,"context_line":"                        \u0027X-Container-Bytes-Used\u0027: info[\u0027bytes_used\u0027]})"},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"        old_items \u003d old_b.list_objects_iter("},{"line_number":553,"context_line":"            limit, marker, end_marker, prefix, delimiter, path,"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_941f456f","line":550,"in_reply_to":"7f287b81_91c0777e","updated":"2017-09-06 02:05:20.000000000","message":"Its been a while, so have to re-look as some of this code, but this method is only called if in SHARDING state, so it _should_n\u0027t be deleted yet. Tho maybe there would be a possibility? have you seen this?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":547,"context_line":"        old_b, pivot_b \u003d broker.get_brokers()"},{"line_number":548,"context_line":"        info, is_deleted \u003d old_b.get_info_is_deleted()"},{"line_number":549,"context_line":"        headers.update({\u0027X-Container-Object-Count\u0027: info[\u0027object_count\u0027],"},{"line_number":550,"context_line":"                        \u0027X-Container-Bytes-Used\u0027: info[\u0027bytes_used\u0027]})"},{"line_number":551,"context_line":""},{"line_number":552,"context_line":"        old_items \u003d old_b.list_objects_iter("},{"line_number":553,"context_line":"            limit, marker, end_marker, prefix, delimiter, path,"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_3b16204b","line":550,"in_reply_to":"7f287b81_941f456f","updated":"2017-09-07 00:17:46.000000000","message":"Ah, yeah... I must not have been paying enough attention to my tracebacks -- I\u0027m pretty sure I was hitting trouble in the proxy, not the container.","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":608,"context_line":"            return HTTPNotFound(request\u003dreq, headers\u003dresp_headers)"},{"line_number":609,"context_line":"        include_deleted \u003d False"},{"line_number":610,"context_line":"        if items and items.lower() \u003d\u003d \"pivot\":"},{"line_number":611,"context_line":"            container_list \u003d broker.build_pivot_ranges()"},{"line_number":612,"context_line":"            if obj:"},{"line_number":613,"context_line":"                container_list \u003d [find_pivot_range(obj, container_list)]"},{"line_number":614,"context_line":"            elif marker or end_marker:"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_79c27036","line":611,"updated":"2017-09-07 00:17:46.000000000","message":"Hmm... so container_list is a list of honest-to-God PivotRanges, not dicts?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"}],"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":113,"context_line":"        self.vm_test_mode \u003d config_true_value(conf.get(\u0027vm_test_mode\u0027, \u0027no\u0027))"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"        self.shard_shrink_point \u003d \\"},{"line_number":116,"context_line":"            float(conf.get(\u0027shard_shrink_point\u0027, 25)) / 100.0"},{"line_number":117,"context_line":"        self.shrink_merge_point \u003d \\"},{"line_number":118,"context_line":"            float(conf.get(\u0027shard_shrink_merge_point\u0027, 75)) / 100.0"},{"line_number":119,"context_line":"        self.split_size \u003d SHARD_CONTAINER_SIZE // 2"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_0b97a13a","line":116,"range":{"start_line":116,"start_character":49,"end_line":116,"end_character":51},"updated":"2017-03-21 23:30:26.000000000","message":"This seems to disagree with the default in the conf?","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":113,"context_line":"        self.vm_test_mode \u003d config_true_value(conf.get(\u0027vm_test_mode\u0027, \u0027no\u0027))"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"        self.shard_shrink_point \u003d \\"},{"line_number":116,"context_line":"            float(conf.get(\u0027shard_shrink_point\u0027, 25)) / 100.0"},{"line_number":117,"context_line":"        self.shrink_merge_point \u003d \\"},{"line_number":118,"context_line":"            float(conf.get(\u0027shard_shrink_merge_point\u0027, 75)) / 100.0"},{"line_number":119,"context_line":"        self.split_size \u003d SHARD_CONTAINER_SIZE // 2"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_74562d2b","line":116,"range":{"start_line":116,"start_character":49,"end_line":116,"end_character":51},"in_reply_to":"1a1ced50_0b97a13a","updated":"2017-03-22 01:13:03.000000000","message":"yup, I guess I changed it in testing.\n\nThere is a trello board item to pick better (and consistant) defaults","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":115,"context_line":"        self.shard_shrink_point \u003d \\"},{"line_number":116,"context_line":"            float(conf.get(\u0027shard_shrink_point\u0027, 25)) / 100.0"},{"line_number":117,"context_line":"        self.shrink_merge_point \u003d \\"},{"line_number":118,"context_line":"            float(conf.get(\u0027shard_shrink_merge_point\u0027, 75)) / 100.0"},{"line_number":119,"context_line":"        self.split_size \u003d SHARD_CONTAINER_SIZE // 2"},{"line_number":120,"context_line":"        self.cpool \u003d GreenAsyncPile(self.cpool)"},{"line_number":121,"context_line":"        self.scanner_batch_size \u003d int(conf.get(\u0027shard_scanner_batch_size\u0027, 10))"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_8bab9174","line":118,"updated":"2017-03-21 23:30:26.000000000","message":"We should raise a ValueError or something if self.shrink_merge_point \u003e\u003d 1","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# The default internal client config body is to support upgrades without"},{"line_number":48,"context_line":"# requiring deployment of the new /etc/swift/internal-client.conf"},{"line_number":49,"context_line":"ic_conf_body \u003d \"\"\""},{"line_number":50,"context_line":"[DEFAULT]"},{"line_number":51,"context_line":"# swift_dir \u003d /etc/swift"},{"line_number":52,"context_line":"# user \u003d swift"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_8390f84b","line":49,"updated":"2017-03-28 20:05:12.000000000","message":"see comment below - I\u0027d prefer to not have this for a new service - upgrade is going to require adding config in order to use the sharder, so I think we can just require a IC conf file","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":105,"context_line":""},{"line_number":106,"context_line":"    def __init__(self, conf, logger\u003dNone, **kargs):"},{"line_number":107,"context_line":"        super(ContainerReplicator, self).__init__(conf, logger\u003dlogger)"},{"line_number":108,"context_line":"        self.conf \u003d conf"},{"line_number":109,"context_line":"        self.logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":110,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path,"},{"line_number":111,"context_line":"                                   \"container-sharder.recon\")"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_7e46df23","line":108,"updated":"2017-03-28 20:05:12.000000000","message":"this is set in the superclass __init__","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":106,"context_line":"    def __init__(self, conf, logger\u003dNone, **kargs):"},{"line_number":107,"context_line":"        super(ContainerReplicator, self).__init__(conf, logger\u003dlogger)"},{"line_number":108,"context_line":"        self.conf \u003d conf"},{"line_number":109,"context_line":"        self.logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":110,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path,"},{"line_number":111,"context_line":"                                   \"container-sharder.recon\")"},{"line_number":112,"context_line":"        self.vm_test_mode \u003d config_true_value(conf.get(\u0027vm_test_mode\u0027, \u0027no\u0027))"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_cec62f69","line":109,"updated":"2017-03-28 20:05:12.000000000","message":"could just pass the logger to superclass init method as an arg","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":107,"context_line":"        super(ContainerReplicator, self).__init__(conf, logger\u003dlogger)"},{"line_number":108,"context_line":"        self.conf \u003d conf"},{"line_number":109,"context_line":"        self.logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":110,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path,"},{"line_number":111,"context_line":"                                   \"container-sharder.recon\")"},{"line_number":112,"context_line":"        self.vm_test_mode \u003d config_true_value(conf.get(\u0027vm_test_mode\u0027, \u0027no\u0027))"},{"line_number":113,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_de284b35","line":110,"updated":"2017-03-28 20:05:12.000000000","message":"ok, so this is necessary to override the name set in superclass","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":109,"context_line":"        self.logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":110,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path,"},{"line_number":111,"context_line":"                                   \"container-sharder.recon\")"},{"line_number":112,"context_line":"        self.vm_test_mode \u003d config_true_value(conf.get(\u0027vm_test_mode\u0027, \u0027no\u0027))"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"        self.shard_shrink_point \u003d \\"},{"line_number":115,"context_line":"            float(conf.get(\u0027shard_shrink_point\u0027, 25)) / 100.0"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_aeeacbd4","line":112,"updated":"2017-03-28 20:05:12.000000000","message":"vm_test_mode is set in superclass init","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":113,"context_line":""},{"line_number":114,"context_line":"        self.shard_shrink_point \u003d \\"},{"line_number":115,"context_line":"            float(conf.get(\u0027shard_shrink_point\u0027, 25)) / 100.0"},{"line_number":116,"context_line":"        self.shrink_merge_point \u003d \\"},{"line_number":117,"context_line":"            float(conf.get(\u0027shard_shrink_merge_point\u0027, 75)) / 100.0"},{"line_number":118,"context_line":"        if self.shrink_merge_point \u003e 1:"},{"line_number":119,"context_line":"            raise ValueError(_(\"shard_shrink_merge_point can\u0027t be greater then\""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_91bdead9","line":116,"updated":"2017-03-28 20:05:12.000000000","message":"nit: this is the one attribute that does not match the name of the conf option","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":116,"context_line":"        self.shrink_merge_point \u003d \\"},{"line_number":117,"context_line":"            float(conf.get(\u0027shard_shrink_merge_point\u0027, 75)) / 100.0"},{"line_number":118,"context_line":"        if self.shrink_merge_point \u003e 1:"},{"line_number":119,"context_line":"            raise ValueError(_(\"shard_shrink_merge_point can\u0027t be greater then\""},{"line_number":120,"context_line":"                               \"100\"))"},{"line_number":121,"context_line":"        self.shard_container_size \u003d int(conf.get(\u0027shard_container_size\u0027,"},{"line_number":122,"context_line":"                                                 10000000))"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_5d59b38b","line":119,"updated":"2017-03-28 20:05:12.000000000","message":"I think we are no longer doing translations?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":126,"context_line":"        self.shard_batch_size \u003d int(conf.get(\u0027shard_batch_size\u0027, 2))"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"        # internal client"},{"line_number":129,"context_line":"        self.conn_timeout \u003d float(conf.get(\u0027conn_timeout\u0027, 5))"},{"line_number":130,"context_line":"        request_tries \u003d int(conf.get(\u0027request_tries\u0027) or 3)"},{"line_number":131,"context_line":"        internal_client_conf_path \u003d conf.get(\u0027internal_client_conf_path\u0027)"},{"line_number":132,"context_line":"        if not internal_client_conf_path:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4ef0dffd","line":129,"updated":"2017-03-28 20:05:12.000000000","message":"this is loaded in superclass init, but here you have different default of 5, but sample conf says default is 0.5\n\ncan this line just be deleted and use the default of 0.5?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":127,"context_line":""},{"line_number":128,"context_line":"        # internal client"},{"line_number":129,"context_line":"        self.conn_timeout \u003d float(conf.get(\u0027conn_timeout\u0027, 5))"},{"line_number":130,"context_line":"        request_tries \u003d int(conf.get(\u0027request_tries\u0027) or 3)"},{"line_number":131,"context_line":"        internal_client_conf_path \u003d conf.get(\u0027internal_client_conf_path\u0027)"},{"line_number":132,"context_line":"        if not internal_client_conf_path:"},{"line_number":133,"context_line":"            self.logger.warning("}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_8831375d","line":130,"updated":"2017-03-28 20:05:12.000000000","message":"these internal client options are not in the container-server.conf-sample file","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":135,"context_line":"                  \u0027defined. Using default configuration, See \u0027"},{"line_number":136,"context_line":"                  \u0027internal-client.conf-sample for options. NOTE: \u0027"},{"line_number":137,"context_line":"                  \u0027\"account_autocreate \u003d true\" is required.\u0027))"},{"line_number":138,"context_line":"            internal_client_conf \u003d ConfigString(ic_conf_body)"},{"line_number":139,"context_line":"        else:"},{"line_number":140,"context_line":"            internal_client_conf \u003d internal_client_conf_path"},{"line_number":141,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c3fcc0b6","line":138,"updated":"2017-03-28 20:05:12.000000000","message":"for the sharder, can we not just require and internal-client.conf file to be deployed? It seems this happened for the reconciler (which has no default config). For container sync the default IC config seems to have been included because there was a change to use IC with the *existing* container sync service, but here we are introducing a new service which requires config file changes anyway.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f0fcfe79c41d2a821064f754f5ef3cb135669d4","unresolved":false,"context_lines":[{"line_number":197,"context_line":"                object_count \u003d pivot.get(\u0027object_count\u0027) or 0"},{"line_number":198,"context_line":"                bytes_used \u003d pivot.get(\u0027bytes_used\u0027) or 0"},{"line_number":199,"context_line":"                meta_timestamp \u003d pivot.get(\u0027meta_timestamp\u0027) or None"},{"line_number":200,"context_line":"                ranges.append(PivotRange(pivot[\u0027name\u0027], created_at, lower,"},{"line_number":201,"context_line":"                                         upper, object_count, bytes_used,"},{"line_number":202,"context_line":"                                         meta_timestamp))"},{"line_number":203,"context_line":"        except ValueError:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3868d6b8","line":200,"updated":"2017-03-30 16:24:02.000000000","message":"since PivotRange takes kwargs I think you could do:\n\n  reanges.append(PivotRange(**pivot))","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f0fcfe79c41d2a821064f754f5ef3cb135669d4","unresolved":false,"context_lines":[{"line_number":378,"context_line":"                queries.append(tmp_q)"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"        def run_query(qry):"},{"line_number":381,"context_line":"            objs \u003d broker.list_objects_iter(CONTAINER_LISTING_LIMIT, **qry)"},{"line_number":382,"context_line":"            if not objs:"},{"line_number":383,"context_line":"                return"},{"line_number":384,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_253782e6","line":381,"updated":"2017-03-30 16:24:02.000000000","message":"we lose information here - the list_objects_iter does not include all the timestamps (data, ctype and meta) from the db rows so the apparent time of the moved object is wrong w.r.t. the original - see test here [1]\n\n[1] https://review.openstack.org/451896","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f0fcfe79c41d2a821064f754f5ef3cb135669d4","unresolved":false,"context_lines":[{"line_number":390,"context_line":"                self.ranges \u003d self._get_pivot_ranges("},{"line_number":391,"context_line":"                    root_account, root_container, newest\u003dTrue)"},{"line_number":392,"context_line":""},{"line_number":393,"context_line":"            pivot_to_obj \u003d {}"},{"line_number":394,"context_line":"            for obj in objs:"},{"line_number":395,"context_line":"                p \u003d find_pivot_range(obj, self.ranges)"},{"line_number":396,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_383d762c","line":393,"updated":"2017-03-30 16:24:02.000000000","message":"use a collections.defaultdict(list) and then there is no need for the if/else at lines 397-400","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f0fcfe79c41d2a821064f754f5ef3cb135669d4","unresolved":false,"context_lines":[{"line_number":684,"context_line":"            self.ranges \u003d []"},{"line_number":685,"context_line":"            self.node_idx \u003d node_idx"},{"line_number":686,"context_line":"            self.node_id \u003d node_id"},{"line_number":687,"context_line":"            self.part \u003d int(part)"},{"line_number":688,"context_line":"            root_account, root_container \u003d \\"},{"line_number":689,"context_line":"                ContainerSharder.get_shard_root_path(broker)"},{"line_number":690,"context_line":"            pivot \u003d ContainerSharder.get_pivot_range(broker)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_7882de02","line":687,"updated":"2017-03-30 16:24:02.000000000","message":"do these need to be instance variables? self.ranges for example seems to be only read in this method. Seems a bit risky to set per-directory state as daemon level variables, in case it were to bleed from one dir in the loop to another","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":789,"context_line":""},{"line_number":790,"context_line":"        self.logger.info(_(\u0027Finished container sharding pass\u0027))"},{"line_number":791,"context_line":""},{"line_number":792,"context_line":"        if time.time() - reported \u003e\u003d 3600:  # once an hour"},{"line_number":793,"context_line":"            self.logger.info("},{"line_number":794,"context_line":"                _(\u0027Since %(time)s Stats: %(scanned)s scanned, \u0027"},{"line_number":795,"context_line":"                  \u0027%(sharded)s sharded, %(shrunk)s shrunk, \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_f1ed40b7","line":792,"updated":"2017-03-28 20:05:12.000000000","message":"I think the following lines could be hugely simplified if the stats were maintained in a dict - I\u0027ll push a patch to illustrate.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":825,"context_line":"        return reported"},{"line_number":826,"context_line":""},{"line_number":827,"context_line":"    def _send_request(self, ip, port, contdevice, partition, op, path,"},{"line_number":828,"context_line":"                      headers_out\u003d{}, node_idx\u003dNone):"},{"line_number":829,"context_line":"        if \u0027user-agent\u0027 not in headers_out:"},{"line_number":830,"context_line":"            headers_out[\u0027user-agent\u0027] \u003d \u0027container-sharder %s\u0027 % \\"},{"line_number":831,"context_line":"                                        os.getpid()"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3ca14795","line":828,"updated":"2017-03-28 20:05:12.000000000","message":"don\u0027t use mutable as a default arg, use None then have:\n\n  headers_out \u003d {} if headers_out is None else headers_out\n\nhttps://pythonconquerstheuniverse.wordpress.com/2012/02/15/mutable-default-arguments/","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":1646,"context_line":"        reported \u003d time.time()"},{"line_number":1647,"context_line":"        time.sleep(random() * self.interval)"},{"line_number":1648,"context_line":"        while True:"},{"line_number":1649,"context_line":"            self.logger.info(_(\u0027Begin container sharder pass.\u0027))"},{"line_number":1650,"context_line":"            begin \u003d time.time()"},{"line_number":1651,"context_line":"            try:"},{"line_number":1652,"context_line":"                self._one_shard_pass(reported)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_f142e0aa","line":1649,"updated":"2017-03-28 20:05:12.000000000","message":"this duplicates similar log at start of _one_shard_pass","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a1036fcf58a227de0757ca631eee5dc5314c6889","unresolved":false,"context_lines":[{"line_number":1655,"context_line":"                self.logger.exception(_(\u0027ERROR sharding\u0027))"},{"line_number":1656,"context_line":"            elapsed \u003d time.time() - begin"},{"line_number":1657,"context_line":"            if elapsed \u003c self.interval:"},{"line_number":1658,"context_line":"                time.sleep(self.interval - elapsed)"},{"line_number":1659,"context_line":"            self.logger.info("},{"line_number":1660,"context_line":"                _(\u0027Container sharder pass completed: %.02fs\u0027), elapsed)"},{"line_number":1661,"context_line":"            dump_recon_cache({\u0027container_sharder_pass_completed\u0027: elapsed},"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9132745d","line":1658,"updated":"2017-03-28 20:05:12.000000000","message":"should we sleep after logging and dumping the stats?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":45,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# The default internal client config body is to support upgrades without"},{"line_number":48,"context_line":"# requiring deployment of the new /etc/swift/internal-client.conf"},{"line_number":49,"context_line":"ic_conf_body \u003d \"\"\""},{"line_number":50,"context_line":"[DEFAULT]"},{"line_number":51,"context_line":"# swift_dir \u003d /etc/swift"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_d6da8f15","line":48,"updated":"2017-04-28 01:38:47.000000000","message":"I know this got picked up from container-sync, but I think we can probably drop it -- I think it\u0027s entirely reasonable to say that if you want this new thing, you need to configure it properly.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":103,"context_line":"class ContainerSharder(ContainerReplicator):"},{"line_number":104,"context_line":"    \"\"\"Shards containers.\"\"\""},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"    def __init__(self, conf, logger\u003dNone, **kargs):"},{"line_number":107,"context_line":"        logger \u003d logger or get_logger(conf, log_route\u003d\u0027container-sharder\u0027)"},{"line_number":108,"context_line":"        super(ContainerReplicator, self).__init__(conf, logger\u003dlogger)"},{"line_number":109,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_b64383af","line":106,"updated":"2017-04-28 01:38:47.000000000","message":"kargs seems to be unused...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":160,"context_line":"        self.container_pivots \u003d 0"},{"line_number":161,"context_line":"        self.containers_misplaced \u003d 0"},{"line_number":162,"context_line":"        self.containers_audit_failed \u003d 0"},{"line_number":163,"context_line":"        self.containers_failed \u003d 0"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"    def _get_local_devices(self):"},{"line_number":166,"context_line":"        self._local_device_ids \u003d set()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_4511de03","line":163,"updated":"2017-04-28 01:38:47.000000000","message":"I would\u0027ve been tempted to do something like\n\n self.stats.update({\n     \u0027containers_scanned\u0027: 0,\n     \u0027containers_sharded\u0027: 0,\n     \u0027containers_shrunk\u0027: 0,\n     \u0027container_pivots\u0027: 0,\n     \u0027containers_misplaced\u0027: 0,\n     \u0027containers_audit_failed\u0027: 0,\n     \u0027containers_failed\u0027: 0,\n })\n\nAny reason to prefer the attrs?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":174,"context_line":"                                        node[\u0027replication_ip\u0027],"},{"line_number":175,"context_line":"                                        node[\u0027replication_port\u0027]):"},{"line_number":176,"context_line":"                results.add(node[\u0027device\u0027])"},{"line_number":177,"context_line":"                self._local_device_ids.add(node[\u0027id\u0027])"},{"line_number":178,"context_line":"        return results"},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"    def _get_pivot_ranges(self, account, container, newest\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_0593d65c","line":177,"updated":"2017-04-28 01:38:47.000000000","message":"Bit of a curious (and not exactly obvious) side effect...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":175,"context_line":"                                        node[\u0027replication_port\u0027]):"},{"line_number":176,"context_line":"                results.add(node[\u0027device\u0027])"},{"line_number":177,"context_line":"                self._local_device_ids.add(node[\u0027id\u0027])"},{"line_number":178,"context_line":"        return results"},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"    def _get_pivot_ranges(self, account, container, newest\u003dFalse):"},{"line_number":181,"context_line":"        path \u003d self.swift.make_path(account, container) + \\"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_65b3ba0b","line":178,"updated":"2017-04-28 01:38:47.000000000","message":"Worth memoizing?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":179,"context_line":""},{"line_number":180,"context_line":"    def _get_pivot_ranges(self, account, container, newest\u003dFalse):"},{"line_number":181,"context_line":"        path \u003d self.swift.make_path(account, container) + \\"},{"line_number":182,"context_line":"            \u0027?items\u003dpivot\u0026format\u003djson\u0027"},{"line_number":183,"context_line":"        headers \u003d dict()"},{"line_number":184,"context_line":"        if newest:"},{"line_number":185,"context_line":"            headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_05cf5656","line":182,"updated":"2017-04-28 01:38:47.000000000","message":"What happens if we have more than container_listing_limit pivots? Do we ever detect the error?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":202,"context_line":"                meta_timestamp \u003d pivot.get(\u0027meta_timestamp\u0027) or None"},{"line_number":203,"context_line":"                ranges.append(PivotRange(pivot[\u0027name\u0027], created_at, lower,"},{"line_number":204,"context_line":"                                         upper, object_count, bytes_used,"},{"line_number":205,"context_line":"                                         meta_timestamp))"},{"line_number":206,"context_line":"        except ValueError:"},{"line_number":207,"context_line":"            # Failed to decode the json response"},{"line_number":208,"context_line":"            return None"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_65329a60","line":205,"updated":"2017-04-28 01:38:47.000000000","message":"Maybe worth putting a from_dict class method on PivotRange?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":206,"context_line":"        except ValueError:"},{"line_number":207,"context_line":"            # Failed to decode the json response"},{"line_number":208,"context_line":"            return None"},{"line_number":209,"context_line":"        return ranges"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"    def _get_shard_broker(self, account, container, policy_index):"},{"line_number":212,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_851a46c9","line":209,"updated":"2017-04-28 01:38:47.000000000","message":"Just to be sure, we expect ranges to be sorted, yeah?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":208,"context_line":"            return None"},{"line_number":209,"context_line":"        return ranges"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"    def _get_shard_broker(self, account, container, policy_index):"},{"line_number":212,"context_line":"        \"\"\""},{"line_number":213,"context_line":"        Get a local instance of the shard container broker that will be"},{"line_number":214,"context_line":"        pushed out."}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_e019f879","line":211,"updated":"2017-04-28 01:38:47.000000000","message":"None of this seems specific to shards (besides the error message) ... I\u0027m kinda surprised we don\u0027t already have this elsewhere. Meh.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":222,"context_line":"        if not node:"},{"line_number":223,"context_line":"            raise DeviceUnavailable("},{"line_number":224,"context_line":"                \u0027No mounted devices found suitable to Handoff sharded \u0027"},{"line_number":225,"context_line":"                \u0027container %s in partition %s\u0027 % (container, part))"},{"line_number":226,"context_line":"        hsh \u003d hash_path(account, container)"},{"line_number":227,"context_line":"        db_dir \u003d storage_directory(DATADIR, part, hsh)"},{"line_number":228,"context_line":"        db_path \u003d os.path.join(self.root, node[\u0027device\u0027], db_dir, hsh + \u0027.db\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_e5fa2a41","line":225,"updated":"2017-04-28 01:38:47.000000000","message":"I\u0027m trying to think of what would trigger this...\n\nI guess having a large container (so we need to shard it) on a node that doesn\u0027t have any non-zero-weight devices in the container ring? We can\u0027t make progress in that case, can we?\n\nBut then, we won\u0027t be a primary for the container, either, so we wouldn\u0027t be able to do anything anyway, yeah? Either way, this seems like a state in which to raise an error...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":242,"context_line":"        Create a list of dictionary items ready to be consumed by"},{"line_number":243,"context_line":"        Broker.merge_items()"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"        :param items: list of objects"},{"line_number":246,"context_line":"        :param policy_index: the Policy index of the container"},{"line_number":247,"context_line":"        :param delete: mark the objects as deleted; default False"},{"line_number":248,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_e08e9877","line":245,"updated":"2017-04-28 01:38:47.000000000","message":"... or pivots","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":263,"context_line":"                    # may exist out there. The idea is, newer object out there"},{"line_number":264,"context_line":"                    # will replace the deleted record, which will be picked up"},{"line_number":265,"context_line":"                    # as a misplaced object and then be pushed and compared to"},{"line_number":266,"context_line":"                    # where it needs to be."},{"line_number":267,"context_line":"                    ts \u003d Timestamp(item[1])"},{"line_number":268,"context_line":"                    ts.offset +\u003d 1"},{"line_number":269,"context_line":"                    timestamp \u003d ts.internal"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_20e800e0","line":266,"updated":"2017-04-28 01:38:47.000000000","message":"For a bit I was confused; isn\u0027t the timestamp the time at which the tombstone was written?\n\nAfter a while I realized, no, we\u0027re talking about recording deletes for the creates that are now pushed out to shards! Which makes perfect sense given the context, duh Tim...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":266,"context_line":"                    # where it needs to be."},{"line_number":267,"context_line":"                    ts \u003d Timestamp(item[1])"},{"line_number":268,"context_line":"                    ts.offset +\u003d 1"},{"line_number":269,"context_line":"                    timestamp \u003d ts.internal"},{"line_number":270,"context_line":"                obj \u003d {"},{"line_number":271,"context_line":"                    \u0027created_at\u0027: timestamp or item[1]}"},{"line_number":272,"context_line":"                if not isinstance(item[2], int):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_fa608403","line":269,"updated":"2017-04-28 01:38:47.000000000","message":"Better as\n\n timestamp \u003d Timestamp(item[1], offset\u003d1).internal\n\n?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":302,"context_line":"    def _get_node_index(self):"},{"line_number":303,"context_line":"        nodes \u003d self.ring.get_part_nodes(self.part)"},{"line_number":304,"context_line":"        indices \u003d [node[\u0027index\u0027] for node in nodes"},{"line_number":305,"context_line":"                   if node[\u0027id\u0027] \u003d\u003d self.node_id]"},{"line_number":306,"context_line":"        return indices[0] if indices else None"},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"    def _add_shard_metadata(self, broker, root_account, root_container,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_e04578b5","line":305,"updated":"2017-04-28 01:38:47.000000000","message":"self.node_id changes as we walk datadirs, yeah? I\u0027m not sure I like having that hang off self...\n\nAt the very least, I feel like we should include a\n\n if not hasattr(self, \u0027node_id\u0027):\n     return None\n\nat the start of this.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":303,"context_line":"        nodes \u003d self.ring.get_part_nodes(self.part)"},{"line_number":304,"context_line":"        indices \u003d [node[\u0027index\u0027] for node in nodes"},{"line_number":305,"context_line":"                   if node[\u0027id\u0027] \u003d\u003d self.node_id]"},{"line_number":306,"context_line":"        return indices[0] if indices else None"},{"line_number":307,"context_line":""},{"line_number":308,"context_line":"    def _add_shard_metadata(self, broker, root_account, root_container,"},{"line_number":309,"context_line":"                            pivot, force\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_40854cce","line":306,"updated":"2017-04-28 01:38:47.000000000","message":"This returns None if we\u0027re on a handoff, yeah? Does this mean only primaries can shard?\n\nWhat happens if container-ring rebalancing has gotten ahead of itself and *all* of the large copies are now on handoffs? Part of the trouble with large containers is that rsync starts timing out when trying to replicate, right? Does this mean that you\u0027ve gotten yourself so far into a hole that there\u0027s just no good path forward?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":309,"context_line":"                            pivot, force\u003dFalse):"},{"line_number":310,"context_line":"        if not broker.metadata.get(\u0027X-Container-Sysmeta-Shard-Account\u0027) \\"},{"line_number":311,"context_line":"                and pivot or force:"},{"line_number":312,"context_line":"            timestamp \u003d Timestamp(time.time()).internal"},{"line_number":313,"context_line":"            broker.update_metadata({"},{"line_number":314,"context_line":"                \u0027X-Container-Sysmeta-Shard-Account\u0027: (root_account, timestamp),"},{"line_number":315,"context_line":"                \u0027X-Container-Sysmeta-Shard-Container\u0027:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ba2b1c93","line":312,"updated":"2017-04-28 01:38:47.000000000","message":"This construction seems fairly common... I wonder why we don\u0027t have a Timestamp.now() method?\n\nProposed https://review.openstack.org/460754","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":320,"context_line":"                    (pivot.timestamp, timestamp),"},{"line_number":321,"context_line":"                \u0027X-Container-Sysmeta-Shard-Meta-Timestamp\u0027:"},{"line_number":322,"context_line":"                    (pivot.meta_timestamp, timestamp),"},{"line_number":323,"context_line":"                \u0027X-Container-Sysmeta-Sharding\u0027: (None, timestamp)})"},{"line_number":324,"context_line":""},{"line_number":325,"context_line":"    def _misplaced_objects(self, broker, root_account, root_container, pivot):"},{"line_number":326,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_336df7cb","line":323,"updated":"2017-04-28 01:38:47.000000000","message":"Wait... this might be set already? How?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":322,"context_line":"                    (pivot.meta_timestamp, timestamp),"},{"line_number":323,"context_line":"                \u0027X-Container-Sysmeta-Sharding\u0027: (None, timestamp)})"},{"line_number":324,"context_line":""},{"line_number":325,"context_line":"    def _misplaced_objects(self, broker, root_account, root_container, pivot):"},{"line_number":326,"context_line":"        \"\"\""},{"line_number":327,"context_line":"        Search for objects in the current broker that don\u0027t belong, and move"},{"line_number":328,"context_line":"        to the container shards they do."}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_04406fa4","line":325,"updated":"2017-04-28 01:38:47.000000000","message":"Should we call this _misplaced_listings, so we\u0027re less likely to confuse it with the container-reconciler\u0027s notion of misplaced objects?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":331,"context_line":"                       been moved."},{"line_number":332,"context_line":"        :param root_account: The root account"},{"line_number":333,"context_line":"        :param root_container: The root container"},{"line_number":334,"context_line":"        :param pivot: The pivot point of the container"},{"line_number":335,"context_line":"        \"\"\""},{"line_number":336,"context_line":""},{"line_number":337,"context_line":"        self.logger.info(_(\u0027Scanning %s/%s for misplaced objects\u0027),"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_892ff83d","line":334,"updated":"2017-04-28 01:38:47.000000000","message":"Maybe better as\n\n\u003e The PivotRange appropriate for this broker, or None if the current broker is for an unsharded or root container\n\n? Is that accurate?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":340,"context_line":"        misplaced_items \u003d [False]"},{"line_number":341,"context_line":"        policy_index \u003d broker.storage_policy_index"},{"line_number":342,"context_line":"        query \u003d dict(marker\u003d\u0027\u0027, end_marker\u003d\u0027\u0027, prefix\u003d\u0027\u0027, delimiter\u003d\u0027\u0027,"},{"line_number":343,"context_line":"                     storage_policy_index\u003dpolicy_index)"},{"line_number":344,"context_line":"        state \u003d broker.get_db_state()"},{"line_number":345,"context_line":""},{"line_number":346,"context_line":"        if state \u003d\u003d DB_STATE_SHARDED or broker.is_deleted():"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_0414ef42","line":343,"updated":"2017-04-28 01:38:47.000000000","message":"What about listings for objects in the wrong storage policy?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":345,"context_line":""},{"line_number":346,"context_line":"        if state \u003d\u003d DB_STATE_SHARDED or broker.is_deleted():"},{"line_number":347,"context_line":"            # It\u0027s a sharded node or deleted, so anything in the object table"},{"line_number":348,"context_line":"            # is treated as a misplaced object."},{"line_number":349,"context_line":"            if broker.get_info()[\u0027object_count\u0027] \u003e 0:"},{"line_number":350,"context_line":"                queries.append(query.copy())"},{"line_number":351,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_732fff3e","line":348,"updated":"2017-04-28 01:38:47.000000000","message":"Do we treat deleted containers this way because we\u0027re assuming it\u0027s a deleted shard, and this gives us a way to accept then redirect object updates that were written for the old shard location?\n\nWhat about the ordinary-container case, where we\u0027ve got a partition where one side sees\n\n t1 DELETE all objects in container\n t3 DELETE container\n\nand the other side sees\n\n t2 PUT new object in container\n\n? How do we handle that?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":355,"context_line":"            # object table that is less then (\u003c) the pivot this node it up to is"},{"line_number":356,"context_line":"            # considered misplaced, anything above is being held."},{"line_number":357,"context_line":"            last_pivot \u003d self.get_metadata_item("},{"line_number":358,"context_line":"                broker, \u0027X-Container-Sysmeta-Shard-Last-%d\u0027 % self.node_id)"},{"line_number":359,"context_line":"            if not last_pivot:"},{"line_number":360,"context_line":"                # This node hasn\u0027t pivoted/sharded anything yet, so all objects"},{"line_number":361,"context_line":"                # in object table are suppose to be there (in holding)."}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ce2bc692","line":358,"updated":"2017-04-28 01:38:47.000000000","message":"The last pivot is for this particular primary? How do they reconcile?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":361,"context_line":"                # in object table are suppose to be there (in holding)."},{"line_number":362,"context_line":"                return"},{"line_number":363,"context_line":"            tmp_q \u003d query.copy()"},{"line_number":364,"context_line":"            tmp_q[\u0027end_marker\u0027] \u003d last_pivot"},{"line_number":365,"context_line":"            queries.append(tmp_q)"},{"line_number":366,"context_line":""},{"line_number":367,"context_line":"        elif pivot is None or state \u003d\u003d DB_STATE_NOTFOUND:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_e910bc50","line":364,"updated":"2017-04-28 01:38:47.000000000","message":"With pivots, we\u0027re exclusive on the lower, inclusive on the upper, right? Both marker and end_marker are exclusive. Should this be\n\n last_pivot + \u0027\\x00\u0027\n\n?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":364,"context_line":"            tmp_q[\u0027end_marker\u0027] \u003d last_pivot"},{"line_number":365,"context_line":"            queries.append(tmp_q)"},{"line_number":366,"context_line":""},{"line_number":367,"context_line":"        elif pivot is None or state \u003d\u003d DB_STATE_NOTFOUND:"},{"line_number":368,"context_line":"            # This is an unsharded root container, so we don\u0027t need to"},{"line_number":369,"context_line":"            # query anything."},{"line_number":370,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_29e904b6","line":367,"updated":"2017-04-28 01:38:47.000000000","message":"When would we hit DB_STATE_NOTFOUND?\n\nI suppose it does seem reasonable to short-circuit in that case...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":377,"context_line":"                queries.append(tmp_q)"},{"line_number":378,"context_line":"            if pivot.lower:"},{"line_number":379,"context_line":"                tmp_q \u003d query.copy()"},{"line_number":380,"context_line":"                tmp_q[\u0027end_marker\u0027] \u003d pivot.lower"},{"line_number":381,"context_line":"                queries.append(tmp_q)"},{"line_number":382,"context_line":""},{"line_number":383,"context_line":"        def run_query(qry):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_09fbe878","line":380,"updated":"2017-04-28 01:38:47.000000000","message":"I think this should be\n\n pivot.lower + \u0027\\x00\u0027\n\nRight? If the pivot.lower name is here, we *do* want it included in the query, yeah?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":381,"context_line":"                queries.append(tmp_q)"},{"line_number":382,"context_line":""},{"line_number":383,"context_line":"        def run_query(qry):"},{"line_number":384,"context_line":"            objs \u003d broker.list_objects_iter(CONTAINER_LISTING_LIMIT, **qry)"},{"line_number":385,"context_line":"            if not objs:"},{"line_number":386,"context_line":"                return"},{"line_number":387,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_a9707409","line":384,"updated":"2017-04-28 01:38:47.000000000","message":"The name makes me think this returns a generator or something, in which case\n\n if not objs:\n\nwill never trigger. But nope, it\u0027s a list: https://github.com/openstack/swift/blob/2.14.0/swift/container/backend.py#L698\n\nThis won\u0027t pick up updates from object DELETEs, will it? Is that going to lead to ghost listings? If a DELETE update hits only root containers, will that DELETE ever make its way to the appropriate shard?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":385,"context_line":"            if not objs:"},{"line_number":386,"context_line":"                return"},{"line_number":387,"context_line":""},{"line_number":388,"context_line":"            misplaced_items[0] \u003d True"},{"line_number":389,"context_line":""},{"line_number":390,"context_line":"            # We have a list of misplaced objects, so we better find a home"},{"line_number":391,"context_line":"            # for them"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_097888c1","line":388,"updated":"2017-04-28 01:38:47.000000000","message":"I wonder if something like http://paste.openstack.org/show/608241/ might be a bit less action-at-a-distance-y","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":393,"context_line":"                self.ranges \u003d self._get_pivot_ranges("},{"line_number":394,"context_line":"                    root_account, root_container, newest\u003dTrue)"},{"line_number":395,"context_line":""},{"line_number":396,"context_line":"            pivot_to_obj \u003d {}"},{"line_number":397,"context_line":"            for obj in objs:"},{"line_number":398,"context_line":"                p \u003d find_pivot_range(obj, self.ranges)"},{"line_number":399,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_891b787f","line":396,"updated":"2017-04-28 01:38:47.000000000","message":"Better with defaultdict http://paste.openstack.org/show/608242/","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":404,"context_line":"                qry[\u0027marker\u0027] \u003d obj[0]"},{"line_number":405,"context_line":""},{"line_number":406,"context_line":"            self.logger.info(_(\u0027preparing to move misplaced objects found \u0027"},{"line_number":407,"context_line":"                               \u0027in %s/%s\u0027), broker.account, broker.container)"},{"line_number":408,"context_line":"            for piv, obj_list in pivot_to_obj.items():"},{"line_number":409,"context_line":"                acct \u003d account_to_pivot_account(root_account)"},{"line_number":410,"context_line":"                part, new_broker, node_id \u003d \\"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_c43287f2","line":407,"updated":"2017-04-28 01:38:47.000000000","message":"Maybe worth dropping\n\n sum(len(obj_list) for obj_list in pivot_to_obj.values())\n\nin here?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":417,"context_line":"                new_broker.merge_items(objects)"},{"line_number":418,"context_line":""},{"line_number":419,"context_line":"                self.cpool.spawn("},{"line_number":420,"context_line":"                    self._replicate_object, part, new_broker.db_file, node_id)"},{"line_number":421,"context_line":""},{"line_number":422,"context_line":"                # Remove the now relocated misplaced items."},{"line_number":423,"context_line":"                items \u003d self._generate_object_list(obj_list, policy_index,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_04998fc2","line":420,"updated":"2017-04-28 01:38:47.000000000","message":"So we go ahead and push the newly-created shards out immediately? If we didn\u0027t, the normal container-replicator would eventually pick them up and get them where they need to go, right?\n\nPart of me wonders whether it\u0027d be better to hold off for now, just to keep this loop tighter.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":423,"context_line":"                items \u003d self._generate_object_list(obj_list, policy_index,"},{"line_number":424,"context_line":"                                                   delete\u003dTrue)"},{"line_number":425,"context_line":"                broker.merge_items(items)"},{"line_number":426,"context_line":"            any(self.cpool)"},{"line_number":427,"context_line":""},{"line_number":428,"context_line":"            if len(objs) \u003d\u003d CONTAINER_LISTING_LIMIT:"},{"line_number":429,"context_line":"                # There could be more, so recurse my pretty"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_a48bfb65","line":426,"updated":"2017-04-28 01:38:47.000000000","message":"This seems like magic. I guess it means, \"wait for one of these to error, or all to complete successfully\"? Maybe worth a comment?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":425,"context_line":"                broker.merge_items(items)"},{"line_number":426,"context_line":"            any(self.cpool)"},{"line_number":427,"context_line":""},{"line_number":428,"context_line":"            if len(objs) \u003d\u003d CONTAINER_LISTING_LIMIT:"},{"line_number":429,"context_line":"                # There could be more, so recurse my pretty"},{"line_number":430,"context_line":"                run_query(qry)"},{"line_number":431,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_49321060","line":428,"updated":"2017-04-28 01:38:47.000000000","message":"Maybe worth always recursing? We\u0027ve already got the\n\n if not objs:\n     return\n\ncheck at the outset, right? What\u0027s one extra stack frame?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":436,"context_line":"            self.logger.increment(\u0027misplaced_items_found\u0027)"},{"line_number":437,"context_line":"            self.containers_misplaced +\u003d 1"},{"line_number":438,"context_line":""},{"line_number":439,"context_line":"        # wipe out the cache do disable bypass in delete_db"},{"line_number":440,"context_line":"        cleanups \u003d self.shard_cleanups or {}"},{"line_number":441,"context_line":"        self.shard_cleanups \u003d self.shard_brokers \u003d None"},{"line_number":442,"context_line":"        self.logger.info(\u0027Cleaning up %d replicated shard containers\u0027,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_44daf74a","line":439,"updated":"2017-04-28 01:38:47.000000000","message":"s/do disable/to disable/, yeah?\n\nThis seems repeated from _one_shard_cycle; why have it in both? Worth kicking out as some _really_delete_dbs helper?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":440,"context_line":"        cleanups \u003d self.shard_cleanups or {}"},{"line_number":441,"context_line":"        self.shard_cleanups \u003d self.shard_brokers \u003d None"},{"line_number":442,"context_line":"        self.logger.info(\u0027Cleaning up %d replicated shard containers\u0027,"},{"line_number":443,"context_line":"                         len(cleanups) if cleanups else 0)"},{"line_number":444,"context_line":"        for container in cleanups.values():"},{"line_number":445,"context_line":"            self.cpool.spawn(self.delete_db, container)"},{"line_number":446,"context_line":"        any(self.cpool)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_a41bfbc7","line":443,"updated":"2017-04-28 01:38:47.000000000","message":"Why the\n\n ... if cleanups else 0\n\n?\n\n len({}) \u003d\u003d 0\n\nno?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":443,"context_line":"                         len(cleanups) if cleanups else 0)"},{"line_number":444,"context_line":"        for container in cleanups.values():"},{"line_number":445,"context_line":"            self.cpool.spawn(self.delete_db, container)"},{"line_number":446,"context_line":"        any(self.cpool)"},{"line_number":447,"context_line":"        self.logger.info(\u0027Finished misplaced shard replication\u0027)"},{"line_number":448,"context_line":""},{"line_number":449,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_a4eadbd4","line":446,"updated":"2017-04-28 01:38:47.000000000","message":"Similar to above, feels mystical. Could we like\n\n self.cpool.waitall()\n\nor something? Is that a thing?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":455,"context_line":"        lower \u003d ContainerSharder.get_metadata_item("},{"line_number":456,"context_line":"            broker, \u0027X-Container-Sysmeta-Shard-Lower\u0027)"},{"line_number":457,"context_line":""},{"line_number":458,"context_line":"        upper \u003d ContainerSharder.get_metadata_item("},{"line_number":459,"context_line":"            broker, \u0027X-Container-Sysmeta-Shard-Upper\u0027)"},{"line_number":460,"context_line":"        if not lower and not upper:"},{"line_number":461,"context_line":"            return None"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_a4289b96","line":458,"updated":"2017-04-28 01:38:47.000000000","message":"Each of these calls to get_metadata_item runs a query to grab *all* metadata, yeah? I feel like it\u0027d be better to grab all the meta *once*, then pick over the result...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":469,"context_line":"                           0, meta_timestamp)"},{"line_number":470,"context_line":"        tmp_info \u003d broker.get_info()"},{"line_number":471,"context_line":"        pivot.obj_count \u003d tmp_info.get(\u0027object_count\u0027, 0)"},{"line_number":472,"context_line":"        pivot.bytes_used \u003d tmp_info.get(\u0027bytes_used\u0027, 0)"},{"line_number":473,"context_line":""},{"line_number":474,"context_line":"        return pivot"},{"line_number":475,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_1f0b1a24","line":472,"updated":"2017-04-28 01:38:47.000000000","message":"Couldn\u0027t we do the get_info() call first, then just pass these to the PivotRange constructor?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":479,"context_line":"        Attempt to get the root shard container name and account for the"},{"line_number":480,"context_line":"        container represented by this broker."},{"line_number":481,"context_line":""},{"line_number":482,"context_line":"        A container shard has \u0027X-Container-Sysmeta-Shard-{Account,Container}"},{"line_number":483,"context_line":"        set, which will contain the relevant values for the root shard"},{"line_number":484,"context_line":"        container. If they don\u0027t exist, then it returns the account and"},{"line_number":485,"context_line":"        container associated directly with the broker."}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_5f53e217","line":482,"updated":"2017-04-28 01:38:47.000000000","message":"Could we ever have one but not the other set?\n\nShould they always have the same timestamp? If so, is it worth validating that they do?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":488,"context_line":"        :return: account, container of the root shard container or the brokers"},{"line_number":489,"context_line":"                 if it can\u0027t be determined."},{"line_number":490,"context_line":"        \"\"\""},{"line_number":491,"context_line":"        broker.get_info()"},{"line_number":492,"context_line":"        account \u003d broker.metadata.get(\u0027X-Container-Sysmeta-Shard-Account\u0027)"},{"line_number":493,"context_line":"        if account:"},{"line_number":494,"context_line":"            account \u003d account[0]"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ff13ce5a","line":491,"updated":"2017-04-28 01:38:47.000000000","message":"Why call get_info() if we don\u0027t use the result? I guess something similar to L236, we want to make sure it\u0027s *ever* been done?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":495,"context_line":"        else:"},{"line_number":496,"context_line":"            account \u003d broker.account"},{"line_number":497,"context_line":""},{"line_number":498,"context_line":"        container \u003d broker.metadata.get(\u0027X-Container-Sysmeta-Shard-Container\u0027)"},{"line_number":499,"context_line":"        if container:"},{"line_number":500,"context_line":"            container \u003d container[0]"},{"line_number":501,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_1f243aae","line":498,"updated":"2017-04-28 01:38:47.000000000","message":"Similar to above, I feel like it\u0027d be better to get broker.metadata once, then do our get()s from the result.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":504,"context_line":"        return account, container"},{"line_number":505,"context_line":""},{"line_number":506,"context_line":"    def _post_replicate_hook(self, broker, info, responses):"},{"line_number":507,"context_line":"        return"},{"line_number":508,"context_line":""},{"line_number":509,"context_line":"    def delete_db(self, broker):"},{"line_number":510,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_bf60a6d0","line":507,"updated":"2017-04-28 01:38:47.000000000","message":"This deserves a comment. We\u0027re consciously overriding and no-op\u0027ing the general container replicator here. What does it do that we don\u0027t want done? Why don\u0027t we want it done? Is there anything else that *should* be done instead?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":522,"context_line":"        # TODO We will need to audit the root (make sure there are no missing"},{"line_number":523,"context_line":"        #      gaps in the ranges."},{"line_number":524,"context_line":"        # TODO Search for overlaps if you find some, keep the newest and"},{"line_number":525,"context_line":"        #      attempt to correct (remove the older one)."},{"line_number":526,"context_line":"        # TODO If the shard container has a sharding lock then see if it\u0027s"},{"line_number":527,"context_line":"        #       needed. Maybe something as simple as sharding lock older then"},{"line_number":528,"context_line":"        #       reclaim age."}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_7f069ed0","line":525,"updated":"2017-04-28 01:38:47.000000000","message":"This seems to be TODONE, yeah?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":536,"context_line":"        if broker.is_deleted():"},{"line_number":537,"context_line":"            return continue_with_container"},{"line_number":538,"context_line":""},{"line_number":539,"context_line":"        if not root_account or not root_container:"},{"line_number":540,"context_line":"            root_account, root_container \u003d \\"},{"line_number":541,"context_line":"                ContainerSharder.get_shard_root_path(broker)"},{"line_number":542,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_1f80ba45","line":539,"updated":"2017-04-28 01:38:47.000000000","message":"Would we ever pass one but not the other?\n\nIs it worth making the arg a tuple or something, to ensure either both or neither is sent?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":540,"context_line":"            root_account, root_container \u003d \\"},{"line_number":541,"context_line":"                ContainerSharder.get_shard_root_path(broker)"},{"line_number":542,"context_line":""},{"line_number":543,"context_line":"        if root_container \u003d\u003d broker.container:"},{"line_number":544,"context_line":"            # This is the root container, and therefore the tome of knowledge,"},{"line_number":545,"context_line":"            # all we can do is check there is nothing screwy with the range"},{"line_number":546,"context_line":"            ranges \u003d broker.build_pivot_ranges()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_dfda5276","line":543,"updated":"2017-04-28 01:38:47.000000000","message":"No check on account? I suppose it probably shouldn\u0027t be a problem...\n\nWhat about pivot? What does that tell us about our state and place in the world?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":553,"context_line":""},{"line_number":554,"context_line":"                # now delete the older overlaps, keeping only the newest"},{"line_number":555,"context_line":"                timestamp \u003d Timestamp(newest.timestamp)"},{"line_number":556,"context_line":"                timestamp.offset +\u003d 1"},{"line_number":557,"context_line":""},{"line_number":558,"context_line":"                def update_timestamp(pivot):"},{"line_number":559,"context_line":"                    pivot.timestamp \u003d timestamp"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_bfbc2606","line":556,"updated":"2017-04-28 01:38:47.000000000","message":"nit: You can include offset in the Timestamp constructor","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":557,"context_line":""},{"line_number":558,"context_line":"                def update_timestamp(pivot):"},{"line_number":559,"context_line":"                    pivot.timestamp \u003d timestamp"},{"line_number":560,"context_line":"                map(update_timestamp, older)"},{"line_number":561,"context_line":"                self._update_pivot_ranges(root_account, root_container,"},{"line_number":562,"context_line":"                                          \u0027DELETE\u0027, older)"},{"line_number":563,"context_line":"                continue_with_container \u003d False"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_7f61fea8","line":560,"updated":"2017-04-28 01:38:47.000000000","message":"I\u0027m included to say\n\n for pivot in older:\n     pivot.timestamp \u003d timestamp\n\ninstead; it\u0027s a little more direct, and avoids the py3 pitfall that map() returns an iterator.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":559,"context_line":"                    pivot.timestamp \u003d timestamp"},{"line_number":560,"context_line":"                map(update_timestamp, older)"},{"line_number":561,"context_line":"                self._update_pivot_ranges(root_account, root_container,"},{"line_number":562,"context_line":"                                          \u0027DELETE\u0027, older)"},{"line_number":563,"context_line":"                continue_with_container \u003d False"},{"line_number":564,"context_line":"            missing_ranges \u003d ContainerSharder.check_complete_ranges(ranges)"},{"line_number":565,"context_line":"            if missing_ranges:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_5fc2c25a","line":562,"updated":"2017-04-28 01:38:47.000000000","message":"What happens when the sharder runs for the now-deleted shards? I guess we should treat everything as misplaced, and push records out to the correct shards? And if there *isn\u0027t* a shard for some part (because we don\u0027t have complete coverage of ranges), those listings get pushed... back here, to the root container?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":565,"context_line":"            if missing_ranges:"},{"line_number":566,"context_line":"                self.logger.error(_(\u0027Missing range(s) dectected: %s\u0027),"},{"line_number":567,"context_line":"                                  \u0027-\u0027.join(missing_ranges))"},{"line_number":568,"context_line":"                continue_with_container \u003d False"},{"line_number":569,"context_line":""},{"line_number":570,"context_line":"            if not continue_with_container:"},{"line_number":571,"context_line":"                self.logger.increment(\u0027audit_failed\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_fff98ec7","line":568,"updated":"2017-04-28 01:38:47.000000000","message":"So when do we backfill the missing ranges?\n\nI\u0027m thinking this could happen if we have a shard that itself needs to split, so we pick a pivot point, write down the first new pivot range, then fall over before writing down the second one. Any other ways come to mind? Maybe some split-brain scenarios? (Always a safe bet there\u0027s at least one ;-)","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":590,"context_line":"            return continue_with_container"},{"line_number":591,"context_line":""},{"line_number":592,"context_line":"        # pivot isn\u0027t in ranges, if it overlaps with an item, were in trouble."},{"line_number":593,"context_line":"        # If there is overlap lets see if it\u0027s newer then this containers,"},{"line_number":594,"context_line":"        # if so, it\u0027s safe to delete (quarantine this container)."},{"line_number":595,"context_line":"        # if it\u0027s newer, then it might not be updated yet, so just let it"},{"line_number":596,"context_line":"        # continue (or maybe we shouldn\u0027t?)."}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_7f90de43","line":593,"updated":"2017-04-28 01:38:47.000000000","message":"nit: s/lets/let\u0027s/, s/then/than/","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":591,"context_line":""},{"line_number":592,"context_line":"        # pivot isn\u0027t in ranges, if it overlaps with an item, were in trouble."},{"line_number":593,"context_line":"        # If there is overlap lets see if it\u0027s newer then this containers,"},{"line_number":594,"context_line":"        # if so, it\u0027s safe to delete (quarantine this container)."},{"line_number":595,"context_line":"        # if it\u0027s newer, then it might not be updated yet, so just let it"},{"line_number":596,"context_line":"        # continue (or maybe we shouldn\u0027t?)."},{"line_number":597,"context_line":"        overlaps \u003d [r for r in self.ranges if r.overlaps(pivot)]"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_5fbe42b4","line":594,"updated":"2017-04-28 01:38:47.000000000","message":"Is it? Or should we treat everything as misplaced, and try to push to the new locations?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":593,"context_line":"        # If there is overlap lets see if it\u0027s newer then this containers,"},{"line_number":594,"context_line":"        # if so, it\u0027s safe to delete (quarantine this container)."},{"line_number":595,"context_line":"        # if it\u0027s newer, then it might not be updated yet, so just let it"},{"line_number":596,"context_line":"        # continue (or maybe we shouldn\u0027t?)."},{"line_number":597,"context_line":"        overlaps \u003d [r for r in self.ranges if r.overlaps(pivot)]"},{"line_number":598,"context_line":"        if overlaps:"},{"line_number":599,"context_line":"            if max(overlaps + [pivot], key\u003dlambda x: x.timestamp) \u003d\u003d pivot:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_bfc2e632","line":596,"updated":"2017-04-28 01:38:47.000000000","message":"At this point, I\u0027m not sure *what* to think!","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":617,"context_line":"                return False"},{"line_number":618,"context_line":"        # pivot doesn\u0027t exist in the root containers ranges, but doesn\u0027t"},{"line_number":619,"context_line":"        # overlap with anything"},{"line_number":620,"context_line":"        return continue_with_container"},{"line_number":621,"context_line":""},{"line_number":622,"context_line":"    def _update_pivot_counts(self, root_account, root_container, broker):"},{"line_number":623,"context_line":"        if broker.container \u003d\u003d root_container:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_bf4546b6","line":620,"updated":"2017-04-28 01:38:47.000000000","message":"This is always True, yeah?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":630,"context_line":""},{"line_number":631,"context_line":"    @staticmethod"},{"line_number":632,"context_line":"    def get_metadata_item(broker, header):"},{"line_number":633,"context_line":"        item \u003d broker.metadata.get(header)"},{"line_number":634,"context_line":"        return None if item is None else item[0]"},{"line_number":635,"context_line":""},{"line_number":636,"context_line":"    def roundrobin_datadirs(self, datadirs):"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_bf47864c","line":633,"updated":"2017-04-28 01:38:47.000000000","message":"Since every broker.metadata is a query, I\u0027m not sure this is a good idea as a helper...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":639,"context_line":"                     self.ring.get_part_nodes(int(part))"},{"line_number":640,"context_line":"                     if node[\u0027id\u0027] \u003d\u003d node_id]"},{"line_number":641,"context_line":"            if not index:"},{"line_number":642,"context_line":"                # TODO Should probably log something here"},{"line_number":643,"context_line":"                continue"},{"line_number":644,"context_line":"            yield part, path, node_id, index[0]"},{"line_number":645,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9fde0ad8","line":642,"updated":"2017-04-28 01:38:47.000000000","message":"This happens whenever there\u0027s a handoff, yeah? I\u0027m not sure it *is* worth logging in that case...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":647,"context_line":"        \"\"\""},{"line_number":648,"context_line":"        The main function, everything the sharder does forks from this method."},{"line_number":649,"context_line":""},{"line_number":650,"context_line":"        The sharder loops through each sharded container on server, on each"},{"line_number":651,"context_line":"        container it:"},{"line_number":652,"context_line":"            - audits the container"},{"line_number":653,"context_line":"            - checks and deals with misplaced items"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ff04ee4b","line":650,"updated":"2017-04-28 01:38:47.000000000","message":"\"each container with sharding enabled\", surely? If they were already sharded, the hard part\u0027s over, right?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":656,"context_line":"                  to next container."},{"line_number":657,"context_line":"                - Phase 2, if there is a pivot defined, shard it."},{"line_number":658,"context_line":"            - Shrinking (check to see if we need to shrink this container)."},{"line_number":659,"context_line":"        :param reported:"},{"line_number":660,"context_line":"        \"\"\""},{"line_number":661,"context_line":"        self.logger.info(_(\u0027Starting container sharding cycle\u0027))"},{"line_number":662,"context_line":"        dirs \u003d []"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9ff72a5a","line":659,"updated":"2017-04-28 01:38:47.000000000","message":"doc","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":661,"context_line":"        self.logger.info(_(\u0027Starting container sharding cycle\u0027))"},{"line_number":662,"context_line":"        dirs \u003d []"},{"line_number":663,"context_line":"        self.shard_brokers \u003d dict()"},{"line_number":664,"context_line":"        self.shard_cleanups \u003d dict()"},{"line_number":665,"context_line":"        self._local_device_ids \u003d set()"},{"line_number":666,"context_line":"        self.ips \u003d whataremyips()"},{"line_number":667,"context_line":"        for node in self.ring.devs:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_e46a2318","line":664,"updated":"2017-04-28 01:38:47.000000000","message":"Maybe drop these, since we reset them on L721-721? Or move those lines up, above the call to _misplaced_objects?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":676,"context_line":"                datadir \u003d os.path.join(self.root, node[\u0027device\u0027], self.datadir)"},{"line_number":677,"context_line":"                if os.path.isdir(datadir):"},{"line_number":678,"context_line":"                    self._local_device_ids.add(node[\u0027id\u0027])"},{"line_number":679,"context_line":"                    dirs.append((datadir, node[\u0027id\u0027]))"},{"line_number":680,"context_line":"        for part, path, node_id, node_idx in self.roundrobin_datadirs(dirs):"},{"line_number":681,"context_line":"            broker \u003d ContainerBroker(path)"},{"line_number":682,"context_line":"            sharded \u003d broker.metadata.get(\u0027X-Container-Sysmeta-Sharding\u0027) or \\"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9f8ccad7","line":679,"updated":"2017-04-28 01:38:47.000000000","message":"This feels a lot like _get_local_devices...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":680,"context_line":"        for part, path, node_id, node_idx in self.roundrobin_datadirs(dirs):"},{"line_number":681,"context_line":"            broker \u003d ContainerBroker(path)"},{"line_number":682,"context_line":"            sharded \u003d broker.metadata.get(\u0027X-Container-Sysmeta-Sharding\u0027) or \\"},{"line_number":683,"context_line":"                broker.metadata.get(\u0027X-Container-Sysmeta-Shard-Account\u0027)"},{"line_number":684,"context_line":"            if not sharded:"},{"line_number":685,"context_line":"                # Not a shard container"},{"line_number":686,"context_line":"                continue"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_1f19d920","line":683,"updated":"2017-04-28 01:38:47.000000000","message":"Where does\n\n X-Container-Sharding: on\n\ncome in? Or was that getting translated to\n\n X-Container-Sysmeta-Sharding\n\nnow? I forget...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":687,"context_line":"            self.ranges \u003d []"},{"line_number":688,"context_line":"            self.node_idx \u003d node_idx"},{"line_number":689,"context_line":"            self.node_id \u003d node_id"},{"line_number":690,"context_line":"            self.part \u003d int(part)"},{"line_number":691,"context_line":"            root_account, root_container \u003d \\"},{"line_number":692,"context_line":"                ContainerSharder.get_shard_root_path(broker)"},{"line_number":693,"context_line":"            pivot \u003d ContainerSharder.get_pivot_range(broker)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ffb2ad48","line":690,"updated":"2017-04-28 01:38:47.000000000","message":"So we\u0027ve got all this per-path data dangling off self (including state, down below), but then broker we schlep all over the place?\n\nI kinda wonder if we want some SharderContext or something for the per-path stuff... Like, I don\u0027t want to have to pass all of this stuff around as args, but I don\u0027t especially like the idea of putting it all on the main ContainerSharder, either...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":701,"context_line":"            # quarantine it."},{"line_number":702,"context_line":"         #   if not self._audit_shard_container(broker, pivot, root_account,"},{"line_number":703,"context_line":"          #                                     root_container):"},{"line_number":704,"context_line":"           #     continue"},{"line_number":705,"context_line":""},{"line_number":706,"context_line":"            # now look and deal with misplaced objects."},{"line_number":707,"context_line":"            self._misplaced_objects(broker, root_account, root_container,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ff606e03","line":704,"updated":"2017-04-28 01:38:47.000000000","message":"No auditing for now?\n\nWould that actually be better put in the container auditor?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":705,"context_line":""},{"line_number":706,"context_line":"            # now look and deal with misplaced objects."},{"line_number":707,"context_line":"            self._misplaced_objects(broker, root_account, root_container,"},{"line_number":708,"context_line":"                                    pivot)"},{"line_number":709,"context_line":""},{"line_number":710,"context_line":"            if broker.is_deleted():"},{"line_number":711,"context_line":"                # This container is deleted so we can skip it. We still want"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_7f245daf","line":708,"updated":"2017-04-28 01:38:47.000000000","message":"I feel like you might want to separately tune actively sharding large containers vs. dealing with misplaced listings -- would it be worth separating the two? The latter makes me think a bit of the container-reconciler -- we\u0027re dealing with an abnormal situation that *hopefully* shouldn\u0027t generally come up...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":710,"context_line":"            if broker.is_deleted():"},{"line_number":711,"context_line":"                # This container is deleted so we can skip it. We still want"},{"line_number":712,"context_line":"                # deleted containers to go via misplaced items, cause they may"},{"line_number":713,"context_line":"                # have new objects in sitting in them that may need to move."},{"line_number":714,"context_line":"                continue"},{"line_number":715,"context_line":""},{"line_number":716,"context_line":"            self.state \u003d broker.get_db_state()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_7ff57dec","line":713,"updated":"2017-04-28 01:38:47.000000000","message":"+1 on the comment. I think I may have left thoughts along the same lines up above... so that\u0027s promising. I think I\u0027m on the right track :-)\n\ns/objects in sitting/objects sitting/","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":731,"context_line":"                # container requires sharding.. that is big enough and"},{"line_number":732,"context_line":"                # sharding hasn\u0027t started yet. Then we need to identify a"},{"line_number":733,"context_line":"                # primary node to scan for shards. The others will wait for"},{"line_number":734,"context_line":"                # shards to appear in there pivot_ranges table"},{"line_number":735,"context_line":"                # (via replication). They will start the sharding process"},{"line_number":736,"context_line":""},{"line_number":737,"context_line":"                if self.state \u003d\u003d DB_STATE_UNSHARDED:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9fda8948","line":734,"updated":"2017-04-28 01:38:47.000000000","message":"s/there/their/","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":742,"context_line":"                            (self.shard_container_size *"},{"line_number":743,"context_line":"                             self.shard_shrink_point):"},{"line_number":744,"context_line":"                        # Shrink"},{"line_number":745,"context_line":"                        # self._shrink(broker, root_account, root_container)"},{"line_number":746,"context_line":"                        continue"},{"line_number":747,"context_line":"                    elif obj_count \u003c\u003d self.shard_container_size:"},{"line_number":748,"context_line":"                        continue"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_dfa251ae","line":745,"updated":"2017-04-28 01:38:47.000000000","message":"What will it take for us to feel comfortable turning on shrinking?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":769,"context_line":"                    if scan_complete:"},{"line_number":770,"context_line":"                        self._shard_on_pivot(broker, root_account,"},{"line_number":771,"context_line":"                                             root_container)"},{"line_number":772,"context_line":"                else:"},{"line_number":773,"context_line":"                    self._shard_on_pivot(broker, root_account,"},{"line_number":774,"context_line":"                                         root_container)"},{"line_number":775,"context_line":"            finally:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_3fa13595","line":772,"updated":"2017-04-28 01:38:47.000000000","message":"So if we aren\u0027t the scanner, but we haven\u0027t gotten any pivots yet, either, at what point do we give up and unset X-Container-Sysmeta-Shard-Scanner so someone else can take over scanning?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":778,"context_line":"                self.logger.increment(\u0027scanned\u0027)"},{"line_number":779,"context_line":"                self.containers_scanned +\u003d 1"},{"line_number":780,"context_line":""},{"line_number":781,"context_line":"        # wipe out the cache do disable bypass in delete_db"},{"line_number":782,"context_line":"        cleanups \u003d self.shard_cleanups"},{"line_number":783,"context_line":"        self.shard_cleanups \u003d None"},{"line_number":784,"context_line":"        if cleanups:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_dfded10f","line":781,"updated":"2017-04-28 01:38:47.000000000","message":"Huh. I kinda expected this to be indented and part of the loop...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":789,"context_line":"                self.cpool.spawn(self.delete_db, container)"},{"line_number":790,"context_line":""},{"line_number":791,"context_line":"        # Now we wait for all threads to finish."},{"line_number":792,"context_line":"        all(self.cpool)"},{"line_number":793,"context_line":""},{"line_number":794,"context_line":"        self.logger.info(_(\u0027Finished container sharding cycle\u0027))"},{"line_number":795,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9f7a8ab7","line":792,"updated":"2017-04-28 01:38:47.000000000","message":"Why all() here, but any() above? Surely one of these is only waiting for *one* thread to finish...","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":793,"context_line":""},{"line_number":794,"context_line":"        self.logger.info(_(\u0027Finished container sharding cycle\u0027))"},{"line_number":795,"context_line":""},{"line_number":796,"context_line":"        if time.time() - reported \u003e\u003d 3600:  # once an hour"},{"line_number":797,"context_line":"            self.logger.info("},{"line_number":798,"context_line":"                _(\u0027Since %(time)s Stats: %(scanned)s scanned, \u0027"},{"line_number":799,"context_line":"                  \u0027%(sharded)s sharded, %(shrunk)s shrunk, \u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ffbf4da7","line":796,"updated":"2017-04-28 01:38:47.000000000","message":"How long does it take to assign pivots? I feel like we may want to dump stats during that loop, too....\n\nAlso, I was kinda expecting this to be part of the roundrobin_datadirs loop.","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ad1f009387f21c7311284e50adcb1cdc49743bb2","unresolved":false,"context_lines":[{"line_number":825,"context_line":"            self.container_pivots \u003d 0"},{"line_number":826,"context_line":"            self.containers_misplaced \u003d 0"},{"line_number":827,"context_line":"            self.containers_audit_failed \u003d 0"},{"line_number":828,"context_line":"            self.containers_failed \u003d 0"},{"line_number":829,"context_line":"        return reported"},{"line_number":830,"context_line":""},{"line_number":831,"context_line":"    def _send_request(self, ip, port, contdevice, partition, op, path,"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_7f021d71","line":828,"updated":"2017-04-28 01:38:47.000000000","message":"Why not use _zero_stats()?","commit_id":"5b331bd36ffb4dc333d3abecadff34cf09350a19"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":158,"context_line":"        \"\"\"Zero out the stats.\"\"\""},{"line_number":159,"context_line":"        # TODO check that none of the replicator stats are useful"},{"line_number":160,"context_line":"        # super(ContainerSharder, self)._zero_stats()"},{"line_number":161,"context_line":"        self.stats \u003d defaultdict(int)"},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"    def _report_stats(self):"},{"line_number":164,"context_line":"        stats \u003d self.stats"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_3301aa07","line":161,"updated":"2017-09-07 00:17:46.000000000","message":"Ah, right! *This* is how the failure_nodes key can be an int instead of a dict...\n\nRegardless of whether the stat is useful or not, we need to not go throwing AttributeErrors when we hit a failure...","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0e3ee9cb7be1642fb46fad96a411e5208eb6cbdf","unresolved":false,"context_lines":[{"line_number":272,"context_line":"            try:"},{"line_number":273,"context_line":"                if isinstance(item, PivotRange):"},{"line_number":274,"context_line":"                    item \u003d ("},{"line_number":275,"context_line":"                        item.name, item.timestamp, item.lower, item.upper,"},{"line_number":276,"context_line":"                        item.object_count, item.bytes_used,"},{"line_number":277,"context_line":"                        item.meta_timestamp)"},{"line_number":278,"context_line":"                ts_str \u003d item[1]"},{"line_number":279,"context_line":"                if delete:"},{"line_number":280,"context_line":"                    # Generate a new delete timestamp based off the existing"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_27d3c13a","line":277,"range":{"start_line":275,"start_character":24,"end_line":277,"end_character":44},"updated":"2017-09-06 02:05:20.000000000","message":"I\u0027ve had to call the .internal method to make the timestamps work as merge_items in the backend are expecting a proper string representations in","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":679,"context_line":"                - Phase 1, if there is no pivot defined, find it, then move"},{"line_number":680,"context_line":"                  to next container."},{"line_number":681,"context_line":"                - Phase 2, if there is a pivot defined, shard it."},{"line_number":682,"context_line":"            - Shrinking (check to see if we need to shrink this container)."},{"line_number":683,"context_line":"        \"\"\""},{"line_number":684,"context_line":"        self.logger.info(\u0027Starting container sharding cycle\u0027)"},{"line_number":685,"context_line":"        dirs \u003d []"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_b310fa01","line":682,"updated":"2017-09-07 00:17:46.000000000","message":"I wonder if we should add some way to configure how much of a single cycle gets completed, if only for the sake of probe tests. For example, I\u0027d really like to be able to arrest sharding and make assertions with both direct and external clients when\n\n- we have a pivot db file but no pivots\n- we have a pivot db file with some pivots, but not enough to cover the object namespace\n- we made one shard but it\u0027s still local (i.e., is a handoff)\n\nHmm... I wonder if we\u0027ll need to increase the recommended number of devices (or even \"servers\") in the SAIO container ring to make it more likely that the shards really *are* on handoffs when they\u0027re local...","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f6594e2ab7d6783d7aee39a095c1eee263b045af","unresolved":false,"context_lines":[{"line_number":957,"context_line":"            return"},{"line_number":958,"context_line":""},{"line_number":959,"context_line":"        self.logger.info(\u0027Best pivot scanner for %s/%s is node %d\u0027,"},{"line_number":960,"context_line":"                         broker.account, broker.container, scanner_id[0])"},{"line_number":961,"context_line":"        self.logger.increment(\u0027scanner_searches\u0027)"},{"line_number":962,"context_line":""},{"line_number":963,"context_line":"        # return scanner_id[0]"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_bff33fe7","line":960,"updated":"2017-09-06 00:55:01.000000000","message":"I\u0027m concerned about how this behaves when there are concurrent sharders. In playing with probe tests, I\u0027m seeing lines like\n\n $ grep \u0027pivot scanner\u0027 /var/log/syslog\n Sep  5 22:40:07 saio container-6041: Best pivot scanner for AUTH_test/container-4a65f760-fc66-4b76-bdc8-08200de2a8ec is node 1\n Sep  5 22:40:07 saio container-6011: Best pivot scanner for AUTH_test/container-4a65f760-fc66-4b76-bdc8-08200de2a8ec is node 2\n Sep  5 22:40:07 saio container-6021: Best pivot scanner for AUTH_test/container-4a65f760-fc66-4b76-bdc8-08200de2a8ec is node 0\n\n(that is, each node thinks it is itself the best pivot scanner)\n\nThankfully (?), none of them actually do any sharding, though...","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0e3ee9cb7be1642fb46fad96a411e5208eb6cbdf","unresolved":false,"context_lines":[{"line_number":957,"context_line":"            return"},{"line_number":958,"context_line":""},{"line_number":959,"context_line":"        self.logger.info(\u0027Best pivot scanner for %s/%s is node %d\u0027,"},{"line_number":960,"context_line":"                         broker.account, broker.container, scanner_id[0])"},{"line_number":961,"context_line":"        self.logger.increment(\u0027scanner_searches\u0027)"},{"line_number":962,"context_line":""},{"line_number":963,"context_line":"        # return scanner_id[0]"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_673549da","line":960,"in_reply_to":"7f287b81_bff33fe7","updated":"2017-09-06 02:05:20.000000000","message":"Lol, well that\u0027s good then :P\n\nHmm, they must all be getting the same counts and checking at the same time so the scanner metadata isn\u0027t set. So yeah, maybe we need to have a better election strategy.\n\nOn the plus side, next time they come around they should check their metadata and see if they are still have quroum as a the scanner and then continue, in which case, the last to update the metadata would win and be the scanner.","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"1509be99dce9f6bc16346d41a0a2b71fc9c9fc4c","unresolved":false,"context_lines":[{"line_number":988,"context_line":"        part, nodes \u003d self.ring.get_nodes(account, container)"},{"line_number":989,"context_line":"        if local:"},{"line_number":990,"context_line":"            nodes \u003d [d for d in nodes"},{"line_number":991,"context_line":"                     if d[\u0027ip\u0027] not in self.ips or"},{"line_number":992,"context_line":"                     d[\u0027port\u0027] !\u003d self.port]"},{"line_number":993,"context_line":""},{"line_number":994,"context_line":"        for node in nodes:"},{"line_number":995,"context_line":"            self.cpool.spawn("}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_8fda6a75","line":992,"range":{"start_line":991,"start_character":21,"end_line":992,"end_character":44},"updated":"2017-09-07 01:42:44.000000000","message":"We should use the existing method here:\n\n  if not is_local_device(                 \n   self.ips, self.port, d[\u0027ip\u0027], d[\u0027port\u0027])]","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f6594e2ab7d6783d7aee39a095c1eee263b045af","unresolved":false,"context_lines":[{"line_number":1329,"context_line":"                self.swift.set_container_metadata(acct, cont, shrink_meta)"},{"line_number":1330,"context_line":"            self.logger.increment(\u0027shrunk_phase_1\u0027)"},{"line_number":1331,"context_line":"        except Exception as ex:"},{"line_number":1332,"context_line":"            self.logger.error(\u0027There was a problem adding the metadata %s\u0027 %"},{"line_number":1333,"context_line":"                              ex)"},{"line_number":1334,"context_line":""},{"line_number":1335,"context_line":"    def _get_merge_range(self, account, container):"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_3f098f94","line":1332,"updated":"2017-09-06 00:55:01.000000000","message":"Log at exception, so we get a traceback","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0e3ee9cb7be1642fb46fad96a411e5208eb6cbdf","unresolved":false,"context_lines":[{"line_number":1329,"context_line":"                self.swift.set_container_metadata(acct, cont, shrink_meta)"},{"line_number":1330,"context_line":"            self.logger.increment(\u0027shrunk_phase_1\u0027)"},{"line_number":1331,"context_line":"        except Exception as ex:"},{"line_number":1332,"context_line":"            self.logger.error(\u0027There was a problem adding the metadata %s\u0027 %"},{"line_number":1333,"context_line":"                              ex)"},{"line_number":1334,"context_line":""},{"line_number":1335,"context_line":"    def _get_merge_range(self, account, container):"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_87a955de","line":1332,"in_reply_to":"7f287b81_3f098f94","updated":"2017-09-06 02:05:20.000000000","message":"Nice yeah","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":1613,"context_line":"            self.stats[\u0027containers_sharded\u0027] +\u003d 1"},{"line_number":1614,"context_line":"        if pivots_done:"},{"line_number":1615,"context_line":"            broker.merge_items(broker.pivot_nodes_to_items(pivots_done))"},{"line_number":1616,"context_line":"        any(self.cpool)"},{"line_number":1617,"context_line":""},{"line_number":1618,"context_line":"        if scan_complete and not pivots_todo:"},{"line_number":1619,"context_line":"            # we\u0027ve finished sharding this container."}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_13bfce61","line":1616,"updated":"2017-09-07 00:17:46.000000000","message":"This can maybe hit a deadlock? At any rate, I had a sharder seem to hang and Sam\u0027s stack-xray script gave me\n\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d 7FE55FE945F0 \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n  File \"/usr/local/bin/swift-container-sharder\", line 6, in \u003cmodule\u003e\n    exec(compile(open(__file__).read(), __file__, \u0027exec\u0027))\n  File \"/vagrant/swift/bin/swift-container-sharder\", line 23, in \u003cmodule\u003e\n    run_daemon(ContainerSharder, conf_file, **options)\n  File \"/vagrant/swift/swift/common/daemon.py\", line 301, in run_daemon\n    DaemonStrategy(klass(conf), logger).run(once\u003donce, **kwargs)\n  File \"/vagrant/swift/swift/common/daemon.py\", line 144, in run\n    self._run(once\u003donce, **kwargs)\n  File \"/vagrant/swift/swift/common/daemon.py\", line 214, in _run\n    return self._run_inline(once, **kwargs)\n  File \"/vagrant/swift/swift/common/daemon.py\", line 138, in _run_inline\n    self.daemon.run(once\u003donce, **kwargs)\n  File \"/vagrant/swift/swift/common/daemon.py\", line 62, in run\n    self.run_once(**kwargs)\n  File \"/vagrant/swift/swift/container/sharder.py\", line 1669, in run_once\n    self._one_shard_cycle()\n  File \"/vagrant/swift/swift/container/sharder.py\", line 794, in _one_shard_cycle\n    root_container)\n  File \"/vagrant/swift/swift/container/sharder.py\", line 1616, in _shard_on_pivot\n    any(self.cpool)\n  File \"/vagrant/swift/swift/common/utils.py\", line 2830, in next\n    rv \u003d self._responses.get()\n  File \"/usr/local/lib/python2.7/dist-packages/eventlet/queue.py\", line 313, in get\n    return waiter.wait()\n  File \"/usr/local/lib/python2.7/dist-packages/eventlet/queue.py\", line 141, in wait\n    return get_hub().switch()\n  File \"/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py\", line 295, in switch\n    return self.greenlet.switch()\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d 7FE55C39CA50 \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n  File \"/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py\", line 347, in run\n    self.wait(sleep_time)\n  File \"/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py\", line 81, in wait\n    time.sleep(seconds)\n  File \"\u003cstring\u003e\", line 1, in \u003cmodule\u003e\n  File \"\u003cstring\u003e\", line 39, in \u003cmodule\u003e\n\nIn logs I saw some\n\n AttributeError: \u0027int\u0027 object has no attribute \u0027setdefault\u0027\n\nwith stacks that get back up into eventlet/hubs/poll.py...","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"22f8b3e7f1c5d2c9dce9399a42fd29de23db2076","unresolved":false,"context_lines":[{"line_number":197,"context_line":""},{"line_number":198,"context_line":"    def _get_pivot_ranges(self, account, container, newest\u003dFalse):"},{"line_number":199,"context_line":"        path \u003d self.swift.make_path(account, container) + \\"},{"line_number":200,"context_line":"            \u0027?items\u003dpivot\u0026format\u003djson\u0027"},{"line_number":201,"context_line":"        headers \u003d dict()"},{"line_number":202,"context_line":"        if newest:"},{"line_number":203,"context_line":"            headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"}],"source_content_type":"text/x-python","patch_set":15,"id":"5f2577a7_9aa431bb","line":200,"updated":"2017-09-12 18:47:02.000000000","message":"I don\u0027t think this works as is -- internal client\u0027s using a proxy-server, and the container controller currently assumes all listings are for objects.","commit_id":"ddd4465614e6310c500f840533621a6fab00c25d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2f14bef046138ef0e65ef234c8b08c3cb22a5269","unresolved":false,"context_lines":[{"line_number":953,"context_line":"        if not self._get_quorum(broker, op\u003d\u0027POST\u0027, headers\u003dheaders):"},{"line_number":954,"context_line":"            self.logger.info(\u0027Failed to set node %d as the pivot scanner \u0027"},{"line_number":955,"context_line":"                             \u0027for %s/%s on remote servers\u0027,"},{"line_number":956,"context_line":"                             scanner_id, broker.account, broker.container)"},{"line_number":957,"context_line":"            return"},{"line_number":958,"context_line":""},{"line_number":959,"context_line":"        self.logger.info(\u0027Best pivot scanner for %s/%s is node %d\u0027,"}],"source_content_type":"text/x-python","patch_set":15,"id":"5f2577a7_620a3c8b","line":956,"updated":"2017-09-08 20:25:42.000000000","message":"So part of the trouble we see with large containers is that the container server gets overloaded and can\u0027t service object update requests, right? Will it be able to service metadata updates? Do the separate tables give us enough isolation that we have some confidence this can succeed?\n\nOr do we rely on the other nodes getting to the point that they have the smaller pivot DB, and *that* gets us out of our hole enough that we can POST...","commit_id":"ddd4465614e6310c500f840533621a6fab00c25d"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2f14bef046138ef0e65ef234c8b08c3cb22a5269","unresolved":false,"context_lines":[{"line_number":960,"context_line":"                         broker.account, broker.container, scanner_id[0])"},{"line_number":961,"context_line":"        self.logger.increment(\u0027scanner_searches\u0027)"},{"line_number":962,"context_line":""},{"line_number":963,"context_line":"        # return scanner_id[0]"},{"line_number":964,"context_line":"        return self.get_metadata_item(broker,"},{"line_number":965,"context_line":"                                      \u0027X-Container-Sysmeta-Shard-Scanner\u0027)"},{"line_number":966,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"5f2577a7_a2d6e4f0","line":963,"updated":"2017-09-08 20:25:42.000000000","message":"Right, so this makes sense: get the latest and greatest from our broker to ensure that after\n\n node A does GET, sees no scanner\n node B does GET, sees no scanner\n node A updates self, sends out POST\n node B updates self, sends out POST, overwriting above\n\nnode A looks locally, returns node B, and moves on since it isn\u0027t the scanner.\n\nBut what if there\u0027s a longer gap between node B doing a GET and node B doing the POST? So we have\n\n node A does GET, sees no scanner\n node B does GET, sees no scanner\n node A updates self, sends out POST\n node A looks locally, returns self, starts scanning\n ...\n node B updates self, sends out POST\n node B looks locally, returns self, starts scanning, comes up with different pivots\n\nI wonder if, after attempting to declare itself the scanner, the sharder should just bail -- have that be all we try to do for that cycle. Then we have time for the metadata to settle before anyone tries to determine pivots.","commit_id":"ddd4465614e6310c500f840533621a6fab00c25d"}],"swift/obj/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":42,"context_line":"    DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \\"},{"line_number":43,"context_line":"    ChunkReadError, DiskFileXattrNotSupported"},{"line_number":44,"context_line":"from swift.obj import ssync_receiver"},{"line_number":45,"context_line":"from swift.common.http import is_success, HTTP_MOVED_PERMANENTLY"},{"line_number":46,"context_line":"from swift.common.base_storage_server import BaseStorageServer"},{"line_number":47,"context_line":"from swift.common.header_key_dict import HeaderKeyDict"},{"line_number":48,"context_line":"from swift.common.request_helpers import get_name_and_placement, \\"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_3d5d8178","line":45,"updated":"2017-03-30 00:23:49.000000000","message":"Unused? Ditto below.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":266,"context_line":"        headers_out[\u0027user-agent\u0027] \u003d \u0027object-server %s\u0027 % os.getpid()"},{"line_number":267,"context_line":"        full_path \u003d \u0027/%s/%s/%s\u0027 % ("},{"line_number":268,"context_line":"            headers_out.get(\u0027X-Backend-Pivot-Account\u0027, account),"},{"line_number":269,"context_line":"            headers_out.get(\u0027X-Backend-Pivot-Container\u0027, container), obj)"},{"line_number":270,"context_line":"        if all([host, partition, contdevice]):"},{"line_number":271,"context_line":"            try:"},{"line_number":272,"context_line":"                with ConnectionTimeout(self.conn_timeout):"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_014e5c6e","line":269,"updated":"2017-09-07 00:17:46.000000000","message":"For a moment, I thought about putting this in sysmeta like we do for the X-Object-Sysmeta-Container-Update-Override-* headers, but then I realized that it\u0027d be a bit strange, putting some inherently dynamic/ephemeral metadata on the object forever.","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"}],"swift/obj/updater.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":219,"context_line":"        successes \u003d update.get(\u0027successes\u0027, [])"},{"line_number":220,"context_line":"        success \u003d True"},{"line_number":221,"context_line":"        new_successes \u003d False"},{"line_number":222,"context_line":"        num_redirects \u003d 2"},{"line_number":223,"context_line":"        for i in range(num_redirects):"},{"line_number":224,"context_line":"            redirects \u003d set()"},{"line_number":225,"context_line":"            headers_out \u003d HeaderKeyDict(update[\u0027headers\u0027].copy())"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_5d3965ef","line":222,"updated":"2017-03-30 00:23:49.000000000","message":"Should this be configurable? I suppose we generally expect there to be at most two requests though; one for the root (which redirects to the shard) and one for the shard which succeeds. Even if we\u0027re splitting a shard or something, presumably the proxy was good enough to send the old shard, that\u0027ll redirect to the current, and the current will succeed.\n\nSo hard-coding it to 2 is probably fine?\n\nHow long does a shard stick around after it\u0027s either split into new shards or gets absorbed into an existing one? Will it eventually start 404ing instead of 30Xing? If that happens will we fall back to trying the root container (which presumably will redirect)?\n\nIf we *do* run out of attempts, do we persist the last-known redirect? It looks like we\u0027ll only do that if there was at least one success...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":224,"context_line":"            redirects \u003d set()"},{"line_number":225,"context_line":"            headers_out \u003d HeaderKeyDict(update[\u0027headers\u0027].copy())"},{"line_number":226,"context_line":"            if headers_out.get(\u0027X-Backend-Pivot-Account\u0027) and \\"},{"line_number":227,"context_line":"                    headers_out.get(\u0027X-Backend-Pivot-Container\u0027):"},{"line_number":228,"context_line":"                acct \u003d headers_out[\u0027X-Backend-Pivot-Account\u0027]"},{"line_number":229,"context_line":"                cont \u003d headers_out[\u0027X-Backend-Pivot-Container\u0027]"},{"line_number":230,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_bd8bd1bd","line":227,"updated":"2017-03-30 00:23:49.000000000","message":"If we only get one or the other header, this behaves differently than the object-server :-/\n\nBut we only ever expect them to come as a pair, right? Makes me wonder a little if it should be a single header -- ...-Pivot-Location: \u003cacct\u003e/\u003ccont\u003e or something?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":225,"context_line":"            headers_out \u003d HeaderKeyDict(update[\u0027headers\u0027].copy())"},{"line_number":226,"context_line":"            if headers_out.get(\u0027X-Backend-Pivot-Account\u0027) and \\"},{"line_number":227,"context_line":"                    headers_out.get(\u0027X-Backend-Pivot-Container\u0027):"},{"line_number":228,"context_line":"                acct \u003d headers_out[\u0027X-Backend-Pivot-Account\u0027]"},{"line_number":229,"context_line":"                cont \u003d headers_out[\u0027X-Backend-Pivot-Container\u0027]"},{"line_number":230,"context_line":"            else:"},{"line_number":231,"context_line":"                acct, cont \u003d update[\u0027account\u0027], update[\u0027container\u0027]"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_dde215a0","line":228,"updated":"2017-03-30 00:23:49.000000000","message":"Do these persist to the pickled update? I think we maybe want this in sysmeta like what we did for crypto\u0027s etag override header...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":246,"context_line":"                elif redirect:"},{"line_number":247,"context_line":"                    redirects.add(redirect)"},{"line_number":248,"context_line":"                else:"},{"line_number":249,"context_line":"                    success \u003d False"},{"line_number":250,"context_line":"            if redirects:"},{"line_number":251,"context_line":"                if len(redirects) \u003e 1:"},{"line_number":252,"context_line":"                    redirect \u003d map(max(lambda x: x[-1], redirects))"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_88c9e926","line":249,"updated":"2017-03-30 00:23:49.000000000","message":"Interesting... so assuming 3 replicas, we could make an initial update, get back\n\n 2xx, 2xx, 3xx\n\ngo follow the redirect, spin up *another* three requests for the new location, and *all three* have to be successful before we\u0027ll consider our job done?\n\nI suppose it\u0027s fair; if we only got back one, we won\u0027t know whether that container is absorbing records from one of the old containers that gave success. To keep our durability guarantees, it\u0027s best to require a fresh set of acks when we change locations.\n\n...but if the original set of updates come back\n\n 2xx, 3xx, timeout\n\nWe\u0027ll have to retry later even if the redirected updates all succeed?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":249,"context_line":"                    success \u003d False"},{"line_number":250,"context_line":"            if redirects:"},{"line_number":251,"context_line":"                if len(redirects) \u003e 1:"},{"line_number":252,"context_line":"                    redirect \u003d map(max(lambda x: x[-1], redirects))"},{"line_number":253,"context_line":"                else:"},{"line_number":254,"context_line":"                    redirect \u003d redirects.pop()"},{"line_number":255,"context_line":"                update[\u0027headers\u0027][\u0027X-Backend-Pivot-Account\u0027] \u003d redirect[0]"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_48a9a18d","line":252,"updated":"2017-03-30 00:23:49.000000000","message":"I think this gets us not one but two TypeErrors. How about\n\n redirect \u003d max(redirects, key\u003dlambda x: x[-1])\n\nThat *should* work even when len(redirects) \u003d\u003d 1...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":265,"context_line":"                                       \u0027redir_acct\u0027: redirect[0],"},{"line_number":266,"context_line":"                                       \u0027redir_cont\u0027: redirect[1]})"},{"line_number":267,"context_line":"            else:"},{"line_number":268,"context_line":"                break"},{"line_number":269,"context_line":""},{"line_number":270,"context_line":"        if success:"},{"line_number":271,"context_line":"            self.successes +\u003d 1"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_e8c87555","line":268,"updated":"2017-03-30 00:23:49.000000000","message":"nit: this feels a little disconnected from what gets us here. How about\n\n if not redirects:\n     break\n\nabove, then carry on handling the redirect?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":293,"context_line":"        :param op: operation performed (ex: \u0027PUT\u0027 or \u0027DELETE\u0027)"},{"line_number":294,"context_line":"        :param obj: object name being updated"},{"line_number":295,"context_line":"        :param headers_out: headers to send with the update"},{"line_number":296,"context_line":"        \"\"\""},{"line_number":297,"context_line":"        try:"},{"line_number":298,"context_line":"            redirect \u003d []"},{"line_number":299,"context_line":"            with ConnectionTimeout(self.conn_timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_48dbe14d","line":296,"updated":"2017-03-30 00:23:49.000000000","message":"We should probably start documenting the return values","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":295,"context_line":"        :param headers_out: headers to send with the update"},{"line_number":296,"context_line":"        \"\"\""},{"line_number":297,"context_line":"        try:"},{"line_number":298,"context_line":"            redirect \u003d []"},{"line_number":299,"context_line":"            with ConnectionTimeout(self.conn_timeout):"},{"line_number":300,"context_line":"                conn \u003d http_connect(node[\u0027ip\u0027], node[\u0027port\u0027], node[\u0027device\u0027],"},{"line_number":301,"context_line":"                                    part, op, obj, headers_out)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_c8193122","line":298,"updated":"2017-03-30 00:23:49.000000000","message":"nit: So this is sometimes a list and sometimes a tuple... Maybe better as\n\n redirect \u003d ()\n\n? And hoist it out of the try, so we can use it for the InternalError case rather than having a third possible return type.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":302,"context_line":"            with Timeout(self.node_timeout):"},{"line_number":303,"context_line":"                resp \u003d conn.getresponse()"},{"line_number":304,"context_line":"                resp.read()"},{"line_number":305,"context_line":"                success \u003d is_success(resp.status)"},{"line_number":306,"context_line":""},{"line_number":307,"context_line":"            if resp.status \u003d\u003d HTTP_MOVED_PERMANENTLY:"},{"line_number":308,"context_line":"                rheaders \u003d HeaderKeyDict(resp.getheaders())"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_a8ddcddd","line":305,"updated":"2017-03-30 00:23:49.000000000","message":"nit: may as well dedent this, too.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":308,"context_line":"                rheaders \u003d HeaderKeyDict(resp.getheaders())"},{"line_number":309,"context_line":"                location \u003d rheaders.get(\u0027Location\u0027)"},{"line_number":310,"context_line":"                if not location:"},{"line_number":311,"context_line":"                    # treat as a normal failure."},{"line_number":312,"context_line":"                    return success, node[\u0027id\u0027], redirect"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"                location \u003d urlparse(location).path"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_28ebfd77","line":311,"updated":"2017-03-30 00:23:49.000000000","message":"No debug logging?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":315,"context_line":"                if location.startswith(\u0027/\u0027):"},{"line_number":316,"context_line":"                    location \u003d location[1:]"},{"line_number":317,"context_line":"                piv_acc, piv_cont, obj_ \u003d location.split(\u0027/\u0027, 2)"},{"line_number":318,"context_line":"                if not piv_acc or not piv_cont:"},{"line_number":319,"context_line":"                    # there has been an error so log it and return"},{"line_number":320,"context_line":"                    self.logger.error(_(\u0027ERROR Container update failed: %s \u0027"},{"line_number":321,"context_line":"                                        \u0027possibly sharded but couldn\\\u0027t \u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_e8ae352c","line":318,"updated":"2017-03-30 00:23:49.000000000","message":"nit: maybe use split_path? Should handle the startswith stuff for us, and we can just have an\n\n except ValueError:","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4ac7abe29afa4de8503da01199642d78532d0780","unresolved":false,"context_lines":[{"line_number":317,"context_line":"                piv_acc, piv_cont, obj_ \u003d location.split(\u0027/\u0027, 2)"},{"line_number":318,"context_line":"                if not piv_acc or not piv_cont:"},{"line_number":319,"context_line":"                    # there has been an error so log it and return"},{"line_number":320,"context_line":"                    self.logger.error(_(\u0027ERROR Container update failed: %s \u0027"},{"line_number":321,"context_line":"                                        \u0027possibly sharded but couldn\\\u0027t \u0027"},{"line_number":322,"context_line":"                                        \u0027find determine shard container \u0027"},{"line_number":323,"context_line":"                                        \u0027location.\u0027) % obj)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_48e8817a","line":320,"range":{"start_line":320,"start_character":41,"end_line":320,"end_character":46},"updated":"2017-03-30 00:23:49.000000000","message":"Doesn\u0027t the `logger.error` kinda imply this?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":263,"context_line":"                    success \u003d False"},{"line_number":264,"context_line":"            if redirects:"},{"line_number":265,"context_line":"                if len(redirects) \u003e 1:"},{"line_number":266,"context_line":"                    redirect \u003d map(max(lambda x: x[-1], redirects))"},{"line_number":267,"context_line":"                else:"},{"line_number":268,"context_line":"                    redirect \u003d redirects.pop()"},{"line_number":269,"context_line":"                update[\u0027headers\u0027][\u0027X-Backend-Pivot-Account\u0027] \u003d redirect[0]"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_c1a174f9","line":266,"updated":"2017-09-07 00:17:46.000000000","message":"This looks wrong...\n\n max(map(lambda x: x[-1], redirects))\n\nmaybe? No...\n\n redirect \u003d max(redirects, key\u003dlambda x: x[-1])\n\n?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":306,"context_line":"        :param part: partition that holds the container"},{"line_number":307,"context_line":"        :param op: operation performed (ex: \u0027PUT\u0027 or \u0027DELETE\u0027)"},{"line_number":308,"context_line":"        :param obj: object name being updated"},{"line_number":309,"context_line":"        :param headers_out: headers to send with the update"},{"line_number":310,"context_line":"        \"\"\""},{"line_number":311,"context_line":"        try:"},{"line_number":312,"context_line":"            redirect \u003d []"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_21ea78d7","line":309,"updated":"2017-09-07 00:17:46.000000000","message":"This could use a\n\n :returns:","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":323,"context_line":"                location \u003d rheaders.get(\u0027Location\u0027)"},{"line_number":324,"context_line":"                if not location:"},{"line_number":325,"context_line":"                    # treat as a normal failure."},{"line_number":326,"context_line":"                    return success, node[\u0027id\u0027], redirect"},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"                location \u003d urlparse(location).path"},{"line_number":329,"context_line":"                if location.startswith(\u0027/\u0027):"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_01065ccf","line":326,"range":{"start_line":326,"start_character":48,"end_line":326,"end_character":56},"updated":"2017-09-07 00:17:46.000000000","message":"If we\u0027re treating this as a normal failure, maybe this should be None? I suppose both None and [] will fail the\n\n elif redirect:\n\nabove... *shrug*","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":346,"context_line":"                    _(\u0027Error code %(status)d is returned from remote \u0027"},{"line_number":347,"context_line":"                      \u0027server %(ip)s: %(port)s / %(device)s\u0027), resp_dict)"},{"line_number":348,"context_line":""},{"line_number":349,"context_line":"            return success, node[\u0027id\u0027], redirect"},{"line_number":350,"context_line":"        except (Exception, Timeout):"},{"line_number":351,"context_line":"            self.logger.exception(_(\u0027ERROR with remote server \u0027"},{"line_number":352,"context_line":"                                    \u0027%(ip)s:%(port)s/%(device)s\u0027), node)"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_4118e41c","line":349,"range":{"start_line":349,"start_character":40,"end_line":349,"end_character":48},"updated":"2017-09-07 00:17:46.000000000","message":"So this is either\n\n- an empty list, or\n- a tuple with three items?\n\nShould we just set it to None at the top?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ea22c6b95b4520689d66b28d883597d34da5b3a6","unresolved":false,"context_lines":[{"line_number":1924,"context_line":""},{"line_number":1925,"context_line":"        return resp"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":"    def _get_pivot_ranges(self, req, account, container, object\u003dNone):"},{"line_number":1928,"context_line":"        ranges \u003d []"},{"line_number":1929,"context_line":"        part, nodes \u003d self.app.container_ring.get_nodes(account, container)"},{"line_number":1930,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_6bc9f558","line":1927,"range":{"start_line":1927,"start_character":57,"end_line":1927,"end_character":63},"updated":"2017-03-21 23:30:26.000000000","message":"Probably don\u0027t want to shadow the builtin type object.","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76c283b3cd88ff0f9d5a9b6c0c0e4cd912e282f5","unresolved":false,"context_lines":[{"line_number":1924,"context_line":""},{"line_number":1925,"context_line":"        return resp"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":"    def _get_pivot_ranges(self, req, account, container, object\u003dNone):"},{"line_number":1928,"context_line":"        ranges \u003d []"},{"line_number":1929,"context_line":"        part, nodes \u003d self.app.container_ring.get_nodes(account, container)"},{"line_number":1930,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"1a1ced50_af4e906c","line":1927,"range":{"start_line":1927,"start_character":57,"end_line":1927,"end_character":63},"in_reply_to":"1a1ced50_6bc9f558","updated":"2017-03-22 01:13:03.000000000","message":"great call\n\ndone","commit_id":"44b6e2b3630b52e350387944668719fe6ffeea67"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":174,"context_line":"        },"},{"line_number":175,"context_line":"        \u0027meta\u0027: meta,"},{"line_number":176,"context_line":"        \u0027sysmeta\u0027: sysmeta,"},{"line_number":177,"context_line":"        \u0027sharding_state\u0027: int(headers.get(\u0027x-backend-sharding-state\u0027,"},{"line_number":178,"context_line":"                                          DB_STATE_UNSHARDED))"},{"line_number":179,"context_line":"    }"},{"line_number":180,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_eae53fdd","line":177,"range":{"start_line":177,"start_character":26,"end_line":177,"end_character":29},"updated":"2017-03-29 01:29:08.000000000","message":"I thought the type conversion usually happened in get_container_info?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":355,"context_line":"    if \u0027object_count\u0027 not in info and \u0027container_size\u0027 in info:"},{"line_number":356,"context_line":"        info[\u0027object_count\u0027] \u003d info.pop(\u0027container_size\u0027)"},{"line_number":357,"context_line":""},{"line_number":358,"context_line":"    for field in (\u0027storage_policy\u0027, \u0027bytes\u0027, \u0027object_count\u0027):"},{"line_number":359,"context_line":"        if info.get(field) is None:"},{"line_number":360,"context_line":"            info[field] \u003d 0"},{"line_number":361,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4abf8ba9","line":358,"updated":"2017-03-29 01:29:08.000000000","message":"Should we handle sharding_state around here, too? That way we can back-populate values during an upgrade.\n\nI suppose it\u0027ll require its own logic, though, since it\u0027ll default to 1 (DB_STATE_UNSHARDED).","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":1930,"context_line":""},{"line_number":1931,"context_line":"        path \u003d \"/%s/%s\" % (account, container)"},{"line_number":1932,"context_line":"        if obj:"},{"line_number":1933,"context_line":"            path \u003d \"%s/%s\" % (path, obj)"},{"line_number":1934,"context_line":"        params \u003d req.params.copy()"},{"line_number":1935,"context_line":"        params.update({"},{"line_number":1936,"context_line":"            \u0027items\u0027: \u0027pivot\u0027,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4a1b0bc9","line":1933,"range":{"start_line":1933,"start_character":36,"end_line":1933,"end_character":39},"updated":"2017-03-29 01:29:08.000000000","message":"So obj is in the path... but it only gets used because we include items\u003dpivot in the query string? And if it weren\u0027t for that, we\u0027d be doing a normal container listing...\n\nThis feels a little weird awkward as an API. Maybe have another query param like obj\u003d\u003cobj name\u003e or for\u003d\u003cobj name\u003e ?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":1936,"context_line":"            \u0027items\u0027: \u0027pivot\u0027,"},{"line_number":1937,"context_line":"            \u0027format\u0027: \u0027json\u0027})"},{"line_number":1938,"context_line":""},{"line_number":1939,"context_line":"        headers \u003d [self.generate_request_headers(req, transfer\u003dTrue)"},{"line_number":1940,"context_line":"                   for _junk in range(len(nodes))]"},{"line_number":1941,"context_line":"        piv_resp \u003d self.make_requests(req, self.app.container_ring,"},{"line_number":1942,"context_line":"                                      part, \"GET\", path, headers,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_8ad8f3ee","line":1939,"range":{"start_line":1939,"start_character":54,"end_line":1939,"end_character":62},"updated":"2017-03-29 01:29:08.000000000","message":"I\u0027m a little leery of transferring client headers given the default is False... but I\u0027m also not familiar with when we generally do vs. not. Is it required for anything?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":1954,"context_line":"                meta_timestamp \u003d pivot.get(\u0027meta_timestamp\u0027) or None"},{"line_number":1955,"context_line":"                ranges.append(PivotRange(pivot[\u0027name\u0027], created_at, lower,"},{"line_number":1956,"context_line":"                                         upper, object_count, bytes_used,"},{"line_number":1957,"context_line":"                                         meta_timestamp))"},{"line_number":1958,"context_line":"        except ValueError:"},{"line_number":1959,"context_line":"            # Failed to decode the json response"},{"line_number":1960,"context_line":"            pass"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_2d2ea1fa","line":1957,"updated":"2017-03-29 01:29:08.000000000","message":"So this is the result we want to cache. Only lookups will be awkward -- account/container are simple enough, but after that we\u0027ll need a lookup by range so that when we learn object \u0027cat\u0027 belongs in pivot \u0027aardvark..frog\u0027, we can re-use that result for \u0027dog\u0027.\n\nUnless we always get the whole container\u0027s worth of pivots and cache the entire result...\n\nI\u0027m fairly certain this will best be left as future work, but just wanted to think aloud a bit.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":1955,"context_line":"                ranges.append(PivotRange(pivot[\u0027name\u0027], created_at, lower,"},{"line_number":1956,"context_line":"                                         upper, object_count, bytes_used,"},{"line_number":1957,"context_line":"                                         meta_timestamp))"},{"line_number":1958,"context_line":"        except ValueError:"},{"line_number":1959,"context_line":"            # Failed to decode the json response"},{"line_number":1960,"context_line":"            pass"},{"line_number":1961,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_8a86b3b8","line":1958,"updated":"2017-03-29 01:29:08.000000000","message":"Could anything else raise a ValueError here? I *think* we\u0027re OK, but I might still prefer something like\n\n try:\n     body \u003d json.loads(piv_resp.body)\n except ValueError:\n     return ranges\n\n for pivot in body:\n     ...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ca5e66353b1ddc100075e4c186b40b2458a8fdb6","unresolved":false,"context_lines":[{"line_number":296,"context_line":"                break"},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"        out_content_type \u003d get_listing_content_type(req)"},{"line_number":299,"context_line":"        return create_container_listing(req, out_content_type, headers,"},{"line_number":300,"context_line":"                                        objects, self.container_name)"},{"line_number":301,"context_line":""},{"line_number":302,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":14,"id":"7f287b81_855db148","line":299,"range":{"start_line":299,"start_character":63,"end_line":299,"end_character":70},"updated":"2017-09-07 00:17:46.000000000","message":"headers, or hdrs? Or should they be merged somehow?","commit_id":"aff784b921ebe7a43f69d654870e158490bde6fe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"22f8b3e7f1c5d2c9dce9399a42fd29de23db2076","unresolved":false,"context_lines":[{"line_number":114,"context_line":"            int(resp.headers.get(\u0027X-Backend-Sharding-State\u0027,"},{"line_number":115,"context_line":"                                 DB_STATE_UNSHARDED))"},{"line_number":116,"context_line":"        if req.method \u003d\u003d \"GET\" and sharding_state in (DB_STATE_SHARDING,"},{"line_number":117,"context_line":"                                                      DB_STATE_SHARDED):"},{"line_number":118,"context_line":"            new_resp \u003d self._get_sharded(req, resp, sharding_state)"},{"line_number":119,"context_line":"            if new_resp:"},{"line_number":120,"context_line":"                resp \u003d new_resp"}],"source_content_type":"text/x-python","patch_set":15,"id":"5f2577a7_faf0d5cb","line":117,"updated":"2017-09-12 18:47:02.000000000","message":"...\n\n and not req.params.get(\u0027items\u0027) \u003d\u003d \u0027pivot\u0027\n\n? IDK, *external* clients shouldn\u0027t really care... but maybe if we had a gatekeeper option to strip it out (and an internal client conf that\u0027s configured to allow it)","commit_id":"ddd4465614e6310c500f840533621a6fab00c25d"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":210,"context_line":"        \"\"\"Handler for HTTP HEAD requests.\"\"\""},{"line_number":211,"context_line":"        return self.GETorHEAD(req)"},{"line_number":212,"context_line":""},{"line_number":213,"context_line":"    def _get_container_meta(self, req, container_info):"},{"line_number":214,"context_line":"        db_state \u003d container_info.get(\u0027sharding_state\u0027, DB_STATE_UNSHARDED)"},{"line_number":215,"context_line":"        if db_state in (DB_STATE_SHARDED, DB_STATE_SHARDING):"},{"line_number":216,"context_line":"            # find the sharded container to send the update too."}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4d444547","line":213,"updated":"2017-03-29 01:29:08.000000000","message":"OK, so this is the additional meta that will be PUT with the object, not the meta for the container we\u0027re going to be updating. Which makes me less weirded out by having that {} in the unsharded case.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":213,"context_line":"    def _get_container_meta(self, req, container_info):"},{"line_number":214,"context_line":"        db_state \u003d container_info.get(\u0027sharding_state\u0027, DB_STATE_UNSHARDED)"},{"line_number":215,"context_line":"        if db_state in (DB_STATE_SHARDED, DB_STATE_SHARDING):"},{"line_number":216,"context_line":"            # find the sharded container to send the update too."},{"line_number":217,"context_line":"            ranges \u003d self._get_pivot_ranges("},{"line_number":218,"context_line":"                req, self.account_name, self.container_name, self.object_name)"},{"line_number":219,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_aa629708","line":216,"range":{"start_line":216,"start_character":60,"end_line":216,"end_character":63},"updated":"2017-03-29 01:29:08.000000000","message":"nit: \"to\". Or maybe \"to, too\". Or maybe\n\n\u003e find the sharded container to which to send the update, as well\n\n:P","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":"            if ranges:"},{"line_number":221,"context_line":"                piv_acct \u003d account_to_pivot_account(self.account_name)"},{"line_number":222,"context_line":"                partition, containers \u003d self.app.container_ring.get_nodes("},{"line_number":223,"context_line":"                    piv_acct, ranges[0].name)"},{"line_number":224,"context_line":"                overide_prefix \u003d \u0027X-Backend-Container-Update-Override\u0027"},{"line_number":225,"context_line":"                shard \u003d ranges[0].name"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_6d34a9de","line":222,"range":{"start_line":222,"start_character":27,"end_line":222,"end_character":37},"updated":"2017-03-29 01:29:08.000000000","message":"Better named \"nodes\", to match the unsharded nomenclature.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":220,"context_line":"            if ranges:"},{"line_number":221,"context_line":"                piv_acct \u003d account_to_pivot_account(self.account_name)"},{"line_number":222,"context_line":"                partition, containers \u003d self.app.container_ring.get_nodes("},{"line_number":223,"context_line":"                    piv_acct, ranges[0].name)"},{"line_number":224,"context_line":"                overide_prefix \u003d \u0027X-Backend-Container-Update-Override\u0027"},{"line_number":225,"context_line":"                shard \u003d ranges[0].name"},{"line_number":226,"context_line":"                container_meta \u003d {"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4a01cb05","line":223,"range":{"start_line":223,"start_character":36,"end_line":223,"end_character":39},"updated":"2017-03-29 01:29:08.000000000","message":"How certain are we that only one will (or should?) be returned?\n\nWhen there are overlapping ranges, how do we ultimately reconcile them? Might we be dumping a bunch of stuff in the shard that ultimately \"loses\"?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":221,"context_line":"                piv_acct \u003d account_to_pivot_account(self.account_name)"},{"line_number":222,"context_line":"                partition, containers \u003d self.app.container_ring.get_nodes("},{"line_number":223,"context_line":"                    piv_acct, ranges[0].name)"},{"line_number":224,"context_line":"                overide_prefix \u003d \u0027X-Backend-Container-Update-Override\u0027"},{"line_number":225,"context_line":"                shard \u003d ranges[0].name"},{"line_number":226,"context_line":"                container_meta \u003d {"},{"line_number":227,"context_line":"                    \u0027%s-Backend-Pivot-Account\u0027 % overide_prefix: piv_acct,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_eda319ff","line":224,"range":{"start_line":224,"start_character":36,"end_line":224,"end_character":43},"updated":"2017-03-29 01:29:08.000000000","message":"How well does this work when the shard is down? Isn\u0027t this why we moved a bunch of stuff to X-Object-Sysmeta-Container-Update-Override-*?\n\nActually, all of the sharding work probably pre-dates https://review.openstack.org/#/c/328205 and the rest of crypto, doesn\u0027t it?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":222,"context_line":"                partition, containers \u003d self.app.container_ring.get_nodes("},{"line_number":223,"context_line":"                    piv_acct, ranges[0].name)"},{"line_number":224,"context_line":"                overide_prefix \u003d \u0027X-Backend-Container-Update-Override\u0027"},{"line_number":225,"context_line":"                shard \u003d ranges[0].name"},{"line_number":226,"context_line":"                container_meta \u003d {"},{"line_number":227,"context_line":"                    \u0027%s-Backend-Pivot-Account\u0027 % overide_prefix: piv_acct,"},{"line_number":228,"context_line":"                    \u0027%s-Backend-Pivot-Container\u0027 % overide_prefix: shard"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_ad151142","line":225,"updated":"2017-03-29 01:29:08.000000000","message":"nit: Maybe move this up so we can use it in the call to get_nodes?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":240,"context_line":"        \"\"\"HTTP POST request handler.\"\"\""},{"line_number":241,"context_line":"        container_info \u003d self.container_info("},{"line_number":242,"context_line":"            self.account_name, self.container_name, req)"},{"line_number":243,"context_line":"        container_partition, containers, container_meta \u003d \\"},{"line_number":244,"context_line":"            self._get_container_meta(req, container_info)"},{"line_number":245,"context_line":"        req.acl \u003d container_info[\u0027write_acl\u0027]"},{"line_number":246,"context_line":"        if \u0027swift.authorize\u0027 in req.environ:"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_ad24f15e","line":243,"range":{"start_line":243,"start_character":29,"end_line":243,"end_character":39},"updated":"2017-03-29 01:29:08.000000000","message":"I preferred the old name :-(","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":281,"context_line":"        headers \u003d [self.generate_request_headers(req, additional\u003dreq.headers)"},{"line_number":282,"context_line":"                   for _junk in range(n_outgoing)]"},{"line_number":283,"context_line":"        if container_meta is None:"},{"line_number":284,"context_line":"            container_meta \u003d {}"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"        def set_container_update(index, container):"},{"line_number":287,"context_line":"            headers[index][\u0027X-Container-Partition\u0027] \u003d container_partition"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_2dd58144","line":284,"updated":"2017-03-29 01:29:08.000000000","message":"I suppose it\u0027s optional for things like GET or HEAD where we don\u0027t care about shards, yeah? No reason to incur the extra requests.","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c5016fef2de2072ff4c050003ffff5a6b5a8455a","unresolved":false,"context_lines":[{"line_number":283,"context_line":"        if container_meta is None:"},{"line_number":284,"context_line":"            container_meta \u003d {}"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"        def set_container_update(index, container):"},{"line_number":287,"context_line":"            headers[index][\u0027X-Container-Partition\u0027] \u003d container_partition"},{"line_number":288,"context_line":"            headers[index][\u0027X-Container-Host\u0027] \u003d csv_append("},{"line_number":289,"context_line":"                headers[index].get(\u0027X-Container-Host\u0027),"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_4d67a5b8","line":286,"updated":"2017-03-29 01:29:08.000000000","message":"I kinda wanna see this get hoisted up into _get_container_meta (just to see all of that centralized a bit), but I guess it\u0027s hard because of the per-node parts to this?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}],"test/unit/common/test_utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":3585,"context_line":"        self.assertIsNone(inf_pr.lower)"},{"line_number":3586,"context_line":""},{"line_number":3587,"context_line":"        for x in range(100):"},{"line_number":3588,"context_line":"            self.assertTrue(x in inf_pr)"},{"line_number":3589,"context_line":""},{"line_number":3590,"context_line":"        for x in (\u0027a\u0027, \u0027z\u0027, \u0027zzzz\u0027, \u0027124fsdf\u0027, \u0027\u0027, 1234):"},{"line_number":3591,"context_line":"            self.assertTrue(x in inf_pr)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_5cacce9d","line":3588,"range":{"start_line":3588,"start_character":28,"end_line":3588,"end_character":29},"updated":"2017-03-29 23:15:54.000000000","message":"str(x)? chr(x)?\n\nShould we be more careful about which types we allow for \u003c, \u003e, and in?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":3609,"context_line":"        # test range \u003e and \u003c"},{"line_number":3610,"context_line":"        self.assertTrue(atof \u003c ftol)"},{"line_number":3611,"context_line":"        self.assertTrue(ltor \u003e ftol)"},{"line_number":3612,"context_line":"        self.assertFalse(rtoz \u003c atof)"},{"line_number":3613,"context_line":"        self.assertFalse(ltor \u003e rtoz)"},{"line_number":3614,"context_line":""},{"line_number":3615,"context_line":"        # test range \u003c and \u003e to an item"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_5df16500","line":3612,"updated":"2017-03-29 23:15:54.000000000","message":"Maybe worth adding more non-adjacent cases?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":3617,"context_line":"        # actually included"},{"line_number":3618,"context_line":"        self.assertTrue(ftol \u003e \u0027f\u0027)"},{"line_number":3619,"context_line":"        self.assertFalse(atof \u003c \u0027f\u0027)"},{"line_number":3620,"context_line":"        self.assertTrue(ltor \u003c \u0027y\u0027)"},{"line_number":3621,"context_line":""},{"line_number":3622,"context_line":"        # Now test ranges with only 1 boundry"},{"line_number":3623,"context_line":"        for x in (\u0027l\u0027, \u0027m\u0027, \u0027z\u0027, \u0027zzz1231sd\u0027):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_bc2d1aff","line":3620,"updated":"2017-03-29 23:15:54.000000000","message":"Should we repeat these with swapped operators? Like\n\n self.assertFalse(ftol \u003c \u0027f\u0027)\n self.assertFalse(atof \u003e \u0027f\u0027)  # !\n self.assertFalse(ltor \u003e \u0027y\u0027)\n\nAnd maybe swapped operands, too?\n\n self.assertTrue(\u0027f\u0027 \u003c ftol)\n self.assertFalse(\u0027f\u0027 \u003e atof)\n self.assertTrue(\u0027y\u0027 \u003e ltor)","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":3650,"context_line":"        rtoz \u003d utils.PivotRange(\u0027r-z\u0027, ts, \u0027r\u0027, \u0027z\u0027)"},{"line_number":3651,"context_line":"        ranges \u003d [atof, ftol, ltor, rtoz]"},{"line_number":3652,"context_line":""},{"line_number":3653,"context_line":"        range \u003d utils.find_pivot_range(\u0027b\u0027, ranges)"},{"line_number":3654,"context_line":"        self.assertEqual(range, atof)"},{"line_number":3655,"context_line":"        range \u003d utils.find_pivot_range(\u0027f\u0027, ranges)"},{"line_number":3656,"context_line":"        self.assertEqual(range, atof)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_da90dbef","line":3653,"range":{"start_line":3653,"start_character":8,"end_line":3653,"end_character":13},"updated":"2017-03-29 23:15:54.000000000","message":"Let\u0027s not shadow range()","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"eef529f6362e66514e3e8a7e0df93b74baf91aa3","unresolved":false,"context_lines":[{"line_number":3658,"context_line":"        self.assertEqual(range, rtoz)"},{"line_number":3659,"context_line":"        range \u003d utils.find_pivot_range(\u0027r\u0027, ranges)"},{"line_number":3660,"context_line":"        self.assertEqual(range, ltor)"},{"line_number":3661,"context_line":""},{"line_number":3662,"context_line":"    def test_modify_priority(self):"},{"line_number":3663,"context_line":"        pid \u003d os.getpid()"},{"line_number":3664,"context_line":"        logger \u003d debug_logger()"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_7d3c2992","line":3661,"updated":"2017-03-29 23:15:54.000000000","message":"No tests with upper/lower \u003d None? Overlapping ranges?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}],"test/unit/container/test_backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":433,"context_line":"            self.assertEqual(conn.execute("},{"line_number":434,"context_line":"                \"SELECT deleted FROM object\").fetchone()[0], 0)"},{"line_number":435,"context_line":""},{"line_number":436,"context_line":"    def test_make_tuple_for_pickle(self):"},{"line_number":437,"context_line":"        record \u003d {\u0027name\u0027: \u0027obj\u0027,"},{"line_number":438,"context_line":"                  \u0027created_at\u0027: \u00271234567890.12345\u0027,"},{"line_number":439,"context_line":"                  \u0027size\u0027: 42,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_95a859db","line":436,"updated":"2017-03-28 00:55:04.000000000","message":"We should have a comparable test for RECORD_TYPE_PIVOT_NODE","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":442,"context_line":"                  \u0027deleted\u0027: \u00271\u0027,"},{"line_number":443,"context_line":"                  \u0027storage_policy_index\u0027: \u00272\u0027,"},{"line_number":444,"context_line":"                  \u0027ctype_timestamp\u0027: None,"},{"line_number":445,"context_line":"                  \u0027meta_timestamp\u0027: None,"},{"line_number":446,"context_line":"                  \u0027record_type\u0027: 0}"},{"line_number":447,"context_line":"        broker \u003d ContainerBroker(\u0027:memory:\u0027, account\u003d\u0027a\u0027, container\u003d\u0027c\u0027)"},{"line_number":448,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_ceabf686","line":445,"updated":"2017-03-28 00:55:04.000000000","message":"off-topic: I kinda wish the two timestamps had distinct values...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":443,"context_line":"                  \u0027storage_policy_index\u0027: \u00272\u0027,"},{"line_number":444,"context_line":"                  \u0027ctype_timestamp\u0027: None,"},{"line_number":445,"context_line":"                  \u0027meta_timestamp\u0027: None,"},{"line_number":446,"context_line":"                  \u0027record_type\u0027: 0}"},{"line_number":447,"context_line":"        broker \u003d ContainerBroker(\u0027:memory:\u0027, account\u003d\u0027a\u0027, container\u003d\u0027c\u0027)"},{"line_number":448,"context_line":""},{"line_number":449,"context_line":"        expect \u003d (\u0027obj\u0027, \u00271234567890.12345\u0027, 42, \u0027text/plain\u0027, \u0027hash_test\u0027,"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_155fa90e","line":446,"updated":"2017-03-28 00:55:04.000000000","message":"Maybe worth using the RECORD_TYPE_OBJECT constant?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":486,"context_line":""},{"line_number":487,"context_line":"        # simulate existing pending items written with old code,"},{"line_number":488,"context_line":"        # i.e. without content_type and meta timestamps"},{"line_number":489,"context_line":"        def old_make_tuple_for_pickle(_, record):"},{"line_number":490,"context_line":"            return (record[\u0027name\u0027], record[\u0027created_at\u0027], record[\u0027size\u0027],"},{"line_number":491,"context_line":"                    record[\u0027content_type\u0027], record[\u0027etag\u0027], record[\u0027deleted\u0027],"},{"line_number":492,"context_line":"                    record[\u0027storage_policy_index\u0027])"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_ee8d9a31","line":489,"updated":"2017-03-28 00:55:04.000000000","message":"Do we now need another iteration of this? With a have-extra-timestamps-but-not-record_type pickle?\n\noff-topic: Do we test with pre-storage-policy pickles?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":626,"context_line":"    @with_tempdir"},{"line_number":627,"context_line":"    def test_get_db_state(self, tempdir):"},{"line_number":628,"context_line":"        # test that the get db state code. This checks for existence of the"},{"line_number":629,"context_line":"        # db_file and/or the pivot_fb_file."},{"line_number":630,"context_line":"        acct \u003d \u0027account\u0027"},{"line_number":631,"context_line":"        cont \u003d \u0027continer\u0027"},{"line_number":632,"context_line":"        hsh \u003d hash_path(acct, cont)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_2e44a2b6","line":629,"range":{"start_line":629,"start_character":29,"end_line":629,"end_character":42},"updated":"2017-03-28 00:55:04.000000000","message":"nit: pivot_db_file","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":638,"context_line":""},{"line_number":639,"context_line":"        # First test NOTFOUND state"},{"line_number":640,"context_line":"        broker \u003d ContainerBroker(db_path, account\u003dacct, container\u003dcont)"},{"line_number":641,"context_line":"        self.assertEqual(broker.get_db_state(), DB_STATE_NOTFOUND)"},{"line_number":642,"context_line":""},{"line_number":643,"context_line":"        # Test UNSHARDED state, that is when db_file exists and pivot_db_file"},{"line_number":644,"context_line":"        # doesn\u0027t"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_81749f69","line":641,"updated":"2017-03-28 00:55:04.000000000","message":"Maybe also make assertions about DB_STATE[broker.get_db_state()]? Like http://paste.openstack.org/show/604359/\n\nOtherwise, I don\u0027t think I see us using DB_STATE anywhere in tests...","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6de1ec492f6aa1473509758e9ccfc4297c4ca3c0","unresolved":false,"context_lines":[{"line_number":647,"context_line":""},{"line_number":648,"context_line":"        # Test the SHARDING state, this is the period when both the db_file and"},{"line_number":649,"context_line":"        # the pivot_db_file exist"},{"line_number":650,"context_line":"        piv_broker \u003d ContainerBroker(db_pivot_path, account\u003dacct,"},{"line_number":651,"context_line":"                                     container\u003dcont)"},{"line_number":652,"context_line":"        piv_broker.initialize(ts.internal, 0)"},{"line_number":653,"context_line":"        self.assertEqual(broker.get_db_state(), DB_STATE_SHARDING)"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffe62b97_9f44fa1c","line":650,"updated":"2017-03-28 00:55:04.000000000","message":"Should we make any assertions about piv_broker.get_db_state()?","commit_id":"11678e2f751fb77f10db2357ed886c846e3f87b3"}]}