)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"31af30e042790914cc71f7131340b93054967e44","unresolved":false,"context_lines":[{"line_number":9,"context_line":"In the root container it is important that sub-shards of a sharding"},{"line_number":10,"context_line":"shard are not considered as being authoritative for the purposes of"},{"line_number":11,"context_line":"listings before they have *all* been cleaved and the sharding shard is"},{"line_number":12,"context_line":"simultaneously deleted."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"However, it is useful for sub-shards to considered authoritative by a"},{"line_number":15,"context_line":"sharding shard as soon as they are individually cleaved so that"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_e698f575","line":12,"updated":"2018-03-14 19:52:08.000000000","message":"This has probably been discussed at length already somewhere, but why? I generally don\u0027t like root container exceptionalism...\n\nWhy shouldn\u0027t the root sub-shard be considered immediately authoritative, and the (SHARDING) shard is now only authoritative for the section of its namespace that isn\u0027t covered by another ACTIVE shard?","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93ec3b1b8ec44972c849d0ed4c897d8f21667d1f","unresolved":false,"context_lines":[{"line_number":9,"context_line":"In the root container it is important that sub-shards of a sharding"},{"line_number":10,"context_line":"shard are not considered as being authoritative for the purposes of"},{"line_number":11,"context_line":"listings before they have *all* been cleaved and the sharding shard is"},{"line_number":12,"context_line":"simultaneously deleted."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"However, it is useful for sub-shards to considered authoritative by a"},{"line_number":15,"context_line":"sharding shard as soon as they are individually cleaved so that"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_417cf870","line":12,"in_reply_to":"df7087c5_e698f575","updated":"2018-03-16 11:04:18.000000000","message":"Interesting. Historically I have always sought to have only a contiguous set of shard ranges be considered \u0027authoritative\u0027. But with [1] the proxy can now cope with overlapping ranges, which is what we would have if the sub-shard and shard were both active.\n\nInferring an object count in the root is trickier when ranges overlap. To be able to sum all active shards, we\u0027d need the shard to reduce its count to compensate for the new sub-shard. Maybe that is possible.\n\nI\u0027m also a fan of avoiding special case behaviours, so I\u0027m open to eliminating the need for CLEAVED. However, for time being I am happier to have CLEAVED than the subtle trickery it replaces :)\n\n[1] https://review.openstack.org/#/c/552913/3","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"31af30e042790914cc71f7131340b93054967e44","unresolved":false,"context_lines":[{"line_number":13,"context_line":""},{"line_number":14,"context_line":"However, it is useful for sub-shards to considered authoritative by a"},{"line_number":15,"context_line":"sharding shard as soon as they are individually cleaved so that"},{"line_number":16,"context_line":"listing requests to the shard can be redirected to them. Maintaining"},{"line_number":17,"context_line":"this distinction between root and shard behaviour requires careful"},{"line_number":18,"context_line":"control of how cleaved sub-shards are reported to the root."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_5f064c1e","line":16,"range":{"start_line":16,"start_character":0,"end_line":16,"end_character":16},"updated":"2018-03-14 19:52:08.000000000","message":"I was thinking more about object updates... do we do anything with redirects for listings? I thought the only actor concerned with shard-authority during listings was the container controller in the proxy-server...","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93ec3b1b8ec44972c849d0ed4c897d8f21667d1f","unresolved":false,"context_lines":[{"line_number":13,"context_line":""},{"line_number":14,"context_line":"However, it is useful for sub-shards to considered authoritative by a"},{"line_number":15,"context_line":"sharding shard as soon as they are individually cleaved so that"},{"line_number":16,"context_line":"listing requests to the shard can be redirected to them. Maintaining"},{"line_number":17,"context_line":"this distinction between root and shard behaviour requires careful"},{"line_number":18,"context_line":"control of how cleaved sub-shards are reported to the root."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_8130d0a0","line":16,"range":{"start_line":16,"start_character":0,"end_line":16,"end_character":16},"in_reply_to":"df7087c5_5f064c1e","updated":"2018-03-16 11:04:18.000000000","message":"The root will redirect updates to a CLEAVED range. \n\nThe root will redirect listing requests to only the active range. However, the shard will redirect to a CLEAVED range [1], so the listing will ultimtely be fetched form the sub-shard (since [2]).\n\nThe probe test is intended to verify this recursive redirection [3].\n\n[1] https://review.openstack.org/#/c/551356/7/swift/container/server.py@588\n[2] https://review.openstack.org/#/c/547662/\n[3] https://review.openstack.org/#/c/551356/7/test/probe/test_sharder.py@555","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"31af30e042790914cc71f7131340b93054967e44","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Before, when a shard container made its periodic update"},{"line_number":43,"context_line":"to the root, it only included its own shard range. Now it"},{"line_number":44,"context_line":"also includes other shard ranges that it knows about, unless"},{"line_number":45,"context_line":"it is shrinking."},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"This change ensures that when a donor shard has shrunk and"},{"line_number":48,"context_line":"cleaved itself to an acceptor, the acceptor will update the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_26cfcd56","line":45,"updated":"2018-03-14 19:52:08.000000000","message":"Almost certainly a good idea... but do (or should) we have any way to back-propagate, to let the source know that its other shard ranges are out of date?","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93ec3b1b8ec44972c849d0ed4c897d8f21667d1f","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Before, when a shard container made its periodic update"},{"line_number":43,"context_line":"to the root, it only included its own shard range. Now it"},{"line_number":44,"context_line":"also includes other shard ranges that it knows about, unless"},{"line_number":45,"context_line":"it is shrinking."},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"This change ensures that when a donor shard has shrunk and"},{"line_number":48,"context_line":"cleaved itself to an acceptor, the acceptor will update the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_01e920e4","line":45,"in_reply_to":"df7087c5_26cfcd56","updated":"2018-03-16 11:04:18.000000000","message":"Been on my mind to have parents always send periodic updates to their children just to be sure nothing is lost to a failed request.","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"31af30e042790914cc71f7131340b93054967e44","unresolved":false,"context_lines":[{"line_number":47,"context_line":"This change ensures that when a donor shard has shrunk and"},{"line_number":48,"context_line":"cleaved itself to an acceptor, the acceptor will update the"},{"line_number":49,"context_line":"root with its own new expanded namespace (as before) and the"},{"line_number":50,"context_line":"deleted donor shard range (which wasn\u0027t happening before)."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Change-Id: I74df5085f999095a7d38444299abf12038e62d76"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_a69bdd15","line":50,"updated":"2018-03-14 19:52:08.000000000","message":"And similarly, a shard that has grown large and cleaved all of its rows out to subshards will report not only its own deleted state, but also the active state of all the subshards.\n\nAre there other circumstances by which a shard would have multiple shard ranges? I can\u0027t think of why B-F would ever need to know about K-R...\n\nI wonder whether we ought to limit things slightly, as a guard against ourselves -- have a shard report all of the shard ranges it has *that overlap with its own range*...","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93ec3b1b8ec44972c849d0ed4c897d8f21667d1f","unresolved":false,"context_lines":[{"line_number":47,"context_line":"This change ensures that when a donor shard has shrunk and"},{"line_number":48,"context_line":"cleaved itself to an acceptor, the acceptor will update the"},{"line_number":49,"context_line":"root with its own new expanded namespace (as before) and the"},{"line_number":50,"context_line":"deleted donor shard range (which wasn\u0027t happening before)."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Change-Id: I74df5085f999095a7d38444299abf12038e62d76"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"df7087c5_817bf0f2","line":50,"in_reply_to":"df7087c5_a69bdd15","updated":"2018-03-16 11:04:18.000000000","message":"We might want to cache more shard ranges in each shard for the purposes of misplaced object handling? To do so we\u0027d need to be sure the shard did not take any action based on knowing about those other ranges.\n\nDefensive is always good and \u0027overlap\u0027 is currently a good metric [1]. Timestamps should always save us though, as in it is wasteful but harmless for a shard to report an out of date copy of any shard range.\n\n[1] https://trello.com/c/Grd1Mqr9","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":42,"context_line":"Before, when a shard container made its periodic update"},{"line_number":43,"context_line":"to the root, it only included its own shard range. Now it"},{"line_number":44,"context_line":"also includes other shard ranges that it knows about, unless"},{"line_number":45,"context_line":"it is shrinking."},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"This change ensures that when a donor shard has shrunk and"},{"line_number":48,"context_line":"cleaved itself to an acceptor, the acceptor will update the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"df7087c5_ef20374c","line":45,"updated":"2018-03-21 01:13:36.000000000","message":"boo, caveat - still sounds like huge improvement in reducing complexity overall...","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":47,"context_line":"This change ensures that when a donor shard has shrunk and"},{"line_number":48,"context_line":"cleaved itself to an acceptor, the acceptor will update the"},{"line_number":49,"context_line":"root with its own new expanded namespace (as before) and the"},{"line_number":50,"context_line":"deleted donor shard range (which wasn\u0027t happening before)."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"Change-Id: I74df5085f999095a7d38444299abf12038e62d76"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"df7087c5_af1a3f14","line":50,"updated":"2018-03-21 01:13:36.000000000","message":"not to self: how is this related?  is this an orthogonal drive-by change?","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"}],"swift/common/utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":4318,"context_line":"    EXPANDING \u003d 4"},{"line_number":4319,"context_line":"    SHRINKING \u003d 5"},{"line_number":4320,"context_line":"    SHARDING \u003d 6"},{"line_number":4321,"context_line":"    SHARDED \u003d 7"},{"line_number":4322,"context_line":"    STATES \u003d {FOUND: \u0027found\u0027,"},{"line_number":4323,"context_line":"              CREATED: \u0027created\u0027,"},{"line_number":4324,"context_line":"              CLEAVED: \u0027cleaved\u0027,"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_b2c4e841","line":4321,"updated":"2018-03-21 01:13:36.000000000","message":"these are different than the states in container.backend","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":4318,"context_line":"    EXPANDING \u003d 4"},{"line_number":4319,"context_line":"    SHRINKING \u003d 5"},{"line_number":4320,"context_line":"    SHARDING \u003d 6"},{"line_number":4321,"context_line":"    SHARDED \u003d 7"},{"line_number":4322,"context_line":"    STATES \u003d {FOUND: \u0027found\u0027,"},{"line_number":4323,"context_line":"              CREATED: \u0027created\u0027,"},{"line_number":4324,"context_line":"              CLEAVED: \u0027cleaved\u0027,"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_91c67916","line":4321,"in_reply_to":"df7087c5_b2c4e841","updated":"2018-03-21 10:38:49.000000000","message":"yes. ShardRange state and db state are separate. db state can vary per replica. ShardRange state is made consistent across replicas. Maybe the two could converge, particularly once we have per db ShardRange records, but its not a trivial step.","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"}],"swift/container/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"31af30e042790914cc71f7131340b93054967e44","unresolved":false,"context_lines":[{"line_number":1690,"context_line":"    def get_shard_usage(self):"},{"line_number":1691,"context_line":"        states \u003d [ShardRange.ACTIVE, ShardRange.SHRINKING,"},{"line_number":1692,"context_line":"                  ShardRange.EXPANDING]"},{"line_number":1693,"context_line":"        if not self.is_root_container():"},{"line_number":1694,"context_line":"            states.append(ShardRange.CLEAVED)"},{"line_number":1695,"context_line":"        shard_ranges \u003d self.get_shard_ranges(states\u003dstates)"},{"line_number":1696,"context_line":"        return {\u0027bytes_used\u0027: sum([sr.bytes_used for sr in shard_ranges]),"}],"source_content_type":"text/x-python","patch_set":7,"id":"df7087c5_466a01ed","line":1693,"updated":"2018-03-14 19:52:08.000000000","message":"Does this function even make much sense in a non-root container? When would we roll up usage stats for a SHARDING shard?","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93ec3b1b8ec44972c849d0ed4c897d8f21667d1f","unresolved":false,"context_lines":[{"line_number":1690,"context_line":"    def get_shard_usage(self):"},{"line_number":1691,"context_line":"        states \u003d [ShardRange.ACTIVE, ShardRange.SHRINKING,"},{"line_number":1692,"context_line":"                  ShardRange.EXPANDING]"},{"line_number":1693,"context_line":"        if not self.is_root_container():"},{"line_number":1694,"context_line":"            states.append(ShardRange.CLEAVED)"},{"line_number":1695,"context_line":"        shard_ranges \u003d self.get_shard_ranges(states\u003dstates)"},{"line_number":1696,"context_line":"        return {\u0027bytes_used\u0027: sum([sr.bytes_used for sr in shard_ranges]),"}],"source_content_type":"text/x-python","patch_set":7,"id":"df7087c5_04a5aea1","line":1693,"in_reply_to":"df7087c5_466a01ed","updated":"2018-03-16 11:04:18.000000000","message":"yeah its a bit at odds with the call site, but might be useful (and is currently a \u0027public\u0027 method) so we should have it do the right thing.","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":1686,"context_line":"            info.get(\u0027object_count\u0027, 0), info.get(\u0027bytes_used\u0027, 0), now)"},{"line_number":1687,"context_line":"        return own_shard_range"},{"line_number":1688,"context_line":""},{"line_number":1689,"context_line":"    # TODO: add unit test"},{"line_number":1690,"context_line":"    def get_shard_usage(self):"},{"line_number":1691,"context_line":"        states \u003d [ShardRange.ACTIVE, ShardRange.SHRINKING,"},{"line_number":1692,"context_line":"                  ShardRange.EXPANDING]"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_f2ca604d","line":1689,"updated":"2018-03-21 01:13:36.000000000","message":"+1","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"}],"swift/container/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"31af30e042790914cc71f7131340b93054967e44","unresolved":false,"context_lines":[{"line_number":589,"context_line":"                    listing.add(\u0027cleaved\u0027)"},{"line_number":590,"context_line":"                updating \u003d {\u0027created\u0027, \u0027cleaved\u0027, \u0027active\u0027, \u0027shrinking\u0027,"},{"line_number":591,"context_line":"                            \u0027expanding\u0027}"},{"line_number":592,"context_line":"                aliases \u003d {\u0027listing\u0027: listing, \u0027updating\u0027: updating}"},{"line_number":593,"context_line":"                for alias, aliased_states in aliases.items():"},{"line_number":594,"context_line":"                    if alias in states:"},{"line_number":595,"context_line":"                        states.remove(alias)"}],"source_content_type":"text/x-python","patch_set":7,"id":"df7087c5_66486586","line":592,"updated":"2018-03-14 19:52:08.000000000","message":"The aliases always seemed like a good idea... but it bugs me a little that their meaning can change server-side. Like, surely the proxy making the request already knows whether it\u0027s asking a root or a shard, right?","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93ec3b1b8ec44972c849d0ed4c897d8f21667d1f","unresolved":false,"context_lines":[{"line_number":589,"context_line":"                    listing.add(\u0027cleaved\u0027)"},{"line_number":590,"context_line":"                updating \u003d {\u0027created\u0027, \u0027cleaved\u0027, \u0027active\u0027, \u0027shrinking\u0027,"},{"line_number":591,"context_line":"                            \u0027expanding\u0027}"},{"line_number":592,"context_line":"                aliases \u003d {\u0027listing\u0027: listing, \u0027updating\u0027: updating}"},{"line_number":593,"context_line":"                for alias, aliased_states in aliases.items():"},{"line_number":594,"context_line":"                    if alias in states:"},{"line_number":595,"context_line":"                        states.remove(alias)"}],"source_content_type":"text/x-python","patch_set":7,"id":"df7087c5_2186a4ba","line":592,"in_reply_to":"df7087c5_66486586","updated":"2018-03-16 11:04:18.000000000","message":"actually the whole point of the aliases is that their Interpretation can change server side. For example if we decide that the root exception for cleaved is unnecessary, or that cleaved is unnecessary, we just change it here without worrying about an proxy vs container-server upgrade issue.\n\nAnother example that has been on my mind - we might decide not to redirect updates until a shard has cleaved. If so, just change the alias here.\n\nWhat bugs me is that I have strngs here when there are perfectly good constants defined in ShardRange! But that is because the API uses strings not ints for state.","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":585,"context_line":"                if not broker.is_root_container():"},{"line_number":586,"context_line":"                    listing.add(\u0027cleaved\u0027)"},{"line_number":587,"context_line":"                updating \u003d {\u0027created\u0027, \u0027cleaved\u0027, \u0027active\u0027, \u0027shrinking\u0027,"},{"line_number":588,"context_line":"                            \u0027expanding\u0027}"},{"line_number":589,"context_line":"                aliases \u003d {\u0027listing\u0027: listing, \u0027updating\u0027: updating}"},{"line_number":590,"context_line":"                for alias, aliased_states in aliases.items():"},{"line_number":591,"context_line":"                    if alias in states:"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_72bdf066","line":588,"updated":"2018-03-21 01:13:36.000000000","message":"I feel like we\u0027ve seen this state logic repeated in more than one place... e.g. get_shard_usage\n\nin *fact* this is probably getting a touch too domain specific to belong in the server instead of the backend - isn\u0027t it?","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":585,"context_line":"                if not broker.is_root_container():"},{"line_number":586,"context_line":"                    listing.add(\u0027cleaved\u0027)"},{"line_number":587,"context_line":"                updating \u003d {\u0027created\u0027, \u0027cleaved\u0027, \u0027active\u0027, \u0027shrinking\u0027,"},{"line_number":588,"context_line":"                            \u0027expanding\u0027}"},{"line_number":589,"context_line":"                aliases \u003d {\u0027listing\u0027: listing, \u0027updating\u0027: updating}"},{"line_number":590,"context_line":"                for alias, aliased_states in aliases.items():"},{"line_number":591,"context_line":"                    if alias in states:"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_31ac4d34","line":588,"in_reply_to":"df7087c5_72bdf066","updated":"2018-03-21 10:38:49.000000000","message":"There\u0027s a few places I am still unsure about the separation between server and backend. This is one. Here it felt like I should deal with the *string* aliases before passing ints to the backend query. But perhaps I should have a backend helper method.\n\nI have touched this and the backend equivalent enough times to know that there is a duplication! :/","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"742c41076d89c85c0627ed8b34b0c63c875ba349","unresolved":false,"context_lines":[{"line_number":654,"context_line":"                                state\u003dShardRange.SHARDED,"},{"line_number":655,"context_line":"                                deleted\u003d1)"},{"line_number":656,"context_line":"                            modified_shard_ranges.append(own_shard_range)"},{"line_number":657,"context_line":"                    broker.merge_shard_ranges(modified_shard_ranges)"},{"line_number":658,"context_line":""},{"line_number":659,"context_line":"                    if broker.set_sharded_state():"},{"line_number":660,"context_line":"                        state \u003d SHARDED"}],"source_content_type":"text/x-python","patch_set":6,"id":"df7087c5_a39c660e","line":657,"updated":"2018-03-13 05:45:50.000000000","message":"wow, that simplifies this bit a bunch, so much less inline comments required. I like that!","commit_id":"1ebf68bedd2d704c9906fa6f3fce90c8e02d7aac"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"31af30e042790914cc71f7131340b93054967e44","unresolved":false,"context_lines":[{"line_number":647,"context_line":"                    for sr in modified_shard_ranges:"},{"line_number":648,"context_line":"                        sr.update_state(ShardRange.ACTIVE)"},{"line_number":649,"context_line":"                    if not broker.is_root_container():"},{"line_number":650,"context_line":"                        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":651,"context_line":"                        if own_shard_range.state !\u003d ShardRange.SHARDED:"},{"line_number":652,"context_line":"                            own_shard_range \u003d own_shard_range.copy("},{"line_number":653,"context_line":"                                timestamp\u003dTimestamp.now(),"}],"source_content_type":"text/x-python","patch_set":7,"id":"df7087c5_e628f597","line":650,"updated":"2018-03-14 19:52:08.000000000","message":"Off-topic: do we need this? We already have own_shard_range defined... why are we getting it again?","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93ec3b1b8ec44972c849d0ed4c897d8f21667d1f","unresolved":false,"context_lines":[{"line_number":647,"context_line":"                    for sr in modified_shard_ranges:"},{"line_number":648,"context_line":"                        sr.update_state(ShardRange.ACTIVE)"},{"line_number":649,"context_line":"                    if not broker.is_root_container():"},{"line_number":650,"context_line":"                        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":651,"context_line":"                        if own_shard_range.state !\u003d ShardRange.SHARDED:"},{"line_number":652,"context_line":"                            own_shard_range \u003d own_shard_range.copy("},{"line_number":653,"context_line":"                                timestamp\u003dTimestamp.now(),"}],"source_content_type":"text/x-python","patch_set":7,"id":"df7087c5_24c9d257","line":650,"in_reply_to":"df7087c5_e628f597","updated":"2018-03-16 11:04:18.000000000","message":"maybe not\n\noff-off-topic: I have a nagging concern about own shard range being changed by another process while the sharder si working off an in memory copy. Need to check if anything terrible could occur if that happened.","commit_id":"b5383278bd915b31badb1c81f2f4952743112c0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":629,"context_line":"                elif (is_leader and broker.get_info()[\u0027object_count\u0027] \u003e\u003d"},{"line_number":630,"context_line":"                      self.shard_container_size):"},{"line_number":631,"context_line":"                    broker.update_sharding_info({\u0027Scan-Done\u0027: \u0027False\u0027})"},{"line_number":632,"context_line":"                    own_shard_range.update_state(ShardRange.SHARDING)"},{"line_number":633,"context_line":"                    broker.set_sharding_state(epoch\u003dTimestamp.now())"},{"line_number":634,"context_line":"                    state \u003d SHARDING"},{"line_number":635,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_6f4bc786","line":632,"updated":"2018-03-21 01:13:36.000000000","message":"it\u0027s not clear to me why this wasn\u0027t already like this?  no tests fail if I comment it out to point me in the right direction.","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":629,"context_line":"                elif (is_leader and broker.get_info()[\u0027object_count\u0027] \u003e\u003d"},{"line_number":630,"context_line":"                      self.shard_container_size):"},{"line_number":631,"context_line":"                    broker.update_sharding_info({\u0027Scan-Done\u0027: \u0027False\u0027})"},{"line_number":632,"context_line":"                    own_shard_range.update_state(ShardRange.SHARDING)"},{"line_number":633,"context_line":"                    broker.set_sharding_state(epoch\u003dTimestamp.now())"},{"line_number":634,"context_line":"                    state \u003d SHARDING"},{"line_number":635,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_2c0dd034","line":632,"in_reply_to":"df7087c5_6f4bc786","updated":"2018-03-21 10:38:49.000000000","message":"tests need improving :/","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"15ae004465098eecea833394f4b23e46f76eda48","unresolved":false,"context_lines":[{"line_number":678,"context_line":"                        sr.update_state(ShardRange.ACTIVE)"},{"line_number":679,"context_line":"                    if not broker.is_root_container():"},{"line_number":680,"context_line":"                        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":681,"context_line":"                        if own_shard_range.state !\u003d ShardRange.SHARDED:"},{"line_number":682,"context_line":"                            own_shard_range \u003d own_shard_range.copy("},{"line_number":683,"context_line":"                                timestamp\u003dTimestamp.now(),"},{"line_number":684,"context_line":"                                state\u003dShardRange.SHARDED,"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_34ebe9dc","line":681,"updated":"2018-03-20 21:47:31.000000000","message":"Can we ever get a shard range with both state\u003dSHARDED and deleted\u003d0?","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":678,"context_line":"                        sr.update_state(ShardRange.ACTIVE)"},{"line_number":679,"context_line":"                    if not broker.is_root_container():"},{"line_number":680,"context_line":"                        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":681,"context_line":"                        if own_shard_range.state !\u003d ShardRange.SHARDED:"},{"line_number":682,"context_line":"                            own_shard_range \u003d own_shard_range.copy("},{"line_number":683,"context_line":"                                timestamp\u003dTimestamp.now(),"},{"line_number":684,"context_line":"                                state\u003dShardRange.SHARDED,"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_4c560409","line":681,"in_reply_to":"df7087c5_34ebe9dc","updated":"2018-03-21 10:38:49.000000000","message":"Perhaps not, but the root shoud probably reach that state. Not sure why it doesn\u0027t, if it doesn\u0027t.","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":724,"context_line":"                        broker. get_shard_ranges("},{"line_number":725,"context_line":"                            include_own\u003dTrue,"},{"line_number":726,"context_line":"                            include_deleted\u003dTrue,"},{"line_number":727,"context_line":"                            exclude_others\u003dshrinking)"},{"line_number":728,"context_line":"                    )"},{"line_number":729,"context_line":"            self.logger.info(\u0027Finished processing %s/%s state %s\u0027,"},{"line_number":730,"context_line":"                             broker.account, broker.container,"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_2f5b8f87","line":727,"updated":"2018-03-21 01:13:36.000000000","message":"exclude_others \u003d shrinking...","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":724,"context_line":"                        broker. get_shard_ranges("},{"line_number":725,"context_line":"                            include_own\u003dTrue,"},{"line_number":726,"context_line":"                            include_deleted\u003dTrue,"},{"line_number":727,"context_line":"                            exclude_others\u003dshrinking)"},{"line_number":728,"context_line":"                    )"},{"line_number":729,"context_line":"            self.logger.info(\u0027Finished processing %s/%s state %s\u0027,"},{"line_number":730,"context_line":"                             broker.account, broker.container,"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_acd48048","line":727,"in_reply_to":"df7087c5_2f5b8f87","updated":"2018-03-21 10:38:49.000000000","message":"while shrinking, a shard has copy of the *expanded* acceptor that shouldn\u0027t be sent to the root until the shrink completes. Then the expanded acceptor and *deleted* donor can be sent to root together.\n\nI\u0027d love to not have this exception. Perhaps if we can figure out how to deal with the acceptor and donor overlapping in the root we can relax this.","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"15ae004465098eecea833394f4b23e46f76eda48","unresolved":false,"context_lines":[{"line_number":805,"context_line":"                        \u0027Unhandled exception while processing %s: %s\u0027,"},{"line_number":806,"context_line":"                        path, err)"},{"line_number":807,"context_line":"            else:"},{"line_number":808,"context_line":"                self.logger.info(\u0027Skipping %s; sharding is not enabled\u0027, path)"},{"line_number":809,"context_line":""},{"line_number":810,"context_line":"        # wipe out the cache do disable bypass in delete_db"},{"line_number":811,"context_line":"        cleanups \u003d self.shard_cleanups"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_94567506","line":808,"updated":"2018-03-20 21:47:31.000000000","message":"Off-topic: this was maybe useful while we were running down the probe-test failures, but we should drop it before merge-to-master...","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":805,"context_line":"                        \u0027Unhandled exception while processing %s: %s\u0027,"},{"line_number":806,"context_line":"                        path, err)"},{"line_number":807,"context_line":"            else:"},{"line_number":808,"context_line":"                self.logger.info(\u0027Skipping %s; sharding is not enabled\u0027, path)"},{"line_number":809,"context_line":""},{"line_number":810,"context_line":"        # wipe out the cache do disable bypass in delete_db"},{"line_number":811,"context_line":"        cleanups \u003d self.shard_cleanups"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_ecb91815","line":808,"in_reply_to":"df7087c5_94567506","updated":"2018-03-21 10:38:49.000000000","message":"+1","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"}],"test/unit/container/test_backend.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":2744,"context_line":""},{"line_number":2745,"context_line":"        # merge row for own shard range"},{"line_number":2746,"context_line":"        own_shard_range \u003d ShardRange(broker.path, next(ts_iter), \u0027l\u0027, \u0027u\u0027,"},{"line_number":2747,"context_line":"                                     state\u003dShardRange.CLEAVED)"},{"line_number":2748,"context_line":"        broker.merge_shard_ranges([own_shard_range])"},{"line_number":2749,"context_line":"        self.assertFalse(broker.get_shard_ranges())"},{"line_number":2750,"context_line":"        self.assertFalse(broker.get_shard_ranges(include_own\u003dFalse))"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_75839a57","line":2747,"updated":"2018-03-21 01:13:36.000000000","message":"is this the expected state for a sub-shard that is cleaving?","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":2744,"context_line":""},{"line_number":2745,"context_line":"        # merge row for own shard range"},{"line_number":2746,"context_line":"        own_shard_range \u003d ShardRange(broker.path, next(ts_iter), \u0027l\u0027, \u0027u\u0027,"},{"line_number":2747,"context_line":"                                     state\u003dShardRange.CLEAVED)"},{"line_number":2748,"context_line":"        broker.merge_shard_ranges([own_shard_range])"},{"line_number":2749,"context_line":"        self.assertFalse(broker.get_shard_ranges())"},{"line_number":2750,"context_line":"        self.assertFalse(broker.get_shard_ranges(include_own\u003dFalse))"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_cc58d457","line":2747,"in_reply_to":"df7087c5_75839a57","updated":"2018-03-21 10:38:49.000000000","message":"SHARDING would be a better reflection of reality\n\nfixed … https://review.openstack.org/#/c/554871","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":2826,"context_line":"        # exclude_states overrides include_own"},{"line_number":2827,"context_line":"        actual \u003d broker.get_shard_ranges(include_own\u003dTrue,"},{"line_number":2828,"context_line":"                                         exclude_states\u003dShardRange.CLEAVED,"},{"line_number":2829,"context_line":"                                         exclude_others\u003dTrue)"},{"line_number":2830,"context_line":"        self.assertFalse(actual)"},{"line_number":2831,"context_line":""},{"line_number":2832,"context_line":"        # if you ask for nothing you\u0027ll get nothing"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_95866e65","line":2829,"updated":"2018-03-21 01:13:36.000000000","message":"is this test weaker if exclude_states is .CREATED?","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"61c5efa2a5b6663404043704c81db1fefee78f37","unresolved":false,"context_lines":[{"line_number":2826,"context_line":"        # exclude_states overrides include_own"},{"line_number":2827,"context_line":"        actual \u003d broker.get_shard_ranges(include_own\u003dTrue,"},{"line_number":2828,"context_line":"                                         exclude_states\u003dShardRange.CLEAVED,"},{"line_number":2829,"context_line":"                                         exclude_others\u003dTrue)"},{"line_number":2830,"context_line":"        self.assertFalse(actual)"},{"line_number":2831,"context_line":""},{"line_number":2832,"context_line":"        # if you ask for nothing you\u0027ll get nothing"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_8c309c09","line":2829,"in_reply_to":"df7087c5_95866e65","updated":"2018-03-21 10:38:49.000000000","message":"not sure what you mean by weaker. It will fail if that change is made. Or, if own_shard_range is given state CREATED at line 2747 then this assertion would need to also change.","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"}],"test/unit/container/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf091d0f257b5475577bf093e5bf421a1ed107e5","unresolved":false,"context_lines":[{"line_number":2520,"context_line":""},{"line_number":2521,"context_line":"        # shard\u0027s shard ranges for updating"},{"line_number":2522,"context_line":"        container_path \u003d \u0027.shards_a/c\u0027"},{"line_number":2523,"context_line":"        do_test(root_path, container_path, params, expected_states)"},{"line_number":2524,"context_line":""},{"line_number":2525,"context_line":"    def test_PUT_object_update_redirected_to_shard(self):"},{"line_number":2526,"context_line":"        ts_iter \u003d make_timestamp_iter()"}],"source_content_type":"text/x-python","patch_set":10,"id":"df7087c5_8f089b91","line":2523,"updated":"2018-03-21 01:13:36.000000000","message":"cool\n\nfwiw, I was confused by root_path vs. container_path","commit_id":"173887aa169cdc7997e6f4c7a20851af69c01f5c"}]}