)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6799044337e2de2c4064a3272911c107d2ba9001","unresolved":false,"context_lines":[{"line_number":28,"context_line":"we have no DB. Wait for replication like in the unsharded case."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"Continue to be willing to create the DB when the sharder is seeding all"},{"line_number":31,"context_line":"the CREATED databases before it starts cleaving, though."},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"Change-Id: I15052f3f17999e6f432951ba7c0731dcdc9475bb"},{"line_number":34,"context_line":"Closes-Bug: #1881210"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"ff570b3c_f92c309a","line":31,"updated":"2020-06-03 17:41:01.000000000","message":"this is all great","commit_id":"e76deee4e22dc1763aad31d11ae0a5ac58869d0e"}],"swift/container/server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a7531693bba91db91fb13841b3ffcc49de87f8e0","unresolved":false,"context_lines":[{"line_number":156,"context_line":"        else:"},{"line_number":157,"context_line":"            self.auto_create_account_prefix \u003d AUTO_CREATE_ACCOUNT_PREFIX"},{"line_number":158,"context_line":"        self.shards_account_prefix \u003d ("},{"line_number":159,"context_line":"            self.auto_create_account_prefix + \u0027shards_\u0027)"},{"line_number":160,"context_line":"        if config_true_value(conf.get(\u0027allow_versions\u0027, \u0027f\u0027)):"},{"line_number":161,"context_line":"            self.save_headers.append(\u0027x-versions-location\u0027)"},{"line_number":162,"context_line":"        if \u0027allow_versions\u0027 in conf:"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_13b92dcb","line":159,"updated":"2020-06-01 23:24:01.000000000","message":"it seems somewhat regrettable that some containers in an account that starts with the \"auto create account prefix\" don\u0027t get auto created.","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"eb31a02c0d0be89ee60b2ee4d10922bb19b9fb3d","unresolved":false,"context_lines":[{"line_number":156,"context_line":"        else:"},{"line_number":157,"context_line":"            self.auto_create_account_prefix \u003d AUTO_CREATE_ACCOUNT_PREFIX"},{"line_number":158,"context_line":"        self.shards_account_prefix \u003d ("},{"line_number":159,"context_line":"            self.auto_create_account_prefix + \u0027shards_\u0027)"},{"line_number":160,"context_line":"        if config_true_value(conf.get(\u0027allow_versions\u0027, \u0027f\u0027)):"},{"line_number":161,"context_line":"            self.save_headers.append(\u0027x-versions-location\u0027)"},{"line_number":162,"context_line":"        if \u0027allow_versions\u0027 in conf:"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_8fa7c9bc","line":159,"in_reply_to":"ff570b3c_13b92dcb","updated":"2020-06-02 07:30:01.000000000","message":"yeah, wish past me knew what future me knows now :P","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a7531693bba91db91fb13841b3ffcc49de87f8e0","unresolved":false,"context_lines":[{"line_number":452,"context_line":"    def _should_autocreate(self, broker, req_timestamp, account,"},{"line_number":453,"context_line":"                           policy_index, put_shard\u003dFalse):"},{"line_number":454,"context_line":"        if not account.startswith(self.auto_create_account_prefix):"},{"line_number":455,"context_line":"            return False"},{"line_number":456,"context_line":"        if account.startswith(self.shards_account_prefix) and not put_shard:"},{"line_number":457,"context_line":"            return False"},{"line_number":458,"context_line":"        return not os.path.exists(broker.db_file)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_49b811c7","line":455,"updated":"2020-06-01 23:24:01.000000000","message":"can we imagine a reality where the shards account prefix is not a subset of the auto create accounts","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a7531693bba91db91fb13841b3ffcc49de87f8e0","unresolved":false,"context_lines":[{"line_number":455,"context_line":"            return False"},{"line_number":456,"context_line":"        if account.startswith(self.shards_account_prefix) and not put_shard:"},{"line_number":457,"context_line":"            return False"},{"line_number":458,"context_line":"        return not os.path.exists(broker.db_file)"},{"line_number":459,"context_line":""},{"line_number":460,"context_line":"    def _maybe_autocreate(self, broker, req_timestamp, account,"},{"line_number":461,"context_line":"                          policy_index, put_shard\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_e9a5a5a0","line":458,"updated":"2020-06-01 23:24:01.000000000","message":"it doesn\u0027t seem like \"os.path.exists\" should really make a difference if it should *auto* create or not","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a7531693bba91db91fb13841b3ffcc49de87f8e0","unresolved":false,"context_lines":[{"line_number":541,"context_line":"                return HTTPBadRequest(\u0027Invalid body: %r\u0027 % err)"},{"line_number":542,"context_line":"            created \u003d self._maybe_autocreate("},{"line_number":543,"context_line":"                broker, req_timestamp, account, requested_policy_index,"},{"line_number":544,"context_line":"                put_shard\u003dTrue)"},{"line_number":545,"context_line":"            self._update_metadata(req, broker, req_timestamp, \u0027PUT\u0027)"},{"line_number":546,"context_line":"            if shard_ranges:"},{"line_number":547,"context_line":"                # TODO: consider writing the shard ranges into the pending"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_a9c8cd54","line":544,"updated":"2020-06-01 23:24:01.000000000","message":"I don\u0027t really understand the implications put shard api - I thought the sharder used replication apis\n\nexcept when creating the shard db\u0027s in the first place I suppose - but in that case it doesn\u0027t need them auto-created!?","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"eb31a02c0d0be89ee60b2ee4d10922bb19b9fb3d","unresolved":false,"context_lines":[{"line_number":541,"context_line":"                return HTTPBadRequest(\u0027Invalid body: %r\u0027 % err)"},{"line_number":542,"context_line":"            created \u003d self._maybe_autocreate("},{"line_number":543,"context_line":"                broker, req_timestamp, account, requested_policy_index,"},{"line_number":544,"context_line":"                put_shard\u003dTrue)"},{"line_number":545,"context_line":"            self._update_metadata(req, broker, req_timestamp, \u0027PUT\u0027)"},{"line_number":546,"context_line":"            if shard_ranges:"},{"line_number":547,"context_line":"                # TODO: consider writing the shard ranges into the pending"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_cff9c180","line":544,"in_reply_to":"ff570b3c_a9c8cd54","updated":"2020-06-02 07:30:01.000000000","message":"Yeah there are a few times we send shards directly. like updating counts in the root container and on shard creation. It seems once the root gets to the sharded state... but I wonder if that actaully makes sense. I guess it speeds up making the shards ACTIVE, so that\u0027s good.\n\nWe use it for shrinking candidates (so atm only during autosharding)\n\nBut I wonder if at some point we relook at all thing and _only_ use the replication approach to move shards around, which could simplify things.\n\nBut that\u0027s a future improvement problem.","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"167aeb6ec85ba0b7d49eef0ff26cb57fc5c1470f","unresolved":false,"context_lines":[{"line_number":545,"context_line":"            created \u003d self._maybe_autocreate("},{"line_number":546,"context_line":"                broker, req_timestamp, account, requested_policy_index,"},{"line_number":547,"context_line":"                allow_create_shard\u003dnot any(sr.state in SHARD_LISTING_STATES"},{"line_number":548,"context_line":"                                           for sr in shard_ranges))"},{"line_number":549,"context_line":"            self._update_metadata(req, broker, req_timestamp, \u0027PUT\u0027)"},{"line_number":550,"context_line":"            if shard_ranges:"},{"line_number":551,"context_line":"                # TODO: consider writing the shard ranges into the pending"}],"source_content_type":"text/x-python","patch_set":2,"id":"ff570b3c_a99a775e","line":548,"updated":"2020-06-03 00:30:13.000000000","message":"I guess we could use something like:\n\n  allow_create_shard \u003d shard_ranges and not any(sr.state in ...)\n  created \u003d self._maybe_autocreate(... allow_create_shard\u003dallow_create_shard\n\nto make sure we don\u0027t create a shards when we have no shardranges, if we want to block that.\n\n\nBut then we\u0027d need to change the test again :P","commit_id":"d0fde3971df8b37550a7e32f5fa5cdc6f199910c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6799044337e2de2c4064a3272911c107d2ba9001","unresolved":false,"context_lines":[{"line_number":29,"context_line":"from swift.container.sync_store import ContainerSyncStore"},{"line_number":30,"context_line":"from swift.container.backend import ContainerBroker, DATADIR, \\"},{"line_number":31,"context_line":"    RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, SHARD_UPDATE_STATES, \\"},{"line_number":32,"context_line":"    SHARD_LISTING_STATES"},{"line_number":33,"context_line":"from swift.container.replicator import ContainerReplicatorRpc"},{"line_number":34,"context_line":"from swift.common.db import DatabaseAlreadyExists"},{"line_number":35,"context_line":"from swift.common.container_sync_realms import ContainerSyncRealms"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_19012406","line":32,"updated":"2020-06-03 17:41:01.000000000","message":"I think pep8 might catch this","commit_id":"e76deee4e22dc1763aad31d11ae0a5ac58869d0e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6799044337e2de2c4064a3272911c107d2ba9001","unresolved":false,"context_lines":[{"line_number":460,"context_line":"            return False"},{"line_number":461,"context_line":"        if account.startswith(self.shards_account_prefix):"},{"line_number":462,"context_line":"            return False"},{"line_number":463,"context_line":"        return True"},{"line_number":464,"context_line":""},{"line_number":465,"context_line":"    def _maybe_autocreate(self, broker, req_timestamp, account,"},{"line_number":466,"context_line":"                          policy_index, autocreate_header\u003dNone):"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_670a3826","line":463,"updated":"2020-06-03 17:41:01.000000000","message":"I find this a little more obvious and it seem to still pass unittests:\n\ndiff --git a/swift/container/server.py b/swift/container/server.py\nindex b94ae5a53..3afbbdad3 100644\n--- a/swift/container/server.py\n+++ b/swift/container/server.py\n@@ -28,8 +28,7 @@ from six.moves.urllib.parse import quote\n import swift.common.db\n from swift.container.sync_store import ContainerSyncStore\n from swift.container.backend import ContainerBroker, DATADIR, \\\n-    RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, SHARD_UPDATE_STATES, \\\n-    SHARD_LISTING_STATES\n+    RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, SHARD_UPDATE_STATES\n from swift.container.replicator import ContainerReplicatorRpc\n from swift.common.db import DatabaseAlreadyExists\n from swift.common.container_sync_realms import ContainerSyncRealms\n@@ -456,11 +455,13 @@ class ContainerController(BaseStorageServer):\n             # If the caller included an explicit X-Backend-Auto-Create header,\n             # assume they know the behavior they want\n             return config_true_value(autocreate_header)\n-        if not account.startswith(self.auto_create_account_prefix):\n-            return False\n         if account.startswith(self.shards_account_prefix):\n+            # we have to specical case this subset of the\n+            # auto_create_account_prefix because shouldn\u0027t accidently\n+            # auto-create shards from the updater; only the sharder creates\n+            # shards and tells explicitly told to do so\n             return False\n-        return True\n+        return account.startswith(self.auto_create_account_prefix)\n \n     def _maybe_autocreate(self, broker, req_timestamp, account,\n                           policy_index, autocreate_header\u003dNone):","commit_id":"e76deee4e22dc1763aad31d11ae0a5ac58869d0e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f5a4c4c3cabc05e0950306d5c090fa492b097cb6","unresolved":false,"context_lines":[{"line_number":450,"context_line":"        return recreated"},{"line_number":451,"context_line":""},{"line_number":452,"context_line":"    def _should_autocreate(self, broker, req_timestamp, account,"},{"line_number":453,"context_line":"                           policy_index, autocreate_header\u003dNone):"},{"line_number":454,"context_line":"        if autocreate_header is not None:"},{"line_number":455,"context_line":"            # If the caller included an explicit X-Backend-Auto-Create header,"},{"line_number":456,"context_line":"            # assume they know the behavior they want"}],"source_content_type":"text/x-python","patch_set":5,"id":"ff570b3c_ef3bbe98","line":453,"updated":"2020-06-03 19:07:00.000000000","message":"sorry it\u0027s late in the game; just noticed some of these params aren\u0027t used in this method (anymore?)","commit_id":"36776cd6c9723e41089b5966dc04b7a95d62ad89"}],"swift/container/sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6799044337e2de2c4064a3272911c107d2ba9001","unresolved":false,"context_lines":[{"line_number":1149,"context_line":"                \u0027X-Container-Sysmeta-Shard-Quoted-Root\u0027: quote("},{"line_number":1150,"context_line":"                    broker.root_path),"},{"line_number":1151,"context_line":"                \u0027X-Container-Sysmeta-Sharding\u0027: \u0027True\u0027,"},{"line_number":1152,"context_line":"                \u0027X-Backend-Auto-Create\u0027: \u0027True\u0027}"},{"line_number":1153,"context_line":"            # NB: we *used* to send along"},{"line_number":1154,"context_line":"            #    \u0027X-Container-Sysmeta-Shard-Root\u0027: broker.root_path"},{"line_number":1155,"context_line":"            # but that isn\u0027t safe for container names with nulls or newlines"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_99141446","line":1152,"updated":"2020-06-03 17:41:01.000000000","message":"quite nice, very obvious","commit_id":"e76deee4e22dc1763aad31d11ae0a5ac58869d0e"}],"test/unit/container/test_server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"ced46694c24b73343b13171b237218302a9dba46","unresolved":false,"context_lines":[{"line_number":2371,"context_line":"        resp \u003d req.get_response(self.controller)"},{"line_number":2372,"context_line":"        self.assertEqual(resp.status_int, 412)"},{"line_number":2373,"context_line":""},{"line_number":2374,"context_line":"    def test_PUT_shard_range_autocreates_shard_container(self):"},{"line_number":2375,"context_line":"        ts_iter \u003d make_timestamp_iter()"},{"line_number":2376,"context_line":"        shard_range \u003d ShardRange(\u0027.shards_a/shard_c\u0027, next(ts_iter))"},{"line_number":2377,"context_line":"        put_timestamp \u003d next(ts_iter).internal"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_be7661a3","line":2374,"updated":"2020-05-29 22:52:51.000000000","message":"PUT with a shard is already covered here -- I probably wouldn\u0027t have noticed otherwise (until the probe tests came back failed).","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a096340383bf6b9c0284d21458459e935ae3c4fa","unresolved":false,"context_lines":[{"line_number":4671,"context_line":"                            headers\u003ddict(headers))"},{"line_number":4672,"context_line":"        resp \u003d req.get_response(self.controller)"},{"line_number":4673,"context_line":"        self.assertEqual(resp.status_int, 404)"},{"line_number":4674,"context_line":""},{"line_number":4675,"context_line":"        req \u003d Request.blank(\u0027/sda1/p/a/.c/o\u0027,"},{"line_number":4676,"context_line":"                            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"},{"line_number":4677,"context_line":"                            headers\u003ddict(headers))"}],"source_content_type":"text/x-python","patch_set":1,"id":"ff570b3c_0181fa4a","line":4674,"updated":"2020-05-29 06:22:30.000000000","message":"Did we need to throw once with `x-backend-record-type \u003d shard` to make sure it\u0027ll 201?","commit_id":"3e540ccf4998ff3db60f3866b7d358de291e13d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"6799044337e2de2c4064a3272911c107d2ba9001","unresolved":false,"context_lines":[{"line_number":2384,"context_line":"        req \u003d Request.blank(\u0027/sda1/p/.shards_a/shard_c\u0027, method\u003d\u0027PUT\u0027, headers\u003dheaders,"},{"line_number":2385,"context_line":"                            body\u003djson.dumps([dict(shard_range)]))"},{"line_number":2386,"context_line":"        resp \u003d req.get_response(self.controller)"},{"line_number":2387,"context_line":"        self.assertEqual(404, resp.status_int)"},{"line_number":2388,"context_line":""},{"line_number":2389,"context_line":"        # PUT shard range to non-existent container with autocreate flag,"},{"line_number":2390,"context_line":"        # missing storage policy"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_790f802d","line":2387,"updated":"2020-06-03 17:41:01.000000000","message":"quite nice","commit_id":"e76deee4e22dc1763aad31d11ae0a5ac58869d0e"}]}