)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e027d2f79da85131a3967a44626af762316c4b9","unresolved":true,"context_lines":[{"line_number":10,"context_line":""},{"line_number":11,"context_line":"TODO: What other scenarios should we consider?"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Change-Id: I983d665584471c9d689506592f48ddd00c0887ef"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"464d2cb1_418615b6","line":13,"updated":"2021-05-28 14:30:41.000000000","message":"does this warrant a launchpad bug?","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"}],"swift/proxy/controllers/container.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9e550060066923416bdd8775aa8904435f2e13bf","unresolved":true,"context_lines":[{"line_number":279,"context_line":"               resp_record_type.lower() \u003d\u003d \u0027shard\u0027)):"},{"line_number":280,"context_line":"            resp \u003d self._get_from_shards(req, resp)"},{"line_number":281,"context_line":""},{"line_number":282,"context_line":"        if not config_true_value("},{"line_number":283,"context_line":"                resp.headers.get(\u0027X-Backend-Cached-Results\u0027)):"},{"line_number":284,"context_line":"            # Cache container metadata. We just made a request to a storage"},{"line_number":285,"context_line":"            # node and got up-to-date information for the container."},{"line_number":286,"context_line":"            resp.headers[\u0027X-Backend-Recheck-Container-Existence\u0027] \u003d str("},{"line_number":287,"context_line":"                self.app.recheck_container_existence)"},{"line_number":288,"context_line":"            set_info_cache(self.app, req.environ, self.account_name,"},{"line_number":289,"context_line":"                           self.container_name, resp)"},{"line_number":290,"context_line":"        if \u0027swift.authorize\u0027 in req.environ:"},{"line_number":291,"context_line":"            req.acl \u003d resp.headers.get(\u0027x-container-read\u0027)"},{"line_number":292,"context_line":"            aresp \u003d req.environ[\u0027swift.authorize\u0027](req)"}],"source_content_type":"text/x-python","patch_set":3,"id":"abee4ad7_e87b37d2","line":289,"range":{"start_line":282,"start_character":8,"end_line":289,"end_character":53},"updated":"2021-05-28 14:36:28.000000000","message":"we might want to move this ahead of the _get_from_shards since the resp may now be replaced with a 503, so we lose the opportunity to cache container metadata","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9e550060066923416bdd8775aa8904435f2e13bf","unresolved":true,"context_lines":[{"line_number":287,"context_line":"                self.app.recheck_container_existence)"},{"line_number":288,"context_line":"            set_info_cache(self.app, req.environ, self.account_name,"},{"line_number":289,"context_line":"                           self.container_name, resp)"},{"line_number":290,"context_line":"        if \u0027swift.authorize\u0027 in req.environ:"},{"line_number":291,"context_line":"            req.acl \u003d resp.headers.get(\u0027x-container-read\u0027)"},{"line_number":292,"context_line":"            aresp \u003d req.environ[\u0027swift.authorize\u0027](req)"},{"line_number":293,"context_line":"            if aresp:"},{"line_number":294,"context_line":"                # Don\u0027t cache this. It doesn\u0027t reflect the state of the"},{"line_number":295,"context_line":"                # container, just that the user can\u0027t access it."},{"line_number":296,"context_line":"                return aresp"},{"line_number":297,"context_line":"        if not req.environ.get(\u0027swift_owner\u0027, False):"},{"line_number":298,"context_line":"            for key in self.app.swift_owner_headers:"},{"line_number":299,"context_line":"                if key in resp.headers:"}],"source_content_type":"text/x-python","patch_set":3,"id":"8e419f59_4a5d9071","line":296,"range":{"start_line":290,"start_character":8,"end_line":296,"end_character":28},"updated":"2021-05-28 14:36:28.000000000","message":"slightly off topic but this might also go before _get_from_shards: if the request is not auth\u0027d there\u0027s no point building the listing from shards","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e027d2f79da85131a3967a44626af762316c4b9","unresolved":true,"context_lines":[{"line_number":396,"context_line":"            sharding_state \u003d shard_resp.headers.get(\u0027x-backend-sharding-state\u0027,"},{"line_number":397,"context_line":"                                                    \u0027unknown\u0027)"},{"line_number":398,"context_line":""},{"line_number":399,"context_line":"            if objs is None:"},{"line_number":400,"context_line":"                # tolerate errors"},{"line_number":401,"context_line":"                self.app.logger.debug("},{"line_number":402,"context_line":"                    \u0027Failed to get objects from shard (state\u003d%s), total \u003d %d\u0027,"}],"source_content_type":"text/x-python","patch_set":3,"id":"8a70124d_285d4561","line":399,"updated":"2021-05-28 14:30:41.000000000","message":"AFAICT objs is None IFF the backend response is not success or the json.loads() didn\u0027t like the response body...so at this point we know we are likely leaving a gap in the listing.\n\nIIRC the thinking back in the days of feature/deep was that this is equivalent to getting an unsharded listing from a container replica that hasn\u0027t sync\u0027d or got asyncs yet: that listing may also have gaps.\n\nHowever, if we\u0027re confident there has been an error, as opposed to \u0027lag\u0027, then perhaps it would be better to crash out. My one concern is if we hit this routinely during sharding, if a shard container 404s before being cleaved, but SHARD_LISTING_STATES \u003e\u003d CLEAVED so shard containers should exist before we use them for listing.\n\nIs it worth raising the log level to warning to see how often this is happening?","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e027d2f79da85131a3967a44626af762316c4b9","unresolved":true,"context_lines":[{"line_number":425,"context_line":"            if end_marker and not reverse and end_marker \u003c\u003d last_name:"},{"line_number":426,"context_line":"                break"},{"line_number":427,"context_line":""},{"line_number":428,"context_line":"        if not objects and not any(r \u003d\u003d 200 for r in all_resp_status):"},{"line_number":429,"context_line":"            self.app.logger.error(\u0027Insufficient shard responses to \u0027"},{"line_number":430,"context_line":"                                  \u0027return empty listing: %r\u0027 % all_resp_status)"},{"line_number":431,"context_line":"            return HTTPServiceUnavailable(request\u003dreq)"}],"source_content_type":"text/x-python","patch_set":3,"id":"67ab8b49_7e7dde29","line":428,"range":{"start_line":428,"start_character":31,"end_line":428,"end_character":34},"updated":"2021-05-28 14:30:41.000000000","message":"tests still pass if I change any to all","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"1da0985adc6fdafc48ab394d84898a20819dc778","unresolved":true,"context_lines":[{"line_number":428,"context_line":"        if not objects and not any(r \u003d\u003d 200 for r in all_resp_status):"},{"line_number":429,"context_line":"            self.app.logger.error(\u0027Insufficient shard responses to \u0027"},{"line_number":430,"context_line":"                                  \u0027return empty listing: %r\u0027 % all_resp_status)"},{"line_number":431,"context_line":"            return HTTPServiceUnavailable(request\u003dreq)"},{"line_number":432,"context_line":"        resp.body \u003d json.dumps(objects).encode(\u0027ascii\u0027)"},{"line_number":433,"context_line":"        constrained \u003d any(req.params.get(constraint) for constraint in ("},{"line_number":434,"context_line":"            \u0027marker\u0027, \u0027end_marker\u0027, \u0027path\u0027, \u0027prefix\u0027, \u0027delimiter\u0027))"}],"source_content_type":"text/x-python","patch_set":3,"id":"af40c669_9bfee87b","line":431,"updated":"2021-05-27 20:14:33.000000000","message":"This definitely seems necessary, but IDK that it\u0027s sufficient. I\u0027ve got a strong suspicion we ought to 503 as soon as we see objs coming back None.","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a085cb5a80cdb81e0ee67e3916a44c3e0927f0a3","unresolved":true,"context_lines":[{"line_number":398,"context_line":""},{"line_number":399,"context_line":"            if objs is None:"},{"line_number":400,"context_line":"                # give up if any non-success response from shard containers"},{"line_number":401,"context_line":"                self.app.logger.debug("},{"line_number":402,"context_line":"                    \u0027Failed to get objects from shard (state\u003d%s), total \u003d %d\u0027,"},{"line_number":403,"context_line":"                    sharding_state, len(objects))"},{"line_number":404,"context_line":"                self.app.logger.error("}],"source_content_type":"text/x-python","patch_set":4,"id":"08e45faf_d9b8ff61","line":401,"range":{"start_line":401,"start_character":32,"end_line":401,"end_character":37},"updated":"2021-06-01 18:14:25.000000000","message":"Should maybe bump this up a level or two, and we probably ought to indicate the shard account/container name in the message. IDK that len(objects) is terribly useful info (anymore?)","commit_id":"b3ad018366faf64c5fda366d1de9b23322b02f01"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a085cb5a80cdb81e0ee67e3916a44c3e0927f0a3","unresolved":true,"context_lines":[{"line_number":402,"context_line":"                    \u0027Failed to get objects from shard (state\u003d%s), total \u003d %d\u0027,"},{"line_number":403,"context_line":"                    sharding_state, len(objects))"},{"line_number":404,"context_line":"                self.app.logger.error("},{"line_number":405,"context_line":"                    \u0027Aborting listing from shards due to bad response: %r\u0027"},{"line_number":406,"context_line":"                    % all_resp_status)"},{"line_number":407,"context_line":"                return HTTPServiceUnavailable(request\u003dreq)"},{"line_number":408,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"3d10f81c_79b796e9","line":405,"range":{"start_line":405,"start_character":57,"end_line":405,"end_character":69},"updated":"2021-06-01 18:14:25.000000000","message":"\"bad response sequence\", maybe? *shrug*","commit_id":"b3ad018366faf64c5fda366d1de9b23322b02f01"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2e80fdb4d212677f6a2a8fec1bb90fed39383c1b","unresolved":true,"context_lines":[{"line_number":396,"context_line":"            sharding_state \u003d shard_resp.headers.get(\u0027x-backend-sharding-state\u0027,"},{"line_number":397,"context_line":"                                                    \u0027unknown\u0027)"},{"line_number":398,"context_line":""},{"line_number":399,"context_line":"            if objs is None:"},{"line_number":400,"context_line":"                # give up if any non-success response from shard containers"},{"line_number":401,"context_line":"                self.app.logger.error("},{"line_number":402,"context_line":"                    \u0027Aborting listing from shards due to bad response: %r\u0027"}],"source_content_type":"text/x-python","patch_set":6,"id":"a2bd6be1_455c7a32","line":399,"updated":"2021-06-23 07:08:48.000000000","message":"Oh so this only works because the _get_container_listing only returns None on a !is_successful and error. If there happened to be no objects it\u0027ll be an empty [], that it pulled out of the json body.\n\nA comment about None means it was unsuccessful (anything not 2**) might be nice, but not a blocker. I can\u0027t think of any other non-successfuls that we would tollerate so returning a 503 makes sense. Will also help us to find and fix any shard listing improvents if we start getting a bunch of 503s 😊","commit_id":"e40cf1ec561c78c8a02d40b581bf195939a6450a"}],"test/unit/proxy/controllers/test_container.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e027d2f79da85131a3967a44626af762316c4b9","unresolved":true,"context_lines":[{"line_number":926,"context_line":"        mock_responses \u003d ["},{"line_number":927,"context_line":"            # status, body, headers"},{"line_number":928,"context_line":"            (200, sr_dicts, root_shard_resp_hdrs),"},{"line_number":929,"context_line":"        ] + [(404, \u0027\u0027, {})] * 6"},{"line_number":930,"context_line":""},{"line_number":931,"context_line":"        codes \u003d (resp[0] for resp in mock_responses)"},{"line_number":932,"context_line":"        bodies \u003d iter([json.dumps(resp[1]).encode(\u0027ascii\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"56049a59_078332cd","line":929,"range":{"start_line":929,"start_character":30,"end_line":929,"end_character":31},"updated":"2021-05-28 14:30:41.000000000","message":"for the 4 replicas version to pass needs to be:\n  \n  2 * self.CONTAINER_REPLICAS","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1e027d2f79da85131a3967a44626af762316c4b9","unresolved":true,"context_lines":[{"line_number":938,"context_line":"            resp \u003d request.get_response(self.app)"},{"line_number":939,"context_line":"        warnings \u003d self.logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":940,"context_line":"        self.assertEqual([\u0027Failed to get container listing from \u0027"},{"line_number":941,"context_line":"                          \u0027/v1/a/c?states\u003dlisting\u0026format\u003djson: 404\u0027], warnings)"},{"line_number":942,"context_line":"        self.assertEqual(1 + 2 * self.CONTAINER_REPLICAS,"},{"line_number":943,"context_line":"                         len(fake_conn.requests))"},{"line_number":944,"context_line":"        self.assertEqual(resp.status_int, 503)"}],"source_content_type":"text/x-python","patch_set":3,"id":"73f8dc7b_8ff550fd","line":941,"updated":"2021-05-28 14:30:41.000000000","message":"I got this passing with\n\n          self.assertEqual([\u0027Failed to get container listing from \u0027\n                          \u0027%s: 404\u0027 % request.path_qs], warnings)","commit_id":"629880ab04af78e1f3d4837365f641d2a0ee3224"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1f09a342b8fc6fd90a9765564da969cabbda8085","unresolved":true,"context_lines":[{"line_number":1646,"context_line":""},{"line_number":1647,"context_line":"        self._check_GET_shard_listing("},{"line_number":1648,"context_line":"            mock_responses, all_objects, expected_requests,"},{"line_number":1649,"context_line":"            expected_status\u003d503)"},{"line_number":1650,"context_line":""},{"line_number":1651,"context_line":"    def test_GET_sharded_container_shard_errors(self):"},{"line_number":1652,"context_line":"        self._check_GET_sharded_container_shard_error(404)"}],"source_content_type":"text/x-python","patch_set":4,"id":"ef06229b_8fd4039d","line":1649,"updated":"2021-06-01 17:57:39.000000000","message":"this test is now pretty similar to the new tests :/","commit_id":"b3ad018366faf64c5fda366d1de9b23322b02f01"}]}