)]}'
{"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cce99f50a5654d874dc729cf2b2dfa0bde66a49c","unresolved":true,"context_lines":[{"line_number":602,"context_line":"                self.logger.error("},{"line_number":603,"context_line":"                    \"Failed to get shard ranges from %s: invalid data: %r\","},{"line_number":604,"context_line":"                    quote(broker.root_path), err)"},{"line_number":605,"context_line":"            return None, None"},{"line_number":606,"context_line":"        finally:"},{"line_number":607,"context_line":"            self.logger.txn_id \u003d None"},{"line_number":608,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"bec0f1c3_29db0bec","line":605,"updated":"2021-01-14 06:58:47.000000000","message":"not sure I like sending the resp out like this yet.. still exploring :)","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cce99f50a5654d874dc729cf2b2dfa0bde66a49c","unresolved":true,"context_lines":[{"line_number":756,"context_line":"                resp_delete_ts \u003d root_resp.headers.get("},{"line_number":757,"context_line":"                        \u0027X-Backend-Delete-Timestamp\u0027)"},{"line_number":758,"context_line":"                resp_put_ts \u003d root_resp.headers.get("},{"line_number":759,"context_line":"                        \u0027X-Backend-Put-Timestamp\u0027)"},{"line_number":760,"context_line":""},{"line_number":761,"context_line":"                def root_is_deleted():"},{"line_number":762,"context_line":"                    # annoyingly if the root is marked at deleted we can still"}],"source_content_type":"text/x-python","patch_set":2,"id":"718ab9fc_a2b36cd5","line":759,"updated":"2021-01-14 06:58:47.000000000","message":"This is because we always return a 200, so we nee to look at timestamps.","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cce99f50a5654d874dc729cf2b2dfa0bde66a49c","unresolved":true,"context_lines":[{"line_number":762,"context_line":"                    # annoyingly if the root is marked at deleted we can still"},{"line_number":763,"context_line":"                    # get a 200 return from it when asking for shards. So we"},{"line_number":764,"context_line":"                    # need to check the timestamps."},{"line_number":765,"context_line":"                    return resp_delete_ts and resp_delete_ts \u003e resp_put_ts"},{"line_number":766,"context_line":""},{"line_number":767,"context_line":"                if root_resp.status_int \u003d\u003d 404 or root_is_deleted():"},{"line_number":768,"context_line":"                    # If the root container returns a 404 then mark its"}],"source_content_type":"text/x-python","patch_set":2,"id":"c123ef28_324e4cf8","line":765,"updated":"2021-01-14 06:58:47.000000000","message":"I may need to add the root doesn\u0027t exist anymore case if it wont actually return a 404. Ie add a: \n\n  or not resp_delete_ts and not resp_put_ts","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"60b9b7f9e6a4c6e026574fe5d9641f01afda9386","unresolved":true,"context_lines":[{"line_number":762,"context_line":"                    # annoyingly if the root is marked at deleted we can still"},{"line_number":763,"context_line":"                    # get a 200 return from it when asking for shards. So we"},{"line_number":764,"context_line":"                    # need to check the timestamps."},{"line_number":765,"context_line":"                    return resp_delete_ts and resp_delete_ts \u003e resp_put_ts"},{"line_number":766,"context_line":""},{"line_number":767,"context_line":"                if root_resp.status_int \u003d\u003d 404 or root_is_deleted():"},{"line_number":768,"context_line":"                    # If the root container returns a 404 then mark its"}],"source_content_type":"text/x-python","patch_set":2,"id":"62b8dcc1_7bdfcce8","line":765,"in_reply_to":"408bb2ce_f837b08f","updated":"2021-01-14 22:01:16.000000000","message":"yeah might need to look into that. it\u0027s tough because when I go look at the contrainer it\u0027s definitely marked as deleted. And a normal GET gets a 404.","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9396121d8d0e0ce198ee0c33888725cee438ce5c","unresolved":true,"context_lines":[{"line_number":762,"context_line":"                    # annoyingly if the root is marked at deleted we can still"},{"line_number":763,"context_line":"                    # get a 200 return from it when asking for shards. So we"},{"line_number":764,"context_line":"                    # need to check the timestamps."},{"line_number":765,"context_line":"                    return resp_delete_ts and resp_delete_ts \u003e resp_put_ts"},{"line_number":766,"context_line":""},{"line_number":767,"context_line":"                if root_resp.status_int \u003d\u003d 404 or root_is_deleted():"},{"line_number":768,"context_line":"                    # If the root container returns a 404 then mark its"}],"source_content_type":"text/x-python","patch_set":2,"id":"408bb2ce_f837b08f","line":765,"in_reply_to":"c123ef28_324e4cf8","updated":"2021-01-14 16:40:49.000000000","message":"i wonder if the root might have set a Delete-Timestamp but still be considered \"alive\" because there\u0027s objects in the OTHER shards...","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cce99f50a5654d874dc729cf2b2dfa0bde66a49c","unresolved":true,"context_lines":[{"line_number":764,"context_line":"                    # need to check the timestamps."},{"line_number":765,"context_line":"                    return resp_delete_ts and resp_delete_ts \u003e resp_put_ts"},{"line_number":766,"context_line":""},{"line_number":767,"context_line":"                if root_resp.status_int \u003d\u003d 404 or root_is_deleted():"},{"line_number":768,"context_line":"                    # If the root container returns a 404 then mark its"},{"line_number":769,"context_line":"                    # delete timestamp in the sysmeta of the shard."},{"line_number":770,"context_line":"                    # if the root container has been reaped then use"}],"source_content_type":"text/x-python","patch_set":2,"id":"b7db2958_077d0c5d","line":767,"updated":"2021-01-14 06:58:47.000000000","message":"This 404 may not ever happen. I\u0027ll now for sure when I finish debugging the last step in the probe test.","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9396121d8d0e0ce198ee0c33888725cee438ce5c","unresolved":true,"context_lines":[{"line_number":783,"context_line":"                            broker.db_file, quote(broker.path))"},{"line_number":784,"context_line":"                elif root_resp.status_int % 100 \u003d\u003d 2 and root_delete_ts and \\"},{"line_number":785,"context_line":"                        not root_is_deleted():"},{"line_number":786,"context_line":"                    broker.set_sharding_sysmeta(\"Root-Delete-Timestamp\", \"\")"},{"line_number":787,"context_line":"            if shard_ranges:"},{"line_number":788,"context_line":"                for shard_range in shard_ranges:"},{"line_number":789,"context_line":"                    # look for this shard range in the list of shard ranges"}],"source_content_type":"text/x-python","patch_set":2,"id":"367fd714_97f3cf59","line":786,"updated":"2021-01-14 16:40:49.000000000","message":"so this is kind of like... letting the root come back\n\n... but we\u0027ll clear it every pass even if we hadn\u0027t previously set it","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"60b9b7f9e6a4c6e026574fe5d9641f01afda9386","unresolved":true,"context_lines":[{"line_number":783,"context_line":"                            broker.db_file, quote(broker.path))"},{"line_number":784,"context_line":"                elif root_resp.status_int % 100 \u003d\u003d 2 and root_delete_ts and \\"},{"line_number":785,"context_line":"                        not root_is_deleted():"},{"line_number":786,"context_line":"                    broker.set_sharding_sysmeta(\"Root-Delete-Timestamp\", \"\")"},{"line_number":787,"context_line":"            if shard_ranges:"},{"line_number":788,"context_line":"                for shard_range in shard_ranges:"},{"line_number":789,"context_line":"                    # look for this shard range in the list of shard ranges"}],"source_content_type":"text/x-python","patch_set":2,"id":"a6ae07b7_ae7930f7","line":786,"in_reply_to":"367fd714_97f3cf59","updated":"2021-01-14 22:01:16.000000000","message":"oh yeah nice spot.","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9396121d8d0e0ce198ee0c33888725cee438ce5c","unresolved":true,"context_lines":[{"line_number":860,"context_line":"            broker.delete_db(root_delete_ts)"},{"line_number":861,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s) using root \u0027"},{"line_number":862,"context_line":"                              \u0027delete timestmp\u0027,"},{"line_number":863,"context_line":"                              broker.db_file, quote(broker.path))"},{"line_number":864,"context_line":""},{"line_number":865,"context_line":"        self._increment_stat(\u0027audit_shard\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":866,"context_line":"        return True"}],"source_content_type":"text/x-python","patch_set":2,"id":"e1bb36bf_8e885de5","line":863,"updated":"2021-01-14 16:40:49.000000000","message":"this part seems almost reasonable... we should mark shards deleted around the time the root is recalaimed\n\n... but I think we might be reclaimable almost immediatley if we\u0027re re-using the root\u0027s timestamp or we might not get deleted at all if we still have rows","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"60b9b7f9e6a4c6e026574fe5d9641f01afda9386","unresolved":true,"context_lines":[{"line_number":860,"context_line":"            broker.delete_db(root_delete_ts)"},{"line_number":861,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s) using root \u0027"},{"line_number":862,"context_line":"                              \u0027delete timestmp\u0027,"},{"line_number":863,"context_line":"                              broker.db_file, quote(broker.path))"},{"line_number":864,"context_line":""},{"line_number":865,"context_line":"        self._increment_stat(\u0027audit_shard\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":866,"context_line":"        return True"}],"source_content_type":"text/x-python","patch_set":2,"id":"4e29d5e9_4a4e1bd0","line":863,"in_reply_to":"e1bb36bf_8e885de5","updated":"2021-01-14 22:01:16.000000000","message":"yeah that was kinda the plan. if the root is reclaimed then do we need the shards, seeing as there is a strong relationship. Ie a root + shards conceptually are the same container.\n\nBUT I did have a thought last night. shards are created all the time. Is there a chance that a shard was created (a shard cleaved on a shard) after the root is marked as deleted. In this case it\u0027ll maen the delete timestamp is before a put timestamp and wont be deleted.. tho having said that, in that case we might be able to take advantage of using time.time() once the root is reclaimed (not sure it\u0027ll work like that in the current code tho).","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"11674d0c7417f798850c15782dcd3b2dc169d2cd","unresolved":true,"context_lines":[{"line_number":588,"context_line":"                return None, None"},{"line_number":589,"context_line":"            record_type \u003d resp.headers.get(\u0027x-backend-record-type\u0027)"},{"line_number":590,"context_line":"            if resp.status_int \u003d\u003d 404:"},{"line_number":591,"context_line":"                return None, resp"},{"line_number":592,"context_line":"            if record_type !\u003d \u0027shard\u0027:"},{"line_number":593,"context_line":"                err \u003d \u0027unexpected record type %r\u0027 % record_type"},{"line_number":594,"context_line":"                self.logger.error(\"Failed to get shard ranges from %s: %s\","}],"source_content_type":"text/x-python","patch_set":3,"id":"c5bb02ae_89926ec1","line":591,"updated":"2021-01-15 01:30:08.000000000","message":"we want 404s to come back and get processed, turns out this only happens after the root is reclaimed because we send X-Backend-Override-Deleted so we can still get shards out of roots marked as deleted.","commit_id":"c3184fd847a38b4a000a7cf7d424455334b6cb98"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"11674d0c7417f798850c15782dcd3b2dc169d2cd","unresolved":true,"context_lines":[{"line_number":802,"context_line":"                    # root may not yet know about this shard container"},{"line_number":803,"context_line":"                    warnings.append(\u0027root has no matching shard range\u0027)"},{"line_number":804,"context_line":"            elif not own_shard_range.deleted and root_resp and root_resp.status_int !\u003d 404:"},{"line_number":805,"context_line":"                    warnings.append(\u0027unable to get shard ranges from root\u0027)"},{"line_number":806,"context_line":"            # else, our shard range is deleted, so root may have reclaimed it"},{"line_number":807,"context_line":"        else:"},{"line_number":808,"context_line":"            errors.append(\u0027missing own shard range\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"1cbbb46f_f344c473","line":805,"updated":"2021-01-15 01:30:08.000000000","message":"opps forgot to fix this. That\u0027ll be in the next patchset.","commit_id":"c3184fd847a38b4a000a7cf7d424455334b6cb98"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"11674d0c7417f798850c15782dcd3b2dc169d2cd","unresolved":true,"context_lines":[{"line_number":860,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s)\u0027,"},{"line_number":861,"context_line":"                              broker.db_file, quote(broker.path))"},{"line_number":862,"context_line":"        elif root_delete_ts \u003c delete_age:"},{"line_number":863,"context_line":"            broker.delete_db(root_delete_ts)"},{"line_number":864,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s) using root \u0027"},{"line_number":865,"context_line":"                              \u0027delete timestmp\u0027,"},{"line_number":866,"context_line":"                              broker.db_file, quote(broker.path))"}],"source_content_type":"text/x-python","patch_set":3,"id":"31f5edee_91315549","line":863,"updated":"2021-01-15 01:30:08.000000000","message":"we could always use a Time.time() here, but then shards that haven\u0027t cleaned themselves up will live an extra reclaim_age.","commit_id":"c3184fd847a38b4a000a7cf7d424455334b6cb98"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1b3061013e2fe39ab3ea049aa213907f7d95e7c9","unresolved":true,"context_lines":[{"line_number":576,"context_line":"                   \u0027X-Backend-Include-Deleted\u0027: str(include_deleted)}"},{"line_number":577,"context_line":"        if newest:"},{"line_number":578,"context_line":"            headers[\u0027X-Newest\u0027] \u003d \u0027true\u0027"},{"line_number":579,"context_line":"        resp \u003d None"},{"line_number":580,"context_line":"        try:"},{"line_number":581,"context_line":"            try:"},{"line_number":582,"context_line":"                resp \u003d self.int_client.make_request("}],"source_content_type":"text/x-python","patch_set":4,"id":"8ac3bff8_49759efb","line":579,"updated":"2021-01-15 15:00:33.000000000","message":"I wonder if the method could be broken out into two steps - first get the response, second parse the shard ranges from response? rather than returning the tuple?...","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1b3061013e2fe39ab3ea049aa213907f7d95e7c9","unresolved":true,"context_lines":[{"line_number":589,"context_line":"            record_type \u003d resp.headers.get(\u0027x-backend-record-type\u0027)"},{"line_number":590,"context_line":"            if resp.status_int \u003d\u003d 404:"},{"line_number":591,"context_line":"                return None, resp"},{"line_number":592,"context_line":"            if record_type !\u003d \u0027shard\u0027:"},{"line_number":593,"context_line":"                err \u003d \u0027unexpected record type %r\u0027 % record_type"},{"line_number":594,"context_line":"                self.logger.error(\"Failed to get shard ranges from %s: %s\","},{"line_number":595,"context_line":"                                  quote(broker.root_path), err)"}],"source_content_type":"text/x-python","patch_set":4,"id":"99f8519d_3e4eb4dd","line":592,"updated":"2021-01-15 15:00:33.000000000","message":"...in fact the remainder looks a lot like https://github.com/openstack/swift/blob/077ba77ea6f735c98709b60e1fc9aa9f0f90676a/swift/proxy/controllers/base.py#L2308 so maybe can re-use as a utils helper method?","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1b3061013e2fe39ab3ea049aa213907f7d95e7c9","unresolved":true,"context_lines":[{"line_number":755,"context_line":"                params\u003d{\u0027marker\u0027: str_to_wsgi(own_shard_range.lower_str),"},{"line_number":756,"context_line":"                        \u0027end_marker\u0027: str_to_wsgi(own_shard_range.upper_str)},"},{"line_number":757,"context_line":"                include_deleted\u003dTrue)"},{"line_number":758,"context_line":"            if root_resp:"},{"line_number":759,"context_line":"                resp_delete_ts \u003d float(root_resp.headers.get("},{"line_number":760,"context_line":"                    \u0027X-Backend-Delete-Timestamp\u0027, 0))"},{"line_number":761,"context_line":"                resp_put_ts \u003d float(root_resp.headers.get("}],"source_content_type":"text/x-python","patch_set":4,"id":"de48b75d_9049ce56","line":758,"updated":"2021-01-15 15:00:33.000000000","message":"... first get a response and do the delete-related stuff...","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1b3061013e2fe39ab3ea049aa213907f7d95e7c9","unresolved":true,"context_lines":[{"line_number":761,"context_line":"                resp_put_ts \u003d float(root_resp.headers.get("},{"line_number":762,"context_line":"                    \u0027X-Backend-Put-Timestamp\u0027, 0))"},{"line_number":763,"context_line":""},{"line_number":764,"context_line":"                def root_is_deleted():"},{"line_number":765,"context_line":"                    # annoyingly if the root is marked at deleted we can still"},{"line_number":766,"context_line":"                    # get a 200 return from it when asking for shards. So we"},{"line_number":767,"context_line":"                    # need to check the timestamps."}],"source_content_type":"text/x-python","patch_set":4,"id":"f16228a2_393516fe","line":764,"updated":"2021-01-15 15:00:33.000000000","message":"does this need to be a function vs a bool var?","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1b3061013e2fe39ab3ea049aa213907f7d95e7c9","unresolved":true,"context_lines":[{"line_number":796,"context_line":"                    self.logger.info("},{"line_number":797,"context_line":"                        \"Removing root delete timestamp in shard %s (%s)\","},{"line_number":798,"context_line":"                        broker.db_file, quote(broker.path))"},{"line_number":799,"context_line":"            if shard_ranges:"},{"line_number":800,"context_line":"                for shard_range in shard_ranges:"},{"line_number":801,"context_line":"                    # look for this shard range in the list of shard ranges"},{"line_number":802,"context_line":"                    # received from root; the root may have different lower and"}],"source_content_type":"text/x-python","patch_set":4,"id":"510a87cd_f3fc8691","line":799,"updated":"2021-01-15 15:00:33.000000000","message":"then parse the shards using something like _parse_shard_ranges() \n\nhmmm, _parse_shard_ranges() takes a req arg...but IDK, maybe it can be refactored, definitely seems like there is scope to have some code re-use","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"360a5212d09650f82d13b0c14a6ec88cb88795ae","unresolved":true,"context_lines":[{"line_number":869,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s)\u0027,"},{"line_number":870,"context_line":"                              broker.db_file, quote(broker.path))"},{"line_number":871,"context_line":"        elif root_delete_ts and root_delete_ts \u003c\u003d delete_age:"},{"line_number":872,"context_line":"            broker.delete_db(Timestamp(root_delete_ts).internal)"},{"line_number":873,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s) using root \u0027"},{"line_number":874,"context_line":"                              \u0027delete timestmp\u0027,"},{"line_number":875,"context_line":"                              broker.db_file, quote(broker.path))"}],"source_content_type":"text/x-python","patch_set":4,"id":"26a638fa_a0eb9c64","line":872,"updated":"2021-01-15 06:13:18.000000000","message":"The real question is should we attempt this and get nice quick cleanups for be safe and use:\n\n  Timestamp.now().internal","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1b3061013e2fe39ab3ea049aa213907f7d95e7c9","unresolved":true,"context_lines":[{"line_number":869,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s)\u0027,"},{"line_number":870,"context_line":"                              broker.db_file, quote(broker.path))"},{"line_number":871,"context_line":"        elif root_delete_ts and root_delete_ts \u003c\u003d delete_age:"},{"line_number":872,"context_line":"            broker.delete_db(Timestamp(root_delete_ts).internal)"},{"line_number":873,"context_line":"            self.logger.debug(\u0027Deleted shard container %s (%s) using root \u0027"},{"line_number":874,"context_line":"                              \u0027delete timestmp\u0027,"},{"line_number":875,"context_line":"                              broker.db_file, quote(broker.path))"}],"source_content_type":"text/x-python","patch_set":4,"id":"570ea4fe_4ddd99fe","line":872,"in_reply_to":"26a638fa_a0eb9c64","updated":"2021-01-15 15:00:33.000000000","message":"if the question is simply \u0027should we be safe or not\u0027 then be safe 😊\n\nbut, what\u0027s the risk? that the root appears to have been deleted for reclaim_age, but in fact has not yet been reclaimed (quite possible) and the shard deletes itself prematurely? I\u0027d vote for being safe in this instance because this fix should ultimately be the backstop (i.e. once we have shrinking, getting here should be exceptional, so play it really safe)\n\nWonder if it is worth somehow marking the shard with what happened? IIRC sysmeta is cleared by delete_db() otherwise we could write down in sysmeta that the shard self-deleted (just in case a human gets involved in some debugging during the shard\u0027s reclaim-age).","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"}],"test/probe/test_sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7f3fbd9aee3d97c383a8f4a63855552cd8c10ca5","unresolved":true,"context_lines":[{"line_number":1786,"context_line":"        # assert that shard containers aren\u0027t marked as deleted but the root is"},{"line_number":1787,"context_line":"        self.assert_container_not_found()"},{"line_number":1788,"context_line":"        for shard_broker in self.yield_shard_brokers(shard_ranges):"},{"line_number":1789,"context_line":"            self.assertFalse(shard_broker.is_deleted())"},{"line_number":1790,"context_line":""},{"line_number":1791,"context_line":"        # Now let\u0027s simulate the root container reaching reclaim_age. Ie delete it"},{"line_number":1792,"context_line":"        files \u003d self.categorize_container_dir_content()"}],"source_content_type":"text/x-python","patch_set":1,"id":"f40960d9_e3bfd17f","line":1789,"updated":"2021-01-13 15:16:34.000000000","message":"a comment could clarify this is just \"how it is\" - not so much that it\u0027s a behavior we have any particular requirement to maintain.","commit_id":"4908d2f783d07943a9731cd2d1010bc0650235e5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7f3fbd9aee3d97c383a8f4a63855552cd8c10ca5","unresolved":true,"context_lines":[{"line_number":1796,"context_line":"        # run the sharders the shards have a chance to look for the root"},{"line_number":1797,"context_line":"        self.run_sharders(shard_ranges)"},{"line_number":1798,"context_line":""},{"line_number":1799,"context_line":"        # One now would hope that the shards know they need to be removed."},{"line_number":1800,"context_line":"        for shard_broker in self.yield_shard_brokers(shard_ranges):"},{"line_number":1801,"context_line":"            self.assertTrue(shard_broker.is_deleted())"},{"line_number":1802,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"8542df18_4bba2851","line":1799,"updated":"2021-01-13 15:16:34.000000000","message":"Haha!  one would hope!","commit_id":"4908d2f783d07943a9731cd2d1010bc0650235e5"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cce99f50a5654d874dc729cf2b2dfa0bde66a49c","unresolved":true,"context_lines":[{"line_number":78,"context_line":"                \u0027auto_shard must be true in all container_sharder configs\u0027)"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"        self.reclaim_age \u003d max(float(c.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":81,"context_line":"                               for c in cont_configs)"},{"line_number":82,"context_line":"        self.max_shard_size \u003d max("},{"line_number":83,"context_line":"            int(c.get(\u0027shard_container_threshold\u0027, \u00271000000\u0027))"},{"line_number":84,"context_line":"            for c in cont_configs)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1561b3f4_22a93170","line":81,"updated":"2021-01-14 06:58:47.000000000","message":"might need to include the container-replicator configs here and also maybe not put this in _maybe_skip_test","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cce99f50a5654d874dc729cf2b2dfa0bde66a49c","unresolved":true,"context_lines":[{"line_number":1825,"context_line":"            shard_broker.set_sharding_sysmeta(\"Root_Delete_Timestamp\", delete_ts.internal)"},{"line_number":1826,"context_line":""},{"line_number":1827,"context_line":"        # run the sharders and then the replicators and..."},{"line_number":1828,"context_line":"        self.run_sharders(shard_ranges)"},{"line_number":1829,"context_line":"        self.replicators.once()"},{"line_number":1830,"context_line":""},{"line_number":1831,"context_line":"        # ... and there gone!"}],"source_content_type":"text/x-python","patch_set":2,"id":"893da7bc_be2fca1a","line":1828,"updated":"2021-01-14 06:58:47.000000000","message":"hmm, this might set the delete timestamp to something reapable, but the put timestamp is newer, so this probably needs to be fixed.. maybe attempting a real reap if unattainable :(","commit_id":"3d8361d5fe728c3f230eef71f828617e79875dd8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"11674d0c7417f798850c15782dcd3b2dc169d2cd","unresolved":true,"context_lines":[{"line_number":1837,"context_line":"        for shard in shard_ranges:"},{"line_number":1838,"context_line":"            self.assertFalse("},{"line_number":1839,"context_line":"                self.categorize_container_dir_content("},{"line_number":1840,"context_line":"                    shard.account, shard.container)[\u0027shard_dbs\u0027])"},{"line_number":1841,"context_line":""},{"line_number":1842,"context_line":""},{"line_number":1843,"context_line":"    def _setup_replication_scenario(self, num_shards, extra_objs\u003d(\u0027alpha\u0027,)):"}],"source_content_type":"text/x-python","patch_set":3,"id":"c1509e11_61c90eed","line":1840,"updated":"2021-01-15 01:30:08.000000000","message":"This actaully worked in my SAIO.. even though the delete_ts we used is \u003c the original put timestamp.. so maybe found a bug.","commit_id":"c3184fd847a38b4a000a7cf7d424455334b6cb98"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"360a5212d09650f82d13b0c14a6ec88cb88795ae","unresolved":true,"context_lines":[{"line_number":1838,"context_line":"        for shard in shard_ranges:"},{"line_number":1839,"context_line":"            self.assertFalse("},{"line_number":1840,"context_line":"                self.categorize_container_dir_content("},{"line_number":1841,"context_line":"                    shard.account, shard.container)[\u0027shard_dbs\u0027])"},{"line_number":1842,"context_line":""},{"line_number":1843,"context_line":"    def _setup_replication_scenario(self, num_shards, extra_objs\u003d(\u0027alpha\u0027,)):"},{"line_number":1844,"context_line":"        # Get cluster to state where 2 replicas are sharding or sharded but 3rd"}],"source_content_type":"text/x-python","patch_set":4,"id":"f0698df3_fa2e29d0","line":1841,"updated":"2021-01-15 06:13:18.000000000","message":"This oribe test passes on my saio! But the delete before the put means it doesn\u0027t actaully remove them or mark them deleted... so not sure why this last bit works. Let me know what happens to you. The unit tests test that they\u0027re marked as deleted in a way.","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"360a5212d09650f82d13b0c14a6ec88cb88795ae","unresolved":true,"context_lines":[{"line_number":805,"context_line":"            sharder.int_client.make_request.side_effect \u003d exc"},{"line_number":806,"context_line":"            self.assertEqual(sharder._fetch_shard_ranges(broker), (None, None))"},{"line_number":807,"context_line":"        lines \u003d sharder.logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":808,"context_line":"        self.assertIn(\u0027Unexpected response: 418\u0027, lines[0])"},{"line_number":809,"context_line":"        self.assertFalse(lines[1:])"},{"line_number":810,"context_line":""},{"line_number":811,"context_line":"    def test_fetch_shard_ranges_bad_record_type(self):"}],"source_content_type":"text/x-python","patch_set":4,"id":"ade3828d_df4815e3","line":808,"updated":"2021-01-15 06:13:18.000000000","message":"We acceot 404 so had to pick a new 4 series error. I\u0027m a teapot wins :P","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1b3061013e2fe39ab3ea049aa213907f7d95e7c9","unresolved":true,"context_lines":[{"line_number":805,"context_line":"            sharder.int_client.make_request.side_effect \u003d exc"},{"line_number":806,"context_line":"            self.assertEqual(sharder._fetch_shard_ranges(broker), (None, None))"},{"line_number":807,"context_line":"        lines \u003d sharder.logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":808,"context_line":"        self.assertIn(\u0027Unexpected response: 418\u0027, lines[0])"},{"line_number":809,"context_line":"        self.assertFalse(lines[1:])"},{"line_number":810,"context_line":""},{"line_number":811,"context_line":"    def test_fetch_shard_ranges_bad_record_type(self):"}],"source_content_type":"text/x-python","patch_set":4,"id":"9be2cfe2_fcc0b27e","line":808,"in_reply_to":"ade3828d_df4815e3","updated":"2021-01-15 15:00:33.000000000","message":"something\u0027s brewing :)","commit_id":"6fdd66e4547a056377cdf099bc1da04367a446b8"}]}