)]}'
{"swift/cli/manage_shard_ranges.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b30384c15b212b35bd20624a4f20b3c5e5c44afb","unresolved":true,"context_lines":[{"line_number":445,"context_line":"    if not args.yes:"},{"line_number":446,"context_line":"        for sequence in compactable:"},{"line_number":447,"context_line":"            acceptor \u003d sequence[-1]"},{"line_number":448,"context_line":"            donors \u003d sequence[:-1]"},{"line_number":449,"context_line":"            print(\u0027Shard %s (object count %d) can expand to accept %d objects \u0027"},{"line_number":450,"context_line":"                  \u0027from:\u0027 %"},{"line_number":451,"context_line":"                  (acceptor, acceptor.object_count, donors.object_count))"}],"source_content_type":"text/x-python","patch_set":6,"id":"4cb6fed4_ab76a6f3","line":448,"updated":"2021-01-20 06:08:44.000000000","message":"Seems when you make a sublist here it isn\u0027t a ShardRangeList anymore. So adding:\n\n  donors \u003d ShardRangeList(sequence[:1]))\n\nWill make this code work. I started looking into what you need to do to __getitem__ to make a sublist be same as the current one, but have run out of time. Will continue with this tomorrow.\n\nOn clay\u0027s probe test if you run this twice (first to compact them into 1 shard and second to push it into the root) it seems to do the right thing.\n\nWill continue looking at this tomorrow depending on whether you get back to this or not.","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"101f5eb75af707a0c452ab294aed33d6cb8eb36e","unresolved":true,"context_lines":[{"line_number":445,"context_line":"    if not args.yes:"},{"line_number":446,"context_line":"        for sequence in compactable:"},{"line_number":447,"context_line":"            acceptor \u003d sequence[-1]"},{"line_number":448,"context_line":"            donors \u003d sequence[:-1]"},{"line_number":449,"context_line":"            print(\u0027Shard %s (object count %d) can expand to accept %d objects \u0027"},{"line_number":450,"context_line":"                  \u0027from:\u0027 %"},{"line_number":451,"context_line":"                  (acceptor, acceptor.object_count, donors.object_count))"}],"source_content_type":"text/x-python","patch_set":6,"id":"538b823a_c373350b","line":448,"in_reply_to":"4cb6fed4_ab76a6f3","updated":"2021-01-20 19:54:40.000000000","message":"good catch!\n\nextending UserList seems to be the answer\n\n(the probe tests would use --yes so not exercise this code)","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b43cbe890b310ff50d503c5f403db038591bffc0","unresolved":true,"context_lines":[{"line_number":424,"context_line":"        return 2"},{"line_number":425,"context_line":""},{"line_number":426,"context_line":"    if not broker.is_sharded():"},{"line_number":427,"context_line":"        print(\u0027WARNING: Container is not yet sharded so cannot be compacted.\u0027)"},{"line_number":428,"context_line":"        return 2"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"    shard_ranges \u003d broker.get_shard_ranges()"}],"source_content_type":"text/x-python","patch_set":14,"id":"07fc2be4_76aa1c66","line":427,"updated":"2021-02-05 01:43:07.000000000","message":"OK, so we have to be on the happy path and this won\u0027t deal with overlaps.","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c06601db3caae55562382e3047461314cf60022b","unresolved":false,"context_lines":[{"line_number":424,"context_line":"        return 2"},{"line_number":425,"context_line":""},{"line_number":426,"context_line":"    if not broker.is_sharded():"},{"line_number":427,"context_line":"        print(\u0027WARNING: Container is not yet sharded so cannot be compacted.\u0027)"},{"line_number":428,"context_line":"        return 2"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"    shard_ranges \u003d broker.get_shard_ranges()"}],"source_content_type":"text/x-python","patch_set":14,"id":"9b20e2f5_a68272bc","line":427,"in_reply_to":"07fc2be4_76aa1c66","updated":"2021-02-05 15:41:50.000000000","message":"Ack","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b43cbe890b310ff50d503c5f403db038591bffc0","unresolved":true,"context_lines":[{"line_number":635,"context_line":"              file\u003dsys.stderr)"},{"line_number":636,"context_line":"        return 2"},{"line_number":637,"context_line":"    print(\u0027Loaded db broker for %s.\u0027 % broker.path, file\u003dsys.stderr)"},{"line_number":638,"context_line":"    return args.func(broker, args) or 0"},{"line_number":639,"context_line":""},{"line_number":640,"context_line":""},{"line_number":641,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":14,"id":"f1ee9548_14d2d01d","line":638,"range":{"start_line":638,"start_character":34,"end_line":638,"end_character":39},"updated":"2021-02-05 01:43:07.000000000","message":"What was returning None? Can we just change it to return 0? I\u0027m not sure about assuming that no explicit return value means success...","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c06601db3caae55562382e3047461314cf60022b","unresolved":false,"context_lines":[{"line_number":635,"context_line":"              file\u003dsys.stderr)"},{"line_number":636,"context_line":"        return 2"},{"line_number":637,"context_line":"    print(\u0027Loaded db broker for %s.\u0027 % broker.path, file\u003dsys.stderr)"},{"line_number":638,"context_line":"    return args.func(broker, args) or 0"},{"line_number":639,"context_line":""},{"line_number":640,"context_line":""},{"line_number":641,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":14,"id":"06cb8a95_aaebe33d","line":638,"range":{"start_line":638,"start_character":34,"end_line":638,"end_character":39},"in_reply_to":"f1ee9548_14d2d01d","updated":"2021-02-05 15:41:50.000000000","message":"db_info? TBH, I can\u0027t remember what caused me to add the 0","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"fda872c8ddb571d661c74e9cefd7b9abb3956f72","unresolved":true,"context_lines":[{"line_number":612,"context_line":"                                     \u0027Defaults to 1. Using values greater \u0027"},{"line_number":613,"context_line":"                                     \u0027than 1 may result in temporary gaps in \u0027"},{"line_number":614,"context_line":"                                     \u0027object listings until all selected \u0027"},{"line_number":615,"context_line":"                                     \u0027shards have shrunk.\u0027)"},{"line_number":616,"context_line":"    compact_parser.add_argument(\u0027--max-expanding\u0027, nargs\u003d\u0027?\u0027,"},{"line_number":617,"context_line":"                                type\u003d_positive_int,"},{"line_number":618,"context_line":"                                default\u003dDEFAULT_MAX_EXPANDING,"}],"source_content_type":"text/x-python","patch_set":16,"id":"9cc90dd4_a872b797","line":615,"updated":"2021-02-07 23:03:52.000000000","message":"GREAT! Yes! happy to land this so long as the default is 1 for the time being. This help explains why.","commit_id":"12bb4839f02cdf32170a85438973fd93da4c1140"}],"swift/common/utils.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"93539e49819941d5fe0f485ce2024a090669ea3e","unresolved":true,"context_lines":[{"line_number":5501,"context_line":"            self.lower \u003d new_lower"},{"line_number":5502,"context_line":"            self.upper \u003d new_upper"},{"line_number":5503,"context_line":"            modified \u003d True"},{"line_number":5504,"context_line":"        return modified"},{"line_number":5505,"context_line":""},{"line_number":5506,"context_line":""},{"line_number":5507,"context_line":"class ShardRangeList(list):"}],"source_content_type":"text/x-python","patch_set":6,"id":"fa8ab538_6e4a92a0","line":5504,"updated":"2021-01-11 00:11:07.000000000","message":"do we need to update a timestamp as there has been a change to the SR?\n\nUPDATE: Oh this happens in finalize_shrinking, so sone seperately.. ok, I guess I still need to load all the moving parts until I can really comment :)","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3e253bded929c3182e81b8680f1ee99fe25b5f8f","unresolved":true,"context_lines":[{"line_number":5520,"context_line":"    @property"},{"line_number":5521,"context_line":"    def upper(self):"},{"line_number":5522,"context_line":"        if not self:"},{"line_number":5523,"context_line":"            return ShardRange.MIN"},{"line_number":5524,"context_line":"        return self[-1].upper"},{"line_number":5525,"context_line":""},{"line_number":5526,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"1b4386e2_dd8176c5","line":5523,"updated":"2021-01-29 04:18:05.000000000","message":"I assume this should be MAX","commit_id":"3e4feb9ac2bea9a5f47f710f45868a16ab1a2ed0"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0067e65e864f14809effe61efa39e280ac6c1ef3","unresolved":false,"context_lines":[{"line_number":5520,"context_line":"    @property"},{"line_number":5521,"context_line":"    def upper(self):"},{"line_number":5522,"context_line":"        if not self:"},{"line_number":5523,"context_line":"            return ShardRange.MIN"},{"line_number":5524,"context_line":"        return self[-1].upper"},{"line_number":5525,"context_line":""},{"line_number":5526,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"e0c0d763_38885a7c","line":5523,"in_reply_to":"1414d01b_7fcf527f","updated":"2021-02-01 01:41:48.000000000","message":"Ack","commit_id":"3e4feb9ac2bea9a5f47f710f45868a16ab1a2ed0"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"09fce696417062d32f7571fa3d19cd6d81559a18","unresolved":true,"context_lines":[{"line_number":5520,"context_line":"    @property"},{"line_number":5521,"context_line":"    def upper(self):"},{"line_number":5522,"context_line":"        if not self:"},{"line_number":5523,"context_line":"            return ShardRange.MIN"},{"line_number":5524,"context_line":"        return self[-1].upper"},{"line_number":5525,"context_line":""},{"line_number":5526,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":12,"id":"1414d01b_7fcf527f","line":5523,"in_reply_to":"1b4386e2_dd8176c5","updated":"2021-01-29 09:33:15.000000000","message":"no I intended MIN - the list is empty so its range is (arbitrarily) defined as MIN-MIN\n\nbut...is that reasonable?","commit_id":"3e4feb9ac2bea9a5f47f710f45868a16ab1a2ed0"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"0067e65e864f14809effe61efa39e280ac6c1ef3","unresolved":true,"context_lines":[{"line_number":5521,"context_line":"    @property"},{"line_number":5522,"context_line":"    def upper(self):"},{"line_number":5523,"context_line":"        if not self:"},{"line_number":5524,"context_line":"            # empty list has range MIN-\u003eMIN"},{"line_number":5525,"context_line":"            return ShardRange.MIN"},{"line_number":5526,"context_line":"        return self[-1].upper"},{"line_number":5527,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"84d121a4_274598d2","line":5524,"updated":"2021-02-01 01:41:48.000000000","message":"Thanks for the comment, that makes sense, it\u0027s empty, so we don\u0027t want it to fill the entire namespace 😊","commit_id":"d8844aa7fb0b82ca4d8d186c65ee4b7ab995e985"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b43cbe890b310ff50d503c5f403db038591bffc0","unresolved":true,"context_lines":[{"line_number":5505,"context_line":"        return modified"},{"line_number":5506,"context_line":""},{"line_number":5507,"context_line":""},{"line_number":5508,"context_line":"class ShardRangeList(UserList):"},{"line_number":5509,"context_line":"    def __getitem__(self, index):"},{"line_number":5510,"context_line":"        # workaround for py3 - not needed for py2.7,py3.8"},{"line_number":5511,"context_line":"        result \u003d self.data[index]"}],"source_content_type":"text/x-python","patch_set":14,"id":"033f6f13_e9911604","line":5508,"updated":"2021-02-05 01:43:07.000000000","message":"Should this include an __init__ that validates the list is\n\n* sorted properly (so upper and lower behave as expected) and\n* has no namespace gaps?","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c06601db3caae55562382e3047461314cf60022b","unresolved":false,"context_lines":[{"line_number":5505,"context_line":"        return modified"},{"line_number":5506,"context_line":""},{"line_number":5507,"context_line":""},{"line_number":5508,"context_line":"class ShardRangeList(UserList):"},{"line_number":5509,"context_line":"    def __getitem__(self, index):"},{"line_number":5510,"context_line":"        # workaround for py3 - not needed for py2.7,py3.8"},{"line_number":5511,"context_line":"        result \u003d self.data[index]"}],"source_content_type":"text/x-python","patch_set":14,"id":"f7dfc044_76003521","line":5508,"in_reply_to":"033f6f13_e9911604","updated":"2021-02-05 15:41:50.000000000","message":"It wasn\u0027t my goal to have this class enforce those properties - to do so would mean checking every list modification - I just wanted to gather up some convenience functions into a class. The onus is on the caller to ensure the contents of the list make sense.\n\nI\u0027ll add some doc comments to alert callers to the assumptions.\n\nI think a smarter auto-sorting class would be v cool (although we might want gaps to be optional), I just don\u0027t feel my current use cases justified the work. Maybe we\u0027ll improve this as we find more use cases (I can imagine a has_gaps() and has_overlaps() method for example)","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b43cbe890b310ff50d503c5f403db038591bffc0","unresolved":true,"context_lines":[{"line_number":5533,"context_line":"    def bytes_used(self):"},{"line_number":5534,"context_line":"        return sum(sr.bytes_used for sr in self)"},{"line_number":5535,"context_line":""},{"line_number":5536,"context_line":"    def includes(self, other):"},{"line_number":5537,"context_line":"        return self.lower \u003c\u003d other.lower and self.upper \u003e\u003d other.upper"},{"line_number":5538,"context_line":""},{"line_number":5539,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"0200fbf2_ca1a1442","line":5536,"updated":"2021-02-05 01:43:07.000000000","message":"OK, so we\u0027ve got this subtle difference between\n\n shard_range_list.includes(shard_range)\n\nand\n\n shard_range in shard_range_list\n\nGood to know.","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c06601db3caae55562382e3047461314cf60022b","unresolved":false,"context_lines":[{"line_number":5533,"context_line":"    def bytes_used(self):"},{"line_number":5534,"context_line":"        return sum(sr.bytes_used for sr in self)"},{"line_number":5535,"context_line":""},{"line_number":5536,"context_line":"    def includes(self, other):"},{"line_number":5537,"context_line":"        return self.lower \u003c\u003d other.lower and self.upper \u003e\u003d other.upper"},{"line_number":5538,"context_line":""},{"line_number":5539,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"64639f38_cc5b8d12","line":5536,"in_reply_to":"0200fbf2_ca1a1442","updated":"2021-02-05 15:41:50.000000000","message":"and this includes() is equivalent to ShardRange.includes() (in fact there is scope for some shared inheritance, but I\u0027m don\u0027t feel the need, yet)","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b30384c15b212b35bd20624a4f20b3c5e5c44afb","unresolved":true,"context_lines":[{"line_number":163,"context_line":"                                     shrink_threshold,"},{"line_number":164,"context_line":"                                     merge_size,"},{"line_number":165,"context_line":"                                     max_shrinking,"},{"line_number":166,"context_line":"                                     max_expanding):"},{"line_number":167,"context_line":"    \"\"\""},{"line_number":168,"context_line":"    Find sequences of shard ranges that could be compacted into a single"},{"line_number":169,"context_line":"    acceptor shard range."}],"source_content_type":"text/x-python","patch_set":6,"id":"7f747767_e44e424c","line":166,"updated":"2021-01-20 06:08:44.000000000","message":"should we support 0 (or unlimited) max_shrinking and maybe even max_expanding? Could be useful in our deleted root container case 😊","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"101f5eb75af707a0c452ab294aed33d6cb8eb36e","unresolved":true,"context_lines":[{"line_number":163,"context_line":"                                     shrink_threshold,"},{"line_number":164,"context_line":"                                     merge_size,"},{"line_number":165,"context_line":"                                     max_shrinking,"},{"line_number":166,"context_line":"                                     max_expanding):"},{"line_number":167,"context_line":"    \"\"\""},{"line_number":168,"context_line":"    Find sequences of shard ranges that could be compacted into a single"},{"line_number":169,"context_line":"    acceptor shard range."}],"source_content_type":"text/x-python","patch_set":6,"id":"6561baf0_7c53b4ef","line":166,"in_reply_to":"7f747767_e44e424c","updated":"2021-01-20 19:54:40.000000000","message":"+1 good idea","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b30384c15b212b35bd20624a4f20b3c5e5c44afb","unresolved":true,"context_lines":[{"line_number":228,"context_line":"            if len(compactable_sequence) \u003e max_shrinking:"},{"line_number":229,"context_line":"                break"},{"line_number":230,"context_line":"            if proposed_object_count \u003e\u003d merge_size:"},{"line_number":231,"context_line":"                break"},{"line_number":232,"context_line":""},{"line_number":233,"context_line":"        if len(compactable_sequence) \u003c 2:"},{"line_number":234,"context_line":"            return None, consumed"}],"source_content_type":"text/x-python","patch_set":6,"id":"e9f0903a_8ec423dc","line":231,"updated":"2021-01-20 06:08:44.000000000","message":"The if\u0027s for 219 amd 230 can both match because:\n  proposed_object_count \u003c\u003d merge_size\n  proposed_object_count \u003e\u003d merge_size\n\nIe when proposed_object count \u003d\u003d merge_size. The way the code works, it\u0027s probably ok, and it just means it hits the line, the shardrange is added tothe sequence and then the loop is broken.. just hoping there is an edge case here we\u0027re missing. Or if we want to act like other sharder things where we decide \u003c\u003d and \u003e or visa-versa.","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"101f5eb75af707a0c452ab294aed33d6cb8eb36e","unresolved":true,"context_lines":[{"line_number":228,"context_line":"            if len(compactable_sequence) \u003e max_shrinking:"},{"line_number":229,"context_line":"                break"},{"line_number":230,"context_line":"            if proposed_object_count \u003e\u003d merge_size:"},{"line_number":231,"context_line":"                break"},{"line_number":232,"context_line":""},{"line_number":233,"context_line":"        if len(compactable_sequence) \u003c 2:"},{"line_number":234,"context_line":"            return None, consumed"}],"source_content_type":"text/x-python","patch_set":6,"id":"111c166c_cebee162","line":231,"in_reply_to":"e9f0903a_8ec423dc","updated":"2021-01-20 19:54:40.000000000","message":"the test at line 219 is to decide if the current shard should be shrunk.\n\nthe test here at line 230 is to decide if we should bother looking for more shards to append to the sequence, so I think the \u003c\u003d vs \u003e\u003d does make sense. Considering the edge case, at line 219 it\u0027s ok to append this shard if the proposed count \u003d\u003d merge size, at line 230 we want to break if the proposed count \u003d\u003d merge size, because we do not want to add any more to the object count.","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b30384c15b212b35bd20624a4f20b3c5e5c44afb","unresolved":true,"context_lines":[{"line_number":269,"context_line":"            # it as an acceptor; state_timestamp defines new epoch for"},{"line_number":270,"context_line":"            # donor and new timestamp for the expanded acceptor below."},{"line_number":271,"context_line":"            donor.epoch \u003d donor.state_timestamp \u003d epoch"},{"line_number":272,"context_line":"    broker.merge_shard_ranges(modified_acceptor_ranges + donor_ranges)"},{"line_number":273,"context_line":""},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"def process_compactable_shard_sequences(broker, sequences):"}],"source_content_type":"text/x-python","patch_set":6,"id":"4c564354_426e41c0","line":272,"updated":"2021-01-20 06:08:44.000000000","message":"So in theory, if we want to use root-container driven shrinking, this is great, however the doner needs the acceptor in it\u0027s shard table. But looking at the code, the fact that we are inserting the expanded acceptor into the root container mean it should now overlap with the donor so when the doner shard is audited it should pull it and the accpter down from the root container. And therefore cleave itself?\n\nI think that should work. Let me check what\u0027s happening on clays patches probe test in https://review.opendev.org/c/openstack/swift/+/771086","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"101f5eb75af707a0c452ab294aed33d6cb8eb36e","unresolved":true,"context_lines":[{"line_number":269,"context_line":"            # it as an acceptor; state_timestamp defines new epoch for"},{"line_number":270,"context_line":"            # donor and new timestamp for the expanded acceptor below."},{"line_number":271,"context_line":"            donor.epoch \u003d donor.state_timestamp \u003d epoch"},{"line_number":272,"context_line":"    broker.merge_shard_ranges(modified_acceptor_ranges + donor_ranges)"},{"line_number":273,"context_line":""},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"def process_compactable_shard_sequences(broker, sequences):"}],"source_content_type":"text/x-python","patch_set":6,"id":"cf79f063_79e6f332","line":272,"in_reply_to":"4c564354_426e41c0","updated":"2021-01-20 19:54:40.000000000","message":"yes, the changes recently made here https://github.com/openstack/swift/blob/d277960161119face2eb9d617822b2645f19b2c1/swift/container/sharder.py#L785-L813 facilitate this - the donor fetches itself from root (and learns it is shrinking) AND fetches the overlapping acceptor from root, which is sufficient for it to start shrinking (cleaving to acceptor).\n\nI think there is a wrinkle when the acceptor needs to be the root - I am working on that right now.","commit_id":"934039d8b11cf34094310613c1fb8953730913f9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5a5171cd7928298c758ad53c11ae5087cf489394","unresolved":true,"context_lines":[{"line_number":928,"context_line":"                          if sr.state in (ShardRange.CREATED,"},{"line_number":929,"context_line":"                                          ShardRange.CLEAVED,"},{"line_number":930,"context_line":"                                          ShardRange.ACTIVE)]"},{"line_number":931,"context_line":"                # TODO: need to get the root own shard range to propagate"},{"line_number":932,"context_line":"                self.logger.warning("},{"line_number":933,"context_line":"                    \u0027Other shard ranges from root have %d useful acceptors\u0027 %"},{"line_number":934,"context_line":"                    len(useful))"}],"source_content_type":"text/x-python","patch_set":7,"id":"6188e91f_8c1a7073","line":931,"updated":"2021-01-21 01:03:23.000000000","message":"One thought is that we can make the container-server (or even the proxy if we want to keep the sharding decision in there because we use internal client) a little smarter. And when we get shard ranges from root and there is only 1 shardrange in the shrinking state that we include it\u0027s own_own_shardrange in the response. Then we\u0027d have the root in the other_shard_ranges above?\n\nOther is inserting one as we have the root path in metadata in the shard, like I did in https://review.opendev.org/c/openstack/swift/+/771343. But maybe inserting shard ranges from a shard goes against the whole root side sharding, were we are expecting decisions to be made at the root side.","commit_id":"3ff2473385cbf398690cbc4ffee7cd00e6e85150"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"cc20e0decb945e58cea0cd665e42d191444b3e46","unresolved":true,"context_lines":[{"line_number":928,"context_line":"                          if sr.state in (ShardRange.CREATED,"},{"line_number":929,"context_line":"                                          ShardRange.CLEAVED,"},{"line_number":930,"context_line":"                                          ShardRange.ACTIVE)]"},{"line_number":931,"context_line":"                # TODO: need to get the root own shard range to propagate"},{"line_number":932,"context_line":"                self.logger.warning("},{"line_number":933,"context_line":"                    \u0027Other shard ranges from root have %d useful acceptors\u0027 %"},{"line_number":934,"context_line":"                    len(useful))"}],"source_content_type":"text/x-python","patch_set":7,"id":"0d568766_c6a2af29","line":931,"in_reply_to":"6188e91f_8c1a7073","updated":"2021-01-21 05:33:14.000000000","message":"My dodgy hack to get this working was to:\n\ndiff --git a/swift/container/server.py b/swift/container/server.py\nindex 0292320..7a5d2e0 100644\n--- a/swift/container/server.py\n+++ b/swift/container/server.py\n@@ -762,6 +762,14 @@ class ContainerController(BaseStorageServer):\n             container_list \u003d broker.get_shard_ranges(\n                 marker, end_marker, includes, reverse, states\u003dstates,\n                 include_deleted\u003dinclude_deleted, fill_gaps\u003dfill_gaps)\n+            if len(container_list) \u003d\u003d 1 and  \\\n+                    all([sr.state \u003d\u003d ShardRange.SHRINKING \n+                         for sr in broker.get_shard_ranges()]) \\\n+                    and broker.is_root_container():\n+                root_sr \u003d broker.get_own_shard_range()\n+                root_sr.update_state(ShardRange.ACTIVE,\n+                                     state_timestamp\u003dTimestamp.now())\n+                container_list.append(root_sr)\n         else:\n             resp_headers \u003d gen_resp_headers(info, is_deleted\u003dis_deleted)\n             if is_deleted:\n\nSo the container server will return the root SR in active state ready for shrinking. So good news this works. But if we go down this path we need to double check our guards and maybe do it in a better place? At the moment it\u0027s:\n\n  - If we\u0027re only returning one range (no other acceptors because they\u0027re all shrinking)\n  - All the shard ranges are shrinking.\n  - And its a root container (as we don\u0027t want shards inserting themselves potentually?).\n\nBut it worked on clay\u0027s relcaim probe test I ran this patches version of the compact command on  a root container, ran the container replicators, ran the sharders (a few times to make sure it was all cleaved), ran the updaters and replicators and things started disappearing (after reclaim ages).\n\nAlthough I\u0027m not sure I like going through all the shard ranges again in the all, as I\u0027ve just hacked it on to the end rather then rework the GET so we can this all shrink info for \"free\"","commit_id":"3ff2473385cbf398690cbc4ffee7cd00e6e85150"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76a61b33806f55529cdc58985d3da595f841b9d5","unresolved":true,"context_lines":[{"line_number":1630,"context_line":"            broker, self.shrink_size, self.merge_size, 1, 1)"},{"line_number":1631,"context_line":"        self.logger.debug(\u0027Found %s compactable sequences of length(s) %s\u0027 %"},{"line_number":1632,"context_line":"                          (len(compactable_sequences),"},{"line_number":1633,"context_line":"                           [len(s) for s in compactable_sequences]))"},{"line_number":1634,"context_line":"        process_compactable_shard_sequences(broker, compactable_sequences)"},{"line_number":1635,"context_line":"        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":1636,"context_line":"        for sequence in compactable_sequences:"}],"source_content_type":"text/x-python","patch_set":13,"id":"8af2cb8b_542b5d1d","line":1633,"updated":"2021-02-02 06:06:16.000000000","message":"I guess as this code stands the length is going to be 1, as you are passing in 1, 1 to find_compactable_shard_sequences.\n\nStill I like this, so logging wont need to change when/if we change the default from 1 😊","commit_id":"d8844aa7fb0b82ca4d8d186c65ee4b7ab995e985"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b43cbe890b310ff50d503c5f403db038591bffc0","unresolved":true,"context_lines":[{"line_number":152,"context_line":"    merge_pairs \u003d {}"},{"line_number":153,"context_line":"    # restrict search to sequences with one donor and one acceptor"},{"line_number":154,"context_line":"    results \u003d find_compactable_shard_sequences(broker, shrink_threshold,"},{"line_number":155,"context_line":"                                               merge_size, 1, 1)"},{"line_number":156,"context_line":"    for sequence in results:"},{"line_number":157,"context_line":"        # map acceptor -\u003e donor list"},{"line_number":158,"context_line":"        merge_pairs[sequence[-1]] \u003d sequence[-2]"}],"source_content_type":"text/x-python","patch_set":14,"id":"2c3554ef_d24b5a16","line":155,"range":{"start_line":155,"start_character":62,"end_line":155,"end_character":63},"updated":"2021-02-05 01:43:07.000000000","message":"I thought this function could return multiple pairs -- shouldn\u0027t this be -1?","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c06601db3caae55562382e3047461314cf60022b","unresolved":false,"context_lines":[{"line_number":152,"context_line":"    merge_pairs \u003d {}"},{"line_number":153,"context_line":"    # restrict search to sequences with one donor and one acceptor"},{"line_number":154,"context_line":"    results \u003d find_compactable_shard_sequences(broker, shrink_threshold,"},{"line_number":155,"context_line":"                                               merge_size, 1, 1)"},{"line_number":156,"context_line":"    for sequence in results:"},{"line_number":157,"context_line":"        # map acceptor -\u003e donor list"},{"line_number":158,"context_line":"        merge_pairs[sequence[-1]] \u003d sequence[-2]"}],"source_content_type":"text/x-python","patch_set":14,"id":"9800dcb9_ece0a0dd","line":155,"range":{"start_line":155,"start_character":62,"end_line":155,"end_character":63},"in_reply_to":"2c3554ef_d24b5a16","updated":"2021-02-05 15:41:50.000000000","message":"oops - also no test coverage 😞","commit_id":"fb68315a3414bb3d75d5c8e689afffa372644a1f"}],"test/unit/common/test_utils.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5fb7d87c52ee9d0afb1f19a8e5e37e20d6826753","unresolved":true,"context_lines":[{"line_number":8397,"context_line":"        self.assertEqual(\u0027a\u0027, srl.lower)"},{"line_number":8398,"context_line":"        self.assertEqual(\u0027y\u0027, srl.upper)"},{"line_number":8399,"context_line":"        self.assertEqual(12, srl.object_count)"},{"line_number":8400,"context_line":"        self.assertEqual(132, srl.bytes_used)"},{"line_number":8401,"context_line":""},{"line_number":8402,"context_line":"    def test_pop(self):"},{"line_number":8403,"context_line":"        srl \u003d ShardRangeList(self.shard_ranges[:2])"}],"source_content_type":"text/x-python","patch_set":13,"id":"02b3d30f_b5cb0271","line":8400,"updated":"2021-02-04 05:28:28.000000000","message":"It makes sense as this is a list that the list inside can contain gaps. I wonder if this would be a good place, in the future, to add some gap logic, like get_gaps, has_gaps But this is out of scope for compact. Just a thought for the future.","commit_id":"d8844aa7fb0b82ca4d8d186c65ee4b7ab995e985"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fe9da78b2124d218b30476e819982a6ed1adb369","unresolved":false,"context_lines":[{"line_number":8397,"context_line":"        self.assertEqual(\u0027a\u0027, srl.lower)"},{"line_number":8398,"context_line":"        self.assertEqual(\u0027y\u0027, srl.upper)"},{"line_number":8399,"context_line":"        self.assertEqual(12, srl.object_count)"},{"line_number":8400,"context_line":"        self.assertEqual(132, srl.bytes_used)"},{"line_number":8401,"context_line":""},{"line_number":8402,"context_line":"    def test_pop(self):"},{"line_number":8403,"context_line":"        srl \u003d ShardRangeList(self.shard_ranges[:2])"}],"source_content_type":"text/x-python","patch_set":13,"id":"10264df2_a7cfe2ae","line":8400,"in_reply_to":"02b3d30f_b5cb0271","updated":"2021-02-04 15:01:53.000000000","message":"Ack","commit_id":"d8844aa7fb0b82ca4d8d186c65ee4b7ab995e985"}]}