)]}'
{"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f3cb90007ab1ebfac23f1f63a48eff7f4d9c716b","unresolved":false,"context_lines":[{"line_number":4279,"context_line":"    ACTIVE \u003d 1"},{"line_number":4280,"context_line":"    SHRINKING \u003d 2"},{"line_number":4281,"context_line":"    SHARDED \u003d 3"},{"line_number":4282,"context_line":"    SHRUNK \u003d 4"},{"line_number":4283,"context_line":"    STATES \u003d {CREATED: \u0027created\u0027,"},{"line_number":4284,"context_line":"              ACTIVE: \u0027active\u0027,"},{"line_number":4285,"context_line":"              SHRINKING: \u0027shrinking\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_5f842bd3","line":4282,"updated":"2018-02-22 00:14:34.000000000","message":"Off-topic: Hmm... I wonder -- why do we differentiate between sharded and shrunk? Would it make sense to replace them both with \"deleted\" or \"inactive\" or something like that?","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4879693736c5fce28780969b24faf4a23f1a9096","unresolved":false,"context_lines":[{"line_number":4279,"context_line":"    ACTIVE \u003d 1"},{"line_number":4280,"context_line":"    SHRINKING \u003d 2"},{"line_number":4281,"context_line":"    SHARDED \u003d 3"},{"line_number":4282,"context_line":"    SHRUNK \u003d 4"},{"line_number":4283,"context_line":"    STATES \u003d {CREATED: \u0027created\u0027,"},{"line_number":4284,"context_line":"              ACTIVE: \u0027active\u0027,"},{"line_number":4285,"context_line":"              SHRINKING: \u0027shrinking\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_453c52c7","line":4282,"in_reply_to":"3fa0c359_5f842bd3","updated":"2018-02-22 11:44:09.000000000","message":"seems that SHRUNK is never used anyway","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"}],"swift/container/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f3cb90007ab1ebfac23f1f63a48eff7f4d9c716b","unresolved":false,"context_lines":[{"line_number":1398,"context_line":"            \u0027COMMIT;\u0027)"},{"line_number":1399,"context_line":""},{"line_number":1400,"context_line":"    def get_shard_usage(self):"},{"line_number":1401,"context_line":"        states \u003d (ShardRange.ACTIVE, ShardRange.SHRINKING)"},{"line_number":1402,"context_line":"        states \u003d \u0027,\u0027.join([str(state) for state in states])"},{"line_number":1403,"context_line":"        self._commit_puts_stale_ok()"},{"line_number":1404,"context_line":"        with self.get() as conn:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_9ff5d359","line":1401,"updated":"2018-02-22 00:14:34.000000000","message":"Interesting...\n\nAm I right to think that while the root container is sharding, we under-report how many objects there are, but when a shard is shrinking, we over-report?","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4879693736c5fce28780969b24faf4a23f1a9096","unresolved":false,"context_lines":[{"line_number":1398,"context_line":"            \u0027COMMIT;\u0027)"},{"line_number":1399,"context_line":""},{"line_number":1400,"context_line":"    def get_shard_usage(self):"},{"line_number":1401,"context_line":"        states \u003d (ShardRange.ACTIVE, ShardRange.SHRINKING)"},{"line_number":1402,"context_line":"        states \u003d \u0027,\u0027.join([str(state) for state in states])"},{"line_number":1403,"context_line":"        self._commit_puts_stale_ok()"},{"line_number":1404,"context_line":"        with self.get() as conn:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_05ff2aab","line":1401,"in_reply_to":"3fa0c359_9ff5d359","updated":"2018-02-22 11:44:09.000000000","message":"there is currently a window of time when the donor has cleaved to the acceptor, then the acceptor updates its stats to the root, before the donor deletes itself from the root.\n\nThere is a TODO to fix that [1], by having the donor include itself as a deleted shard range when cleaving to the acceptor, so that when the acceptor next updates the root it will delete the donor.\n\nRelated to this, but separate issue, is [2], which stops the donor, once sharded, reporting non-zero stats to the account.\n\n[1] https://github.com/openstack/swift/blob/58d1afafa2838f7041eb5a640c42a299b8ef4339/swift/container/sharder.py#L1126-L1131\n\n[2] https://review.openstack.org/546741","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f3cb90007ab1ebfac23f1f63a48eff7f4d9c716b","unresolved":false,"context_lines":[{"line_number":1429,"context_line":"                conditions \u003d []"},{"line_number":1430,"context_line":"                if not include_deleted:"},{"line_number":1431,"context_line":"                    conditions.append(\u0027deleted\u003d0\u0027)"},{"line_number":1432,"context_line":"                if isinstance(state, list):"},{"line_number":1433,"context_line":"                    state_list \u003d \u0027,\u0027.join([str(st) for st in state])"},{"line_number":1434,"context_line":"                    conditions.append(\u0027state in (%s)\u0027 % state_list)"},{"line_number":1435,"context_line":"                elif state is not None:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_bf8e8fce","line":1432,"updated":"2018-02-22 00:14:34.000000000","message":"nit: no tuple support?","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4879693736c5fce28780969b24faf4a23f1a9096","unresolved":false,"context_lines":[{"line_number":1429,"context_line":"                conditions \u003d []"},{"line_number":1430,"context_line":"                if not include_deleted:"},{"line_number":1431,"context_line":"                    conditions.append(\u0027deleted\u003d0\u0027)"},{"line_number":1432,"context_line":"                if isinstance(state, list):"},{"line_number":1433,"context_line":"                    state_list \u003d \u0027,\u0027.join([str(st) for st in state])"},{"line_number":1434,"context_line":"                    conditions.append(\u0027state in (%s)\u0027 % state_list)"},{"line_number":1435,"context_line":"                elif state is not None:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_889b5115","line":1432,"in_reply_to":"3fa0c359_bf8e8fce","updated":"2018-02-22 11:44:09.000000000","message":"Done","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"}],"swift/container/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f3cb90007ab1ebfac23f1f63a48eff7f4d9c716b","unresolved":false,"context_lines":[{"line_number":587,"context_line":"                    # map auto to states in which shard range is available for"},{"line_number":588,"context_line":"                    # listing"},{"line_number":589,"context_line":"                    states.remove(\u0027listing\u0027)"},{"line_number":590,"context_line":"                    states \u003d states | {\u0027active\u0027, \u0027shrinking\u0027}"},{"line_number":591,"context_line":"                try:"},{"line_number":592,"context_line":"                    states \u003d [ShardRange.STATES_BY_NAME[st] for st in states]"},{"line_number":593,"context_line":"                except KeyError:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_df527b35","line":590,"updated":"2018-02-22 00:14:34.000000000","message":"nit: could do\n\n |\u003d","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4879693736c5fce28780969b24faf4a23f1a9096","unresolved":false,"context_lines":[{"line_number":587,"context_line":"                    # map auto to states in which shard range is available for"},{"line_number":588,"context_line":"                    # listing"},{"line_number":589,"context_line":"                    states.remove(\u0027listing\u0027)"},{"line_number":590,"context_line":"                    states \u003d states | {\u0027active\u0027, \u0027shrinking\u0027}"},{"line_number":591,"context_line":"                try:"},{"line_number":592,"context_line":"                    states \u003d [ShardRange.STATES_BY_NAME[st] for st in states]"},{"line_number":593,"context_line":"                except KeyError:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_088f41cd","line":590,"in_reply_to":"3fa0c359_df527b35","updated":"2018-02-22 11:44:09.000000000","message":"Done","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f3cb90007ab1ebfac23f1f63a48eff7f4d9c716b","unresolved":false,"context_lines":[{"line_number":591,"context_line":"                try:"},{"line_number":592,"context_line":"                    states \u003d [ShardRange.STATES_BY_NAME[st] for st in states]"},{"line_number":593,"context_line":"                except KeyError:"},{"line_number":594,"context_line":"                    return HTTPBadRequest(request\u003dreq, body\u003d\u0027Bad state\u0027)"},{"line_number":595,"context_line":"            container_list \u003d broker.get_shard_ranges("},{"line_number":596,"context_line":"                marker, end_marker, includes, reverse, state\u003dstates)"},{"line_number":597,"context_line":"            if states and ShardRange.ACTIVE in states:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_3f427f69","line":594,"updated":"2018-02-22 00:14:34.000000000","message":"Off-topic: we should probably add a *little* more context to this error message -- it sure *sounds like* it ought to be a 500 (or *maybe* a 409 or 422) as-is :P","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"}],"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f3cb90007ab1ebfac23f1f63a48eff7f4d9c716b","unresolved":false,"context_lines":[{"line_number":992,"context_line":"                continue"},{"line_number":993,"context_line":""},{"line_number":994,"context_line":"            proposed_object_count \u003d donor.object_count + acceptor.object_count"},{"line_number":995,"context_line":"            if (donor.state \u003d\u003d ShardRange.SHRINKING or"},{"line_number":996,"context_line":"                    (donor.object_count \u003c self.shrink_size and"},{"line_number":997,"context_line":"                     proposed_object_count \u003c self.merge_size)):"},{"line_number":998,"context_line":"                # include previously identified merge pairs on presumption that"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_65a63812","line":995,"range":{"start_line":995,"start_character":16,"end_line":995,"end_character":51},"updated":"2018-02-22 00:14:34.000000000","message":"Won\u0027t this make us bump the acceptor\u0027s stats even more? I was kinda expecting\n\n if ShardRange.SHRINKING in (acceptor.state, donor.state):\n\nabove...","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4879693736c5fce28780969b24faf4a23f1a9096","unresolved":false,"context_lines":[{"line_number":992,"context_line":"                continue"},{"line_number":993,"context_line":""},{"line_number":994,"context_line":"            proposed_object_count \u003d donor.object_count + acceptor.object_count"},{"line_number":995,"context_line":"            if (donor.state \u003d\u003d ShardRange.SHRINKING or"},{"line_number":996,"context_line":"                    (donor.object_count \u003c self.shrink_size and"},{"line_number":997,"context_line":"                     proposed_object_count \u003c self.merge_size)):"},{"line_number":998,"context_line":"                # include previously identified merge pairs on presumption that"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_2b5d5714","line":995,"range":{"start_line":995,"start_character":16,"end_line":995,"end_character":51},"in_reply_to":"3fa0c359_65a63812","updated":"2018-02-22 11:44:09.000000000","message":"I want to include the pair in the merge_pairs in case the donor PUT failed on the previous attempt. The modified acceptor is not merged here, so its stats are not repeatedly incremented. If the acceptor has reported new stats which include the cleaved donor\u0027s objects then the donor should no longer be in shrinking state (it should be deleted).\n\nBut I should not be updating the donor state below if it is already shrinking.","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f3cb90007ab1ebfac23f1f63a48eff7f4d9c716b","unresolved":false,"context_lines":[{"line_number":1026,"context_line":"            acceptor.bytes_used +\u003d donor.bytes_used"},{"line_number":1027,"context_line":"            # Set donor state to shrinking so that next cycle won\u0027t use it as"},{"line_number":1028,"context_line":"            # an acceptor"},{"line_number":1029,"context_line":"            donor.update_state(ShardRange.SHRINKING)"},{"line_number":1030,"context_line":"            broker.merge_shard_ranges([donor])"},{"line_number":1031,"context_line":"            # PUT the acceptor shard range to the donor shard container; this"},{"line_number":1032,"context_line":"            # forces the donor to asynchronously cleave its entire contents to"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_a53c4093","line":1029,"updated":"2018-02-22 00:14:34.000000000","message":"How do shards manage their state transitions? Would the next stat-update (or an in-progress one) overwrite this?","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4879693736c5fce28780969b24faf4a23f1a9096","unresolved":false,"context_lines":[{"line_number":1026,"context_line":"            acceptor.bytes_used +\u003d donor.bytes_used"},{"line_number":1027,"context_line":"            # Set donor state to shrinking so that next cycle won\u0027t use it as"},{"line_number":1028,"context_line":"            # an acceptor"},{"line_number":1029,"context_line":"            donor.update_state(ShardRange.SHRINKING)"},{"line_number":1030,"context_line":"            broker.merge_shard_ranges([donor])"},{"line_number":1031,"context_line":"            # PUT the acceptor shard range to the donor shard container; this"},{"line_number":1032,"context_line":"            # forces the donor to asynchronously cleave its entire contents to"}],"source_content_type":"text/x-python","patch_set":5,"id":"3fa0c359_e851b537","line":1029,"in_reply_to":"3fa0c359_a53c4093","updated":"2018-02-22 11:44:09.000000000","message":"state updates are governed by state timestamp in merge_shard_ranges, similar to how content-type updates have their own timestamp in object rows, so a subsequent stats update will not revert the state change.","commit_id":"68a2696c71dbd0ade2711916d4a6b2a7a50e370f"}]}