)]}'
{"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"05d2287e4d2bb6485db1d7c40db1fcc8cfd7db9a","unresolved":false,"context_lines":[{"line_number":540,"context_line":"                    if is_leader:"},{"line_number":541,"context_line":"                        self._find_shrinks(broker, node, part)"},{"line_number":542,"context_line":"                else:"},{"line_number":543,"context_line":"                    # let the root know about this shard\u0027s sharded shard ranges"},{"line_number":544,"context_line":"                    # TODO: why did I think this was necessary? as it is, this"},{"line_number":545,"context_line":"                    # is not a good thing to do - when we cleaved this shard to"},{"line_number":546,"context_line":"                    # it\u0027s acceptor we set it\u0027s object count to the actual"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_87b94a2f","line":543,"range":{"start_line":543,"start_character":51,"end_line":543,"end_character":72},"updated":"2018-01-04 23:34:39.000000000","message":"lol, 3 shards in a row!","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"05d2287e4d2bb6485db1d7c40db1fcc8cfd7db9a","unresolved":false,"context_lines":[{"line_number":589,"context_line":"                        [own_shard_range])"},{"line_number":590,"context_line":"            self.logger.info(\u0027Finished processing %s/%s state %s\u0027,"},{"line_number":591,"context_line":"                             broker.account, broker.container,"},{"line_number":592,"context_line":"                             broker.get_db_state())"},{"line_number":593,"context_line":"        finally:"},{"line_number":594,"context_line":"            self.logger.increment(\u0027scanned\u0027)"},{"line_number":595,"context_line":"            self.stats[\u0027containers_scanned\u0027] +\u003d 1"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_47d4a220","line":592,"range":{"start_line":592,"start_character":29,"end_line":592,"end_character":50},"updated":"2018-01-04 23:34:39.000000000","message":"from swift.container.backend import ..., DB_STATE\n  ...\n\n  self.logger.info(\u0027Finished processing %s/%s state %s\u0027,\n                   broker.account, broker.container,\n                   DB_STATE[broker.get_db_state()])","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6a9b8b59e84ec3e60e75770839c1ca540490bfff","unresolved":false,"context_lines":[{"line_number":589,"context_line":"                        [own_shard_range])"},{"line_number":590,"context_line":"            self.logger.info(\u0027Finished processing %s/%s state %s\u0027,"},{"line_number":591,"context_line":"                             broker.account, broker.container,"},{"line_number":592,"context_line":"                             broker.get_db_state())"},{"line_number":593,"context_line":"        finally:"},{"line_number":594,"context_line":"            self.logger.increment(\u0027scanned\u0027)"},{"line_number":595,"context_line":"            self.stats[\u0027containers_scanned\u0027] +\u003d 1"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_468edb2e","line":592,"range":{"start_line":592,"start_character":29,"end_line":592,"end_character":50},"in_reply_to":"9f91af0f_47d4a220","updated":"2018-01-05 11:03:35.000000000","message":"Done","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"05d2287e4d2bb6485db1d7c40db1fcc8cfd7db9a","unresolved":false,"context_lines":[{"line_number":796,"context_line":"        # with a newer timestamp means that updates from the new shard"},{"line_number":797,"context_line":"        # container will be ignored until cleaving replicates objects to it and"},{"line_number":798,"context_line":"        # simultaneously updates timestamps to the persisted shard range"},{"line_number":799,"context_line":"        # timestamps."},{"line_number":800,"context_line":"        # TODO: we may need to mark the state of these freshly minted"},{"line_number":801,"context_line":"        # shard_ranges so that we do not redirect object updates to them until"},{"line_number":802,"context_line":"        # the PUTs succeed (e.g. morph deleted filed to a state field)...or"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_67a526b5","line":799,"updated":"2018-01-04 23:34:39.000000000","message":"genius!","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"05d2287e4d2bb6485db1d7c40db1fcc8cfd7db9a","unresolved":false,"context_lines":[{"line_number":801,"context_line":"        # shard_ranges so that we do not redirect object updates to them until"},{"line_number":802,"context_line":"        # the PUTs succeed (e.g. morph deleted filed to a state field)...or"},{"line_number":803,"context_line":"        # just accept that initially some redirected updates may fail and"},{"line_number":804,"context_line":"        # eventually come back to the root"},{"line_number":805,"context_line":"        persisted_ranges \u003d [dict(sr, created_at\u003dTimestamp.now().internal)"},{"line_number":806,"context_line":"                            for sr in found_ranges]"},{"line_number":807,"context_line":"        broker.merge_shard_ranges(persisted_ranges)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_079a1a62","line":804,"updated":"2018-01-04 23:34:39.000000000","message":"OK so we send the ranges up to the root before we send a container PUT for the shardrange containers.\n\nMy gut reaction is to KISS. We already have swift mechamism called async_pending, so lets just let it pick up after us if something is not quite ready, ie handling any updates that come before the PUTs. Adding new state here makes me feel we have more places where things can go wrong.","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6a9b8b59e84ec3e60e75770839c1ca540490bfff","unresolved":false,"context_lines":[{"line_number":801,"context_line":"        # shard_ranges so that we do not redirect object updates to them until"},{"line_number":802,"context_line":"        # the PUTs succeed (e.g. morph deleted filed to a state field)...or"},{"line_number":803,"context_line":"        # just accept that initially some redirected updates may fail and"},{"line_number":804,"context_line":"        # eventually come back to the root"},{"line_number":805,"context_line":"        persisted_ranges \u003d [dict(sr, created_at\u003dTimestamp.now().internal)"},{"line_number":806,"context_line":"                            for sr in found_ranges]"},{"line_number":807,"context_line":"        broker.merge_shard_ranges(persisted_ranges)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_f0b6c5fa","line":804,"in_reply_to":"9f91af0f_079a1a62","updated":"2018-01-05 11:03:35.000000000","message":"+1 for KISS. However, we may need to add state anyway because we currently have no way of tracking whether the shard container PUTs that follow actually succeed, and therefore no re-try if they do not.\n\n(Note that before we only persisted shard ranges whose containers had been PUT, so if PUT failed, we\u0027d retry those shard ranges...BUT, retry meant repeat the scanning which is expensive. Now, we persist the result of scanning, but that means we need a state to indicate that a container PUT has succeeded)","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"05d2287e4d2bb6485db1d7c40db1fcc8cfd7db9a","unresolved":false,"context_lines":[{"line_number":823,"context_line":"                self.logger.error(\u0027PUT of new shard containers failed, \u0027"},{"line_number":824,"context_line":"                                  \u0027aborting split of %s/%s. \u0027"},{"line_number":825,"context_line":"                                  \u0027Will try again next cycle\u0027,"},{"line_number":826,"context_line":"                                  broker.account, broker.container)"},{"line_number":827,"context_line":"                break"},{"line_number":828,"context_line":"            # TODO: increment here or in the try clause above?"},{"line_number":829,"context_line":"            self.logger.increment(\u0027shard_ranges_found\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_a7c1ee30","line":826,"updated":"2018-01-04 23:34:39.000000000","message":"I guess the async pendings will pile up until this resolves itself.. but I think I\u0027m ok with that :) At least we\u0027re not dropping anything on the floor and doing our best.","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6a9b8b59e84ec3e60e75770839c1ca540490bfff","unresolved":false,"context_lines":[{"line_number":823,"context_line":"                self.logger.error(\u0027PUT of new shard containers failed, \u0027"},{"line_number":824,"context_line":"                                  \u0027aborting split of %s/%s. \u0027"},{"line_number":825,"context_line":"                                  \u0027Will try again next cycle\u0027,"},{"line_number":826,"context_line":"                                  broker.account, broker.container)"},{"line_number":827,"context_line":"                break"},{"line_number":828,"context_line":"            # TODO: increment here or in the try clause above?"},{"line_number":829,"context_line":"            self.logger.increment(\u0027shard_ranges_found\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_50a531b4","line":826,"in_reply_to":"9f91af0f_a7c1ee30","updated":"2018-01-05 11:03:35.000000000","message":"Yep, but (see previous comment)...currently nothing resolves this :(","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"05d2287e4d2bb6485db1d7c40db1fcc8cfd7db9a","unresolved":false,"context_lines":[{"line_number":935,"context_line":"            # eventually need sharding, we don\u0027t want Scan-Done prematurely"},{"line_number":936,"context_line":"            # set. This is an argument for basing \u0027scan done condition\u0027 on the"},{"line_number":937,"context_line":"            # existence of a shard range whose upper \u003e\u003d shard own range, rather"},{"line_number":938,"context_line":"            # than using sysmeta Scan-Done."},{"line_number":939,"context_line":"            self._update_shard_ranges("},{"line_number":940,"context_line":"                donor.account, donor.container, \u0027PUT\u0027, [acceptor])"},{"line_number":941,"context_line":"            # TODO: not sure we really want to delete the donor yet but need"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_670a068b","line":938,"updated":"2018-01-04 23:34:39.000000000","message":"hmm, interesting idea. I do like reducing required sysmeta with something we can deduce from current state.","commit_id":"3a4064cdd7a5da680a49886c785bf1018c8c656c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"74ff1c07f4e251254dfb81cad05eff43756896ec","unresolved":false,"context_lines":[{"line_number":547,"context_line":"                    # content of the new cleave broker, which is only the"},{"line_number":548,"context_line":"                    # objects being merged form the donor, so is gauranteed to"},{"line_number":549,"context_line":"                    # be less than the new total for the acceptor shard!"},{"line_number":550,"context_line":"                    # Update: we now update root container with shard"},{"line_number":551,"context_line":"                    # container\u0027s shard ranges during find_ranges, so a"},{"line_number":552,"context_line":"                    # sharding shard updates root with the new shards there"},{"line_number":553,"context_line":"                    # TODO: these shard ranges may have existed for some time"},{"line_number":554,"context_line":"                    # and already be updating the root with their usage e.g."},{"line_number":555,"context_line":"                    # multiple cycles before we finished cleaving, or process"}],"source_content_type":"text/x-python","patch_set":5,"id":"7f96bb07_f70d8d90","line":552,"range":{"start_line":550,"start_character":21,"end_line":552,"end_character":75},"updated":"2018-01-12 09:58:45.000000000","message":"lol, it makes sense, but so many shards in one sentence :P","commit_id":"f2d8a3638efda3e1172bf6331ff19583509ce290"}]}