)]}'
{"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":4208,"context_line":"        :param lower: the lower bound of object names contained in the shard;"},{"line_number":4209,"context_line":"            the lower bound is not included in the shard namespace."},{"line_number":4210,"context_line":"        :param upper: the upper bound of object names contained in the shard;"},{"line_number":4211,"context_line":"            the upper bound is included in the shard namespace."},{"line_number":4212,"context_line":"        :param object_count: the number of objects in the shard, defaults to"},{"line_number":4213,"context_line":"            zero."},{"line_number":4214,"context_line":"        :param bytes_used: the number of bytes in the shard; defaults to zero."}],"source_content_type":"text/x-python","patch_set":2,"id":"9f5c4f37_23e30e36","line":4211,"updated":"2017-09-21 20:04:47.000000000","message":"+1. Maybe we can even make if bold when we generate docs? Seems worth highlighting given how `lower` is basically identical to `marker` while `upper` is different from `end_marker` in this subtle way.","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"}],"swift/container/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":1679,"context_line":"            \u0027X-Container-Sysmeta-Shard-Container\u0027, (None, None))"},{"line_number":1680,"context_line":"        return account or self.account, container or self.container"},{"line_number":1681,"context_line":""},{"line_number":1682,"context_line":"    def _get_next_shard_range_upper(self, shard_size, last_upper\u003dNone,"},{"line_number":1683,"context_line":"                                    connection\u003dNone):"},{"line_number":1684,"context_line":"        \"\"\""},{"line_number":1685,"context_line":"        Finds the next entry in the shards table to use as a shard range"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f5c4f37_23742e35","line":1682,"range":{"start_line":1682,"start_character":42,"end_line":1682,"end_character":52},"updated":"2017-09-21 20:04:47.000000000","message":"OK, so it\u0027s more than a simple name change -- we\u0027ve changed the *meaning* of this param... but I\u0027m pretty sure I like it better this way.","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":1699,"context_line":"                args.append(last_upper)"},{"line_number":1700,"context_line":"            sql +\u003d \"ORDER BY name LIMIT 1 OFFSET %d\" % (shard_size - 1)"},{"line_number":1701,"context_line":"            row \u003d connection.execute(sql, args).fetchone()"},{"line_number":1702,"context_line":"            return row[\u0027name\u0027] if row else None"},{"line_number":1703,"context_line":""},{"line_number":1704,"context_line":"        self._commit_puts_stale_ok()"},{"line_number":1705,"context_line":"        if connection:"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_86c54a5e","line":1702,"updated":"2017-09-21 20:04:47.000000000","message":"*So* much better not to have the\n\n except Exception:\n     return \u0027\u0027","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":1713,"context_line":"            finally:"},{"line_number":1714,"context_line":"                self._create_connection()"},{"line_number":1715,"context_line":""},{"line_number":1716,"context_line":"    def _generate_shard_range_name(self, shard_range, timestamp):"},{"line_number":1717,"context_line":"        root_account, root_container \u003d self.get_shard_root_path()"},{"line_number":1718,"context_line":"        md5sum \u003d md5(\"%s-%s\" % (shard_range, timestamp.internal)).hexdigest()"},{"line_number":1719,"context_line":"        return \"%s-%s\" % (root_container, md5sum)"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f5c4f37_03d36a3a","line":1716,"range":{"start_line":1716,"start_character":41,"end_line":1716,"end_character":52},"updated":"2017-09-21 20:04:47.000000000","message":"Better as range_upper, or just upper? I had this brief fear that changing ShardRange\u0027s __str__ might change how we generate names...\n\nThough I suppose it shouldn\u0027t really matter -- names are assigned on creation, but after that they\u0027re an opaque identifier, yeah?","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":1754,"context_line":"            last_shard_upper \u003d existing_ranges[-1].upper"},{"line_number":1755,"context_line":"            if last_shard_upper \u003d\u003d cont_upper:"},{"line_number":1756,"context_line":"                # all ranges were previously found"},{"line_number":1757,"context_line":"                # TODO: this assumes that cont_upper does not change - safe?"},{"line_number":1758,"context_line":"                return [], True"},{"line_number":1759,"context_line":"        elif shard_size \u003e\u003d object_count:"},{"line_number":1760,"context_line":"            # container not big enough to shard"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_e6ce3e82","line":1757,"updated":"2017-09-21 20:04:47.000000000","message":"cont_upper can change, but that should only happen when we\u0027re shrinking, right? So it shouldn\u0027t impact us here.","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":1769,"context_line":"                # TODO: should we refresh object count before this test or are"},{"line_number":1770,"context_line":"                # we safe assuming that no updates to the container will be"},{"line_number":1771,"context_line":"                # made while we\u0027re scanning?"},{"line_number":1772,"context_line":"                next_shard_upper \u003d None"},{"line_number":1773,"context_line":"            else:"},{"line_number":1774,"context_line":"                try:"},{"line_number":1775,"context_line":"                    next_shard_upper \u003d self._get_next_shard_range_upper("}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_a6c84674","line":1772,"updated":"2017-09-21 20:04:47.000000000","message":"So is this just to try to save us a query? Granted, Matt showed these OFFSET queries aren\u0027t exactly cheap, but I wonder if it\u0027d be better to check and *know* when we\u0027re done than to guess, even if we\u0027re mostly accurate.\n\nOn the other hand, in the \"normal\" case we\u0027re cutting it down from 2 such queries to 1, which ain\u0027t nothing...","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"302440885350f5251175d4fe948379b3f8810b0b","unresolved":false,"context_lines":[{"line_number":1769,"context_line":"                # TODO: should we refresh object count before this test or are"},{"line_number":1770,"context_line":"                # we safe assuming that no updates to the container will be"},{"line_number":1771,"context_line":"                # made while we\u0027re scanning?"},{"line_number":1772,"context_line":"                next_shard_upper \u003d None"},{"line_number":1773,"context_line":"            else:"},{"line_number":1774,"context_line":"                try:"},{"line_number":1775,"context_line":"                    next_shard_upper \u003d self._get_next_shard_range_upper("}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_384d5c72","line":1772,"in_reply_to":"7f515b1d_a6c84674","updated":"2017-09-25 09:08:17.000000000","message":"I have an idea how to make this more accurate by tracking progress in sysmeta. But like you I also wondered if we should \u0027just do it\u0027.","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"}],"swift/container/sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":845,"context_line":"                    \u0027X-Container-Sysmeta-Shard-Timestamp\u0027:"},{"line_number":846,"context_line":"                        shard_range.timestamp.internal,"},{"line_number":847,"context_line":"                    \u0027X-Container-Sysmeta-Shard-Meta-Timestamp\u0027:"},{"line_number":848,"context_line":"                        shard_range.timestamp.internal})"},{"line_number":849,"context_line":"                self.swift.create_container(shard_account, shard_range.name,"},{"line_number":850,"context_line":"                                            headers\u003dheaders)"},{"line_number":851,"context_line":"                shard_ranges.append(shard_range)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_06127ab2","line":848,"range":{"start_line":848,"start_character":36,"end_line":848,"end_character":45},"updated":"2017-09-21 20:04:47.000000000","message":"nit: meta_timestamp. Though, since this was *just* created, the two should match.","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"302440885350f5251175d4fe948379b3f8810b0b","unresolved":false,"context_lines":[{"line_number":845,"context_line":"                    \u0027X-Container-Sysmeta-Shard-Timestamp\u0027:"},{"line_number":846,"context_line":"                        shard_range.timestamp.internal,"},{"line_number":847,"context_line":"                    \u0027X-Container-Sysmeta-Shard-Meta-Timestamp\u0027:"},{"line_number":848,"context_line":"                        shard_range.timestamp.internal})"},{"line_number":849,"context_line":"                self.swift.create_container(shard_account, shard_range.name,"},{"line_number":850,"context_line":"                                            headers\u003dheaders)"},{"line_number":851,"context_line":"                shard_ranges.append(shard_range)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_d882c84f","line":848,"range":{"start_line":848,"start_character":36,"end_line":848,"end_character":45},"in_reply_to":"7f515b1d_06127ab2","updated":"2017-09-25 09:08:17.000000000","message":"+1 precision is good vs implication, good catch","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":862,"context_line":""},{"line_number":863,"context_line":"        # persist shard ranges for successfully created shard containers and"},{"line_number":864,"context_line":"        # replicate this container db"},{"line_number":865,"context_line":"        # TODO: better if we persist the found ranges even if we fail to create"},{"line_number":866,"context_line":"        # all the shard containers, so we do not waste the work done to find"},{"line_number":867,"context_line":"        # them. that implies havng a way to track if containers have been"},{"line_number":868,"context_line":"        # created or not for persisted shard ranges"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_46f4f25b","line":865,"range":{"start_line":865,"start_character":16,"end_line":865,"end_character":53},"updated":"2017-09-21 20:04:47.000000000","message":"+1, agreed","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":862,"context_line":""},{"line_number":863,"context_line":"        # persist shard ranges for successfully created shard containers and"},{"line_number":864,"context_line":"        # replicate this container db"},{"line_number":865,"context_line":"        # TODO: better if we persist the found ranges even if we fail to create"},{"line_number":866,"context_line":"        # all the shard containers, so we do not waste the work done to find"},{"line_number":867,"context_line":"        # them. that implies havng a way to track if containers have been"},{"line_number":868,"context_line":"        # created or not for persisted shard ranges"},{"line_number":869,"context_line":"        items \u003d self._generate_object_list(shard_ranges, 0)"}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_664b6e87","line":866,"range":{"start_line":865,"start_character":59,"end_line":866,"end_character":34},"updated":"2017-09-21 20:04:47.000000000","message":"This should be unlikely, though, right? We\u0027re all about trading off consistency for availability -- odds should be pretty good that container creation will succeed, but some replicas may be on handoffs.","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":864,"context_line":"        # replicate this container db"},{"line_number":865,"context_line":"        # TODO: better if we persist the found ranges even if we fail to create"},{"line_number":866,"context_line":"        # all the shard containers, so we do not waste the work done to find"},{"line_number":867,"context_line":"        # them. that implies havng a way to track if containers have been"},{"line_number":868,"context_line":"        # created or not for persisted shard ranges"},{"line_number":869,"context_line":"        items \u003d self._generate_object_list(shard_ranges, 0)"},{"line_number":870,"context_line":"        broker.merge_items(items)"},{"line_number":871,"context_line":"        self.cpool.spawn("}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_86a78a50","line":868,"range":{"start_line":867,"start_character":50,"end_line":868,"end_character":51},"updated":"2017-09-21 20:04:47.000000000","message":"How much does this actually matter? We\u0027re still doing the record-movement locally, yeah? So won\u0027t replication eventually get it where it belongs?","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"302440885350f5251175d4fe948379b3f8810b0b","unresolved":false,"context_lines":[{"line_number":864,"context_line":"        # replicate this container db"},{"line_number":865,"context_line":"        # TODO: better if we persist the found ranges even if we fail to create"},{"line_number":866,"context_line":"        # all the shard containers, so we do not waste the work done to find"},{"line_number":867,"context_line":"        # them. that implies havng a way to track if containers have been"},{"line_number":868,"context_line":"        # created or not for persisted shard ranges"},{"line_number":869,"context_line":"        items \u003d self._generate_object_list(shard_ranges, 0)"},{"line_number":870,"context_line":"        broker.merge_items(items)"},{"line_number":871,"context_line":"        self.cpool.spawn("}],"source_content_type":"text/x-python","patch_set":2,"id":"7f515b1d_18898074","line":868,"range":{"start_line":867,"start_character":50,"end_line":868,"end_character":51},"in_reply_to":"7f515b1d_86a78a50","updated":"2017-09-25 09:08:17.000000000","message":"what I really want to avoid (and hence keep in mid by the TODO) is the root starting to redirect GETs to shard containers that have not yet been created/populated. Maybe that is already handled by code that I have yet to fully work through. But it seemed like we might want to persist a shard range that has been found but otherwise has no resources associated with it, yet.","commit_id":"b466d13ee7123d6719802196dc223c6f02e3504e"}],"test/probe/test_sharder.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"5bc90a688c2a26d816d6555f5807cd3d175518ae","unresolved":false,"context_lines":[{"line_number":111,"context_line":""},{"line_number":112,"context_line":"    def _test_sharded_listing(self, run_replicators\u003dFalse):"},{"line_number":113,"context_line":"        sharders \u003d Manager([\u0027container-sharder\u0027])"},{"line_number":114,"context_line":"        obj_names \u003d [\u0027obj%03d\u0027 % x for x in range(self.max_shard_size + 1)]"},{"line_number":115,"context_line":""},{"line_number":116,"context_line":"        self.brain.put_container()"},{"line_number":117,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"9f5c4f37_a35b7e2c","side":"PARENT","line":114,"updated":"2017-09-21 20:04:47.000000000","message":"Cool. I never really liked that +1, but included it to *make sure* it cleaved :-)","commit_id":"3737fee4943160e13d37c8a387d943fa636669dd"}]}