)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"177af4e427b720140682f551c2c68e7a9414bf40","unresolved":true,"context_lines":[{"line_number":20,"context_line":""},{"line_number":21,"context_line":"The patch also adds SHRUNK to the in_progress ShardRange state check so"},{"line_number":22,"context_line":"a final shrinking cleave (treated as cleaving) can also update the"},{"line_number":23,"context_line":"current cleaving states."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I5fa39d41f9cd3b211e45d2012fd709f4135f595e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"95e56b5c_43de4b0c","line":23,"updated":"2021-02-08 06:16:43.000000000","message":"Shrinking tends to not take more then a cycle atm, so maybe this isn\u0027t required, but did it for completeness sake.","commit_id":"5ce8cb89eb4068477baaf431f42e4ac8095bbd20"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1a033ee8c82423d3ee793b271167bb2d95ec7d47","unresolved":true,"context_lines":[{"line_number":20,"context_line":""},{"line_number":21,"context_line":"The patch also adds SHRUNK to the in_progress ShardRange state check so"},{"line_number":22,"context_line":"a final shrinking cleave (treated as cleaving) can also update the"},{"line_number":23,"context_line":"current cleaving states."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I5fa39d41f9cd3b211e45d2012fd709f4135f595e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"f14cde75_cc5ca210","line":23,"in_reply_to":"95e56b5c_43de4b0c","updated":"2021-02-08 13:52:09.000000000","message":"not sure I understand this change - shrinking shouldn\u0027t change acceptor states so there isn\u0027t really any progress through \u0027sub-shards\u0027 - or am I missing something?","commit_id":"5ce8cb89eb4068477baaf431f42e4ac8095bbd20"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5ba33aed03112412cf1fa9c6efa5639e81534cf8","unresolved":true,"context_lines":[{"line_number":20,"context_line":""},{"line_number":21,"context_line":"The patch also adds SHRUNK to the in_progress ShardRange state check so"},{"line_number":22,"context_line":"a final shrinking cleave (treated as cleaving) can also update the"},{"line_number":23,"context_line":"current cleaving states."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"Change-Id: I5fa39d41f9cd3b211e45d2012fd709f4135f595e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"a27406ce_98c1dd54","line":23,"in_reply_to":"f14cde75_cc5ca210","updated":"2021-02-09 00:37:27.000000000","message":"Your probably right, I haven\u0027t gotten far enough to actually do any testing yet. Was more thinking about if a shard ever needs to cleave to a bunch of other shards (ie fixing via shrinking), then there could be more shards than what could be completed in one run of the sharder, and so we may want to monitor this progress too.\n\nBut yeah, thinking about it, we may not get per shardrange status updates during the shard, but might get some of the stats that could be useful.. ie have the shard finished, ie in SHRUNK state.","commit_id":"5ce8cb89eb4068477baaf431f42e4ac8095bbd20"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c56d5c68c56eac52d139afc0f7a314690b71a8cb","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Report final in_progress when sharding is complete"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"On every sharder cycle up update in progress recon stats for each sharding"},{"line_number":10,"context_line":"container. However, we tend to not run it one final time once sharding"},{"line_number":11,"context_line":"is complete because the DB state is changed to SHARDED and therefore the"},{"line_number":12,"context_line":"in_progress stats never get their final update."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"473e26dd_27df0380","line":9,"range":{"start_line":9,"start_character":17,"end_line":9,"end_character":37},"updated":"2021-02-23 13:25:05.000000000","message":"something wonky here - maybe \u0027up\u0027 is a typo?","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"}],"etc/container-server.conf-sample":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":446,"context_line":"# The recon_sharded_timeout parameter says for how long a container whose just finished"},{"line_number":447,"context_line":"# sharding should be checked by the in-progress check. This is to allow anything monitoring"},{"line_number":448,"context_line":"# the sharding recon dump to have enough time to collate and see things complete."},{"line_number":449,"context_line":"# The default 12 hours (12 x 60 x 60)"},{"line_number":450,"context_line":"# recon_sharded_timeout \u003d 43200"},{"line_number":451,"context_line":"#"},{"line_number":452,"context_line":"# Large databases tend to take a while to work with, but we want to make sure"}],"source_content_type":"application/octet-stream","patch_set":13,"id":"7e6b1cfa_1f75d857","line":449,"range":{"start_line":449,"start_character":2,"end_line":449,"end_character":22},"updated":"2021-02-26 14:48:10.000000000","message":"typo: The default is 12 hours\n\nalso, be worth mentioning that this setting should be \u003c\u003d reclaim_age and will be effectively capped at reclaim_age","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"}],"swift/common/middleware/recon.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c56d5c68c56eac52d139afc0f7a314690b71a8cb","unresolved":true,"context_lines":[{"line_number":379,"context_line":"        elif rcheck \u003d\u003d \"time\":"},{"line_number":380,"context_line":"            content \u003d self.get_time()"},{"line_number":381,"context_line":"        elif rcheck \u003d\u003d \"sharding\":"},{"line_number":382,"context_line":"            content \u003d self.get_sharding_info()"},{"line_number":383,"context_line":"        else:"},{"line_number":384,"context_line":"            content \u003d \"Invalid path: %s\" % req.path"},{"line_number":385,"context_line":"            return Response(request\u003dreq, status\u003d\"404 Not Found\","}],"source_content_type":"text/x-python","patch_set":4,"id":"b6807a92_3cc63bce","line":382,"updated":"2021-02-23 13:25:05.000000000","message":"this feels like a worthy change in its own right - could be broken out into a pre-patch? it\u0027s going to need unit tests either way :)","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b52aba218514d81df98148c1f65cbb043bb82543","unresolved":true,"context_lines":[{"line_number":379,"context_line":"        elif rcheck \u003d\u003d \"time\":"},{"line_number":380,"context_line":"            content \u003d self.get_time()"},{"line_number":381,"context_line":"        elif rcheck \u003d\u003d \"sharding\":"},{"line_number":382,"context_line":"            content \u003d self.get_sharding_info()"},{"line_number":383,"context_line":"        else:"},{"line_number":384,"context_line":"            content \u003d \"Invalid path: %s\" % req.path"},{"line_number":385,"context_line":"            return Response(request\u003dreq, status\u003d\"404 Not Found\","}],"source_content_type":"text/x-python","patch_set":4,"id":"df8ea260_4cd4abbe","line":382,"in_reply_to":"b6807a92_3cc63bce","updated":"2021-02-24 00:17:42.000000000","message":"yup good idea. Ended up piping it all through because I couldn\u0027t think of a better way of accessing the recon data for the recon test, so it ended up being hacked on. But yeah, it is extending the scope of this change.. so will split it 😊","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"5af8f6bbd97e78c3f7d9ae8556a838948eda5728","unresolved":true,"context_lines":[{"line_number":1485,"context_line":"            modified_shard_ranges.append(own_shard_range)"},{"line_number":1486,"context_line":"            broker.merge_shard_ranges(modified_shard_ranges)"},{"line_number":1487,"context_line":"            # Update status before we move the DB to sharded state so"},{"line_number":1488,"context_line":"            # we can update final sharding stats."},{"line_number":1489,"context_line":"            if broker.set_sharded_state():"},{"line_number":1490,"context_line":"                cleaving_context.delete(broker)"},{"line_number":1491,"context_line":"                self._record_sharding_progress(broker, node, None, force\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":2,"id":"f0bd33e9_e3b9e2ba","line":1488,"updated":"2021-02-18 10:03:00.000000000","message":"opss forgot to move the comment as well.","commit_id":"32d976b110e22536893fc7cb366d1ed237994b7c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c56d5c68c56eac52d139afc0f7a314690b71a8cb","unresolved":true,"context_lines":[{"line_number":493,"context_line":""},{"line_number":494,"context_line":"    def _record_sharding_progress(self, broker, node, error, force\u003dFalse):"},{"line_number":495,"context_line":"        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":496,"context_line":"        if (force or (broker.get_db_state() in (UNSHARDED, SHARDING) and"},{"line_number":497,"context_line":"                      own_shard_range.state in (ShardRange.SHARDING,"},{"line_number":498,"context_line":"                                                ShardRange.SHARDED))):"},{"line_number":499,"context_line":"            info \u003d self._make_stats_info(broker, node, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fdc92eb9_b6f19110","line":496,"range":{"start_line":496,"start_character":12,"end_line":496,"end_character":17},"updated":"2021-02-23 13:25:05.000000000","message":"I\u0027m thinking that maybe SHARDED should be included on *every cycle* for some fixed time period after completion, so at least an op has some idea how long to expect the data to be available. Otherwise the new state might disappear before it\u0027s read.a\n\nDeducing the time since sharding completed is surprisingly not obvious to me! We may need to tweak the state_timestamp in _complete_sharding, or actually measure time from the start of sharding using the epoch???","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b52aba218514d81df98148c1f65cbb043bb82543","unresolved":true,"context_lines":[{"line_number":493,"context_line":""},{"line_number":494,"context_line":"    def _record_sharding_progress(self, broker, node, error, force\u003dFalse):"},{"line_number":495,"context_line":"        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":496,"context_line":"        if (force or (broker.get_db_state() in (UNSHARDED, SHARDING) and"},{"line_number":497,"context_line":"                      own_shard_range.state in (ShardRange.SHARDING,"},{"line_number":498,"context_line":"                                                ShardRange.SHARDED))):"},{"line_number":499,"context_line":"            info \u003d self._make_stats_info(broker, node, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":4,"id":"638b2a5c_3b902830","line":496,"range":{"start_line":496,"start_character":12,"end_line":496,"end_character":17},"in_reply_to":"fdc92eb9_b6f19110","updated":"2021-02-24 00:17:42.000000000","message":"Yeah, some kind of timing would be ideal. Just not sure how to do that. You mention here: https://review.opendev.org/c/openstack/swift/+/558154 why we don\u0027t increment the state timestamp, mostly only wanting the leader to do it. But maybe it\u0027s safe to revise that idea and think through the consequences.\n\nMy initial thought of the \"force\" is to give us the ability to force recording progress in case we ever need to.","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c56d5c68c56eac52d139afc0f7a314690b71a8cb","unresolved":true,"context_lines":[{"line_number":1487,"context_line":"            if broker.set_sharded_state():"},{"line_number":1488,"context_line":"                cleaving_context.delete(broker)"},{"line_number":1489,"context_line":"                # Update the in_progress stats to say it\u0027s finished sharding"},{"line_number":1490,"context_line":"                self._record_sharding_progress(broker, node, None, force\u003dTrue)"},{"line_number":1491,"context_line":"                return True"},{"line_number":1492,"context_line":"            else:"},{"line_number":1493,"context_line":"                self.logger.warning("}],"source_content_type":"text/x-python","patch_set":4,"id":"73c64c52_b7191d23","line":1490,"updated":"2021-02-23 13:25:05.000000000","message":"ok, but this happens just once, so may only be reported for the duration of one sharder cycle","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b52aba218514d81df98148c1f65cbb043bb82543","unresolved":true,"context_lines":[{"line_number":1487,"context_line":"            if broker.set_sharded_state():"},{"line_number":1488,"context_line":"                cleaving_context.delete(broker)"},{"line_number":1489,"context_line":"                # Update the in_progress stats to say it\u0027s finished sharding"},{"line_number":1490,"context_line":"                self._record_sharding_progress(broker, node, None, force\u003dTrue)"},{"line_number":1491,"context_line":"                return True"},{"line_number":1492,"context_line":"            else:"},{"line_number":1493,"context_line":"                self.logger.warning("}],"source_content_type":"text/x-python","patch_set":4,"id":"112e1616_71a88161","line":1490,"in_reply_to":"73c64c52_b7191d23","updated":"2021-02-24 00:17:42.000000000","message":"We could move this up to before the:\n\n  if broker.set_sharded_state()\n\nLike I had in the previous patchset. This would mean we wont have to have the \"force\" option, only the complete results will say it\u0027s still in sharding state in the progress out put.","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"670e1dfa33eab013a1b4c0b1142eec7a5c991900","unresolved":true,"context_lines":[{"line_number":274,"context_line":"                    except ValueError:"},{"line_number":275,"context_line":"                        continue"},{"line_number":276,"context_line":"                elif include_deleted:"},{"line_number":277,"context_line":"                    yield val, timestamp"},{"line_number":278,"context_line":""},{"line_number":279,"context_line":"    @classmethod"},{"line_number":280,"context_line":"    def load(cls, broker):"}],"source_content_type":"text/x-python","patch_set":9,"id":"67596d6b_5830674b","line":277,"updated":"2021-02-25 15:05:33.000000000","message":"do we care about deleted contexts - they should be at least reclaim_age older than the completion of sharding, and we only care about contexts for 12 hours after completion?","commit_id":"95d1244e020abd25242fd37745c93694837b552a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"670e1dfa33eab013a1b4c0b1142eec7a5c991900","unresolved":true,"context_lines":[{"line_number":504,"context_line":"                and own_shard_range.state in (ShardRange.SHARDING,"},{"line_number":505,"context_line":"                                              ShardRange.SHARDED)):"},{"line_number":506,"context_line":"            if db_state \u003d\u003d SHARDED:"},{"line_number":507,"context_line":"                context_ts \u003d max([float(ts) for c, ts in"},{"line_number":508,"context_line":"                                  CleavingContext.load_all("},{"line_number":509,"context_line":"                                      broker, include_deleted\u003dTrue)]) or None"},{"line_number":510,"context_line":"                if not context_ts or (context_ts + self.recon_sharded_timeout"},{"line_number":511,"context_line":"                                      \u003c Timestamp.now().timestamp):"},{"line_number":512,"context_line":"                    # not contexts or last context timestamp too old for the"}],"source_content_type":"text/x-python","patch_set":9,"id":"52374087_68e8d84a","line":509,"range":{"start_line":507,"start_character":29,"end_line":509,"end_character":68},"updated":"2021-02-25 15:05:33.000000000","message":"or\n\n  max([float(ts)\n      for c, ts in CleavingContext.load_all(broker) if c.done()]\n\n\nthe done() method seems to be useful here, but was undertested: see https://review.opendev.org/c/openstack/swift/+/777604","commit_id":"95d1244e020abd25242fd37745c93694837b552a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"83afc87198997dcbb5866b02a03553f8ac9faa41","unresolved":true,"context_lines":[{"line_number":504,"context_line":"                and own_shard_range.state in (ShardRange.SHARDING,"},{"line_number":505,"context_line":"                                              ShardRange.SHARDED)):"},{"line_number":506,"context_line":"            if db_state \u003d\u003d SHARDED:"},{"line_number":507,"context_line":"                context_ts \u003d max([float(ts) for c, ts in"},{"line_number":508,"context_line":"                                  CleavingContext.load_all("},{"line_number":509,"context_line":"                                      broker, include_deleted\u003dTrue)]) or None"},{"line_number":510,"context_line":"                if not context_ts or (context_ts + self.recon_sharded_timeout"},{"line_number":511,"context_line":"                                      \u003c Timestamp.now().timestamp):"},{"line_number":512,"context_line":"                    # not contexts or last context timestamp too old for the"}],"source_content_type":"text/x-python","patch_set":9,"id":"2683d548_2d4d505f","line":509,"range":{"start_line":507,"start_character":29,"end_line":509,"end_character":68},"in_reply_to":"52374087_68e8d84a","updated":"2021-02-25 18:36:01.000000000","message":"ignore previous comment.\n\nIf we don\u0027t delete cleaving contexts immediately in _complete_sharding then what you have works but the include_deleted is unnecessary","commit_id":"95d1244e020abd25242fd37745c93694837b552a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"670e1dfa33eab013a1b4c0b1142eec7a5c991900","unresolved":true,"context_lines":[{"line_number":855,"context_line":"            if last_mod.timestamp + self.reclaim_age \u003c \\"},{"line_number":856,"context_line":"                    now.timestamp:"},{"line_number":857,"context_line":"                last_mod.offset +\u003d 1"},{"line_number":858,"context_line":"                context.delete(broker, timestamp\u003dlast_mod)"},{"line_number":859,"context_line":""},{"line_number":860,"context_line":"    def _audit_container(self, broker):"},{"line_number":861,"context_line":"        if broker.is_deleted():"}],"source_content_type":"text/x-python","patch_set":9,"id":"cc398700_fd983009","line":858,"updated":"2021-02-25 15:05:33.000000000","message":"so this halves the grace period that a cleaving context is retained in sysmeta in order to freeze the last modified time...but why do we need to do this? last_mod is at least reclaim_age in the past - isn\u0027t that way older than when we\u0027d still want to report sharding progress.\n\nOIC - because you\u0027re relying on only the timestamp to exclude from the progress report, so the timestamp can\u0027t move forward when the context is deleted.","commit_id":"95d1244e020abd25242fd37745c93694837b552a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"83afc87198997dcbb5866b02a03553f8ac9faa41","unresolved":true,"context_lines":[{"line_number":855,"context_line":"            if last_mod.timestamp + self.reclaim_age \u003c \\"},{"line_number":856,"context_line":"                    now.timestamp:"},{"line_number":857,"context_line":"                last_mod.offset +\u003d 1"},{"line_number":858,"context_line":"                context.delete(broker, timestamp\u003dlast_mod)"},{"line_number":859,"context_line":""},{"line_number":860,"context_line":"    def _audit_container(self, broker):"},{"line_number":861,"context_line":"        if broker.is_deleted():"}],"source_content_type":"text/x-python","patch_set":9,"id":"38648bda_a22ab691","line":858,"in_reply_to":"cc398700_fd983009","updated":"2021-02-25 18:36:01.000000000","message":"Update: I now see that contexts get deleted in _complete_sharding() too 😞, so some of my earlier comments are invalid.\n\nBut I\u0027d suggest we just don\u0027t do that delete - we\u0027re deleting the precise state that we need to be able to continue to report progress!\n\nIf we\u0027re worried about contexts hanging around too long, we could modify this audit step to delete after 12 hours (or w/e) if context.done() is True.\n\nAnd we should still otherwise wait for reclaim_age to delete stale contexts that don\u0027t ever get to done.","commit_id":"95d1244e020abd25242fd37745c93694837b552a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":850,"context_line":"            is_done \u003d context.done() and last_mod.timestamp + \\"},{"line_number":851,"context_line":"                self.recon_sharded_timeout \u003c now.timestamp"},{"line_number":852,"context_line":"            is_stale \u003d last_mod.timestamp + self.reclaim_age \u003c now.timestamp"},{"line_number":853,"context_line":"            if is_done or is_stale:"},{"line_number":854,"context_line":"                context.delete(broker)"},{"line_number":855,"context_line":""},{"line_number":856,"context_line":"    def _audit_container(self, broker):"}],"source_content_type":"text/x-python","patch_set":13,"id":"0f7718e3_3f855c89","line":853,"updated":"2021-02-26 14:48:10.000000000","message":"ok, nice and clear","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"683c1ef2008729897b682cb5db936557412ea097","unresolved":true,"context_lines":[{"line_number":1485,"context_line":"            modified_shard_ranges.append(own_shard_range)"},{"line_number":1486,"context_line":"            broker.merge_shard_ranges(modified_shard_ranges)"},{"line_number":1487,"context_line":"            if broker.set_sharded_state():"},{"line_number":1488,"context_line":"                cleaving_context.delete(broker)"},{"line_number":1489,"context_line":"                return True"},{"line_number":1490,"context_line":"            else:"},{"line_number":1491,"context_line":"                self.logger.warning("}],"source_content_type":"text/x-python","patch_set":14,"id":"1b0da02f_6d1d726d","side":"PARENT","line":1488,"updated":"2021-02-26 16:17:24.000000000","message":"noice","commit_id":"b1309c95e5d1219a9241bdf1243d9dbdd17cb2d0"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"683c1ef2008729897b682cb5db936557412ea097","unresolved":true,"context_lines":[{"line_number":850,"context_line":"            is_done \u003d context.done() and last_mod.timestamp + \\"},{"line_number":851,"context_line":"                self.recon_sharded_timeout \u003c now.timestamp"},{"line_number":852,"context_line":"            is_stale \u003d last_mod.timestamp + self.reclaim_age \u003c now.timestamp"},{"line_number":853,"context_line":"            if is_done or is_stale:"},{"line_number":854,"context_line":"                context.delete(broker)"},{"line_number":855,"context_line":""},{"line_number":856,"context_line":"    def _audit_container(self, broker):"}],"source_content_type":"text/x-python","patch_set":14,"id":"aa37263a_f586420a","line":853,"updated":"2021-02-26 16:17:24.000000000","message":"for a tricky as it is; this is quite readable","commit_id":"1de98348167f9923710d1a6d634ef6d8008eea5d"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c56d5c68c56eac52d139afc0f7a314690b71a8cb","unresolved":true,"context_lines":[{"line_number":177,"context_line":"        try:"},{"line_number":178,"context_line":"            resp \u003d conn.getresponse()"},{"line_number":179,"context_line":"            if resp.status // 100 \u003d\u003d 2:"},{"line_number":180,"context_line":"                data \u003d json.loads(resp.read())"},{"line_number":181,"context_line":"        except ValueError:"},{"line_number":182,"context_line":"            pass"},{"line_number":183,"context_line":"        return data"}],"source_content_type":"text/x-python","patch_set":4,"id":"50326dc0_6b5afb75","line":180,"updated":"2021-02-23 13:25:05.000000000","message":"else ?\n\nnot sure how great this idea is but you could perhaps drop the resp status into the data dict then assert its value in the test.\n\nBut I actually wonder if it would be better to add a direct_get_recon function to direct_client and make use of _make_req() there to do status checking etc?","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c56d5c68c56eac52d139afc0f7a314690b71a8cb","unresolved":true,"context_lines":[{"line_number":2665,"context_line":"        self.assertEqual(0, int(metadata.get(\u0027x-account-object-count\u0027)))"},{"line_number":2666,"context_line":"        self.assertEqual(0, int(metadata.get(\u0027x-account-bytes-used\u0027)))"},{"line_number":2667,"context_line":""},{"line_number":2668,"context_line":"    def test_sharded_in_progress_report(self):"},{"line_number":2669,"context_line":"        # verify that in_progress reports are udated when a container finishes"},{"line_number":2670,"context_line":"        # sharding. So anyone watching the shard via recon knows when it\u0027s"},{"line_number":2671,"context_line":"        # complete"}],"source_content_type":"text/x-python","patch_set":4,"id":"0fe268cf_4c1d5b53","line":2668,"updated":"2021-02-23 13:25:05.000000000","message":"can we drop the assertions into an existing probe test? I know that for unit tests its good to have separate tests for separate concerns, but probe tests take so long to run that I wonder if we could build on what we have","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b52aba218514d81df98148c1f65cbb043bb82543","unresolved":true,"context_lines":[{"line_number":2665,"context_line":"        self.assertEqual(0, int(metadata.get(\u0027x-account-object-count\u0027)))"},{"line_number":2666,"context_line":"        self.assertEqual(0, int(metadata.get(\u0027x-account-bytes-used\u0027)))"},{"line_number":2667,"context_line":""},{"line_number":2668,"context_line":"    def test_sharded_in_progress_report(self):"},{"line_number":2669,"context_line":"        # verify that in_progress reports are udated when a container finishes"},{"line_number":2670,"context_line":"        # sharding. So anyone watching the shard via recon knows when it\u0027s"},{"line_number":2671,"context_line":"        # complete"}],"source_content_type":"text/x-python","patch_set":4,"id":"232479d0_5911bbb9","line":2668,"in_reply_to":"0fe268cf_4c1d5b53","updated":"2021-02-24 00:17:42.000000000","message":"Good point :)\n\nFor a while I was trying to test as a unit test, but there was just too much to mock it stopped making sense. Although if we find a why of using a timeout check sharded. The the update progress can always happen back in the main sharding loop (And not in complete_sharding) and therefore be testing in unit testing.","commit_id":"9f461f9699b211e69e88cb26a7c2a77e7352072f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":892,"context_line":"        # Contexts should still be there, and should be complete"},{"line_number":893,"context_line":"        contexts \u003d list([ctx.done()"},{"line_number":894,"context_line":"                         for ctx, _ in CleavingContext.load_all(broker)])"},{"line_number":895,"context_line":"        self.assertTrue(all(contexts))"},{"line_number":896,"context_line":"        self.direct_delete_container(expect_failure\u003dTrue)"},{"line_number":897,"context_line":""},{"line_number":898,"context_line":"        self.assertLengthEqual(found[\u0027normal_dbs\u0027], 2)"}],"source_content_type":"text/x-python","patch_set":13,"id":"935d4ff4_d83bdb63","line":895,"updated":"2021-02-26 14:48:10.000000000","message":"all([]) is True :(","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f59ca96d7c4f6af73c421b742783002d89cdcca0","unresolved":true,"context_lines":[{"line_number":2041,"context_line":"                                         \u0027file_size\u0027: mock.ANY,"},{"line_number":2042,"context_line":"                                         \u0027meta_timestamp\u0027: mock.ANY,"},{"line_number":2043,"context_line":"                                         \u0027node_index\u0027: 0,"},{"line_number":2044,"context_line":"                                         \u0027object_count\u0027: 200,"},{"line_number":2045,"context_line":"                                         \u0027container\u0027: mock.ANY,"},{"line_number":2046,"context_line":"                                         \u0027path\u0027: mock.ANY,"},{"line_number":2047,"context_line":"                                         \u0027root\u0027: mock.ANY}]}"}],"source_content_type":"text/x-python","patch_set":13,"id":"80f4d5b1_e342035b","line":2044,"range":{"start_line":2044,"start_character":57,"end_line":2044,"end_character":60},"updated":"2021-02-26 16:03:30.000000000","message":"think you must have a different config to mine! ;) needs to be len(obj_names)","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":2123,"context_line":"                                         \u0027path\u0027: mock.ANY,"},{"line_number":2124,"context_line":"                                         \u0027root\u0027: mock.ANY}]}"},{"line_number":2125,"context_line":"        actual \u003d recon[\u0027sharding_stats\u0027][\u0027sharding\u0027][\u0027sharding_in_progress\u0027]"},{"line_number":2126,"context_line":"        self.assertEqual(expected_in_progress, actual)"},{"line_number":2127,"context_line":""},{"line_number":2128,"context_line":"    def test_sharded_delete(self):"},{"line_number":2129,"context_line":"        all_obj_names \u003d self._make_object_names(self.max_shard_size)"}],"source_content_type":"text/x-python","patch_set":13,"id":"feef21d6_a116eefd","line":2126,"updated":"2021-02-26 14:48:10.000000000","message":"nice!","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"683c1ef2008729897b682cb5db936557412ea097","unresolved":true,"context_lines":[{"line_number":41,"context_line":"from test.probe.common import ReplProbeTest, get_server_number, \\"},{"line_number":42,"context_line":"    wait_for_server_to_hangup"},{"line_number":43,"context_line":"from test.debug_logger import debug_logger"},{"line_number":44,"context_line":"import mock"},{"line_number":45,"context_line":""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"MIN_SHARD_CONTAINER_THRESHOLD \u003d 4"}],"source_content_type":"text/x-python","patch_set":14,"id":"657ab7cd_e01850fd","line":44,"updated":"2021-02-26 16:17:24.000000000","message":"oic, i guess this is new","commit_id":"1de98348167f9923710d1a6d634ef6d8008eea5d"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"670e1dfa33eab013a1b4c0b1142eec7a5c991900","unresolved":true,"context_lines":[{"line_number":731,"context_line":"            # recon_sharded_timeout timer."},{"line_number":732,"context_line":"            cxt \u003d CleavingContext.load(brokers[0])"},{"line_number":733,"context_line":"            with mock_timestamp_now(next(self.ts_iter)):"},{"line_number":734,"context_line":"                cxt.store(brokers[0])"},{"line_number":735,"context_line":"            ts_now \u003d next(self.ts_iter)"},{"line_number":736,"context_line":"            cxt.delete(brokers[0], timestamp\u003dts_now)"},{"line_number":737,"context_line":"            brokers[0].set_sharded_state()"}],"source_content_type":"text/x-python","patch_set":9,"id":"126a4a68_1c5da00c","line":734,"updated":"2021-02-25 15:05:33.000000000","message":"in reality the context would satisfy context.done() \u003d\u003d True","commit_id":"95d1244e020abd25242fd37745c93694837b552a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"3325ad49bb823d91067c3d5488ebe73b132f7cf1","unresolved":true,"context_lines":[{"line_number":728,"context_line":"            own_sr.update_state(ShardRange.SHARDED)"},{"line_number":729,"context_line":"            brokers[0].merge_shard_ranges(shard_ranges + [own_sr])"},{"line_number":730,"context_line":"            # make and destroy a cleave context, this is used for the"},{"line_number":731,"context_line":"            # recon_sharded_timeout timer."},{"line_number":732,"context_line":"            cxt \u003d CleavingContext.load(brokers[0])"},{"line_number":733,"context_line":"            cxt.misplaced_done \u003d cxt.cleaving_done \u003d True"},{"line_number":734,"context_line":"            ts_now \u003d next(self.ts_iter)"}],"source_content_type":"text/x-python","patch_set":10,"id":"57ca0f73_16908858","line":731,"updated":"2021-02-26 01:34:02.000000000","message":"Opps I guess this comment isn\u0027t correct anymore","commit_id":"898e9ca4ed9a3f341153a31b1dbd89add9badac4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":760,"context_line":"                         \u0027state\u0027: \u0027sharding\u0027, \u0027db_state\u0027: \u0027unsharded\u0027,"},{"line_number":761,"context_line":"                         \u0027error\u0027: None}]}"},{"line_number":762,"context_line":"            self._assert_stats("},{"line_number":763,"context_line":"                expected_in_progress_stats, sharder, \u0027sharding_in_progress\u0027)"},{"line_number":764,"context_line":""},{"line_number":765,"context_line":"            # when me move recon_sharded_timeout + 1 seconds into the future,"},{"line_number":766,"context_line":"            # bocker 1 will be removed."}],"source_content_type":"text/x-python","patch_set":13,"id":"70fbd120_287010d5","line":763,"updated":"2021-02-26 14:48:10.000000000","message":"could do one more cycle at recon_shard_timeout to check the completed broker is still reported","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":762,"context_line":"            self._assert_stats("},{"line_number":763,"context_line":"                expected_in_progress_stats, sharder, \u0027sharding_in_progress\u0027)"},{"line_number":764,"context_line":""},{"line_number":765,"context_line":"            # when me move recon_sharded_timeout + 1 seconds into the future,"},{"line_number":766,"context_line":"            # bocker 1 will be removed."},{"line_number":767,"context_line":"            ts_now \u003d Timestamp(ts_now.timestamp +"},{"line_number":768,"context_line":"                               sharder.recon_sharded_timeout + 1)"}],"source_content_type":"text/x-python","patch_set":13,"id":"9e1e7b2a_947a77df","line":765,"range":{"start_line":765,"start_character":19,"end_line":765,"end_character":22},"updated":"2021-02-26 14:48:10.000000000","message":"typo","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":763,"context_line":"                expected_in_progress_stats, sharder, \u0027sharding_in_progress\u0027)"},{"line_number":764,"context_line":""},{"line_number":765,"context_line":"            # when me move recon_sharded_timeout + 1 seconds into the future,"},{"line_number":766,"context_line":"            # bocker 1 will be removed."},{"line_number":767,"context_line":"            ts_now \u003d Timestamp(ts_now.timestamp +"},{"line_number":768,"context_line":"                               sharder.recon_sharded_timeout + 1)"},{"line_number":769,"context_line":"            with mock.patch(\u0027eventlet.sleep\u0027), \\"}],"source_content_type":"text/x-python","patch_set":13,"id":"d0b32b6a_f5096bd4","line":766,"range":{"start_line":766,"start_character":14,"end_line":766,"end_character":20},"updated":"2021-02-26 14:48:10.000000000","message":"typo","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":782,"context_line":"                         \u0027state\u0027: \u0027sharding\u0027, \u0027db_state\u0027: \u0027unsharded\u0027,"},{"line_number":783,"context_line":"                         \u0027error\u0027: None}]}"},{"line_number":784,"context_line":"            self._assert_stats("},{"line_number":785,"context_line":"                expected_in_progress_stats, sharder, \u0027sharding_in_progress\u0027)"},{"line_number":786,"context_line":""},{"line_number":787,"context_line":"    def test_ratelimited_roundrobin(self):"},{"line_number":788,"context_line":"        n_databases \u003d 100"}],"source_content_type":"text/x-python","patch_set":13,"id":"99dfd74c_57e56632","line":785,"updated":"2021-02-26 14:48:10.000000000","message":"nice!","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":5489,"context_line":"        # Setup some cleaving contexts"},{"line_number":5490,"context_line":"        id_old, id_newish, id_complete \u003d [str(uuid4()) for _ in range(3)]"},{"line_number":5491,"context_line":"        ts_old, ts_newish, ts_complete \u003d (Timestamp(1),"},{"line_number":5492,"context_line":"                                          Timestamp(reclaim_age // 2),"},{"line_number":5493,"context_line":"                                          Timestamp(recon_sharded_timeout))"},{"line_number":5494,"context_line":"        contexts \u003d ((id_old, ts_old, False),"},{"line_number":5495,"context_line":"                    (id_newish, ts_newish, False),"},{"line_number":5496,"context_line":"                    (id_complete, ts_complete, True))"}],"source_content_type":"text/x-python","patch_set":13,"id":"9f5e0d30_19574dee","line":5493,"range":{"start_line":5492,"start_character":42,"end_line":5493,"end_character":74},"updated":"2021-02-26 14:48:10.000000000","message":"ok, so these two timestamps are actually the same because reclaim_age \u003d 100 and recon_sharded_timeout \u003d 50\n\nnit: the test might be easier to understand if ts_complete had been declared as (reclaim_age - recon_sharded_timeout); also, that way the test still passes if recon_sharded_timeout is made \u003e 50","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":5519,"context_line":"        # recon_sharded_timeout time since it was marked completed so it\u0027s"},{"line_number":5520,"context_line":"        # been removed"},{"line_number":5521,"context_line":"        complete_ctx \u003d get_context(id_complete, broker)"},{"line_number":5522,"context_line":"        self.assertEqual(complete_ctx, \"\")"},{"line_number":5523,"context_line":""},{"line_number":5524,"context_line":"        # If we push time another reclaim age later, and they all be removed"},{"line_number":5525,"context_line":"        # minus id_missing_lm as it has a later last_modified."}],"source_content_type":"text/x-python","patch_set":13,"id":"ee251bb4_99e375b1","line":5522,"updated":"2021-02-26 14:48:10.000000000","message":"ok, fails here if i revert the new condition from _audit_cleave_context()","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8378d0beaeb26e0184c0300241debe6ade50059","unresolved":true,"context_lines":[{"line_number":5521,"context_line":"        complete_ctx \u003d get_context(id_complete, broker)"},{"line_number":5522,"context_line":"        self.assertEqual(complete_ctx, \"\")"},{"line_number":5523,"context_line":""},{"line_number":5524,"context_line":"        # If we push time another reclaim age later, and they all be removed"},{"line_number":5525,"context_line":"        # minus id_missing_lm as it has a later last_modified."},{"line_number":5526,"context_line":"        with self._mock_sharder(sharder_conf) as sharder:"},{"line_number":5527,"context_line":"            with mock_timestamp_now(Timestamp(reclaim_age * 2)):"},{"line_number":5528,"context_line":"                sharder._audit_cleave_contexts(broker)"}],"source_content_type":"text/x-python","patch_set":13,"id":"a3ae4dcb_d97795ba","line":5525,"range":{"start_line":5524,"start_character":10,"end_line":5525,"end_character":62},"updated":"2021-02-26 14:48:10.000000000","message":"nit: this comment has been stale since https://github.com/openstack/swift/commit/370ac4cd70489a49b2b6408638c9b35006f57053","commit_id":"1d8b4101dd64d76352246a2a03909d8b4a586821"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"683c1ef2008729897b682cb5db936557412ea097","unresolved":true,"context_lines":[{"line_number":749,"context_line":"                         \u0027path\u0027: brokers[0].db_file, \u0027root\u0027: \u0027a/c0\u0027,"},{"line_number":750,"context_line":"                         \u0027node_index\u0027: 0,"},{"line_number":751,"context_line":"                         \u0027found\u0027: 0, \u0027created\u0027: 0, \u0027cleaved\u0027: 4, \u0027active\u0027: 1,"},{"line_number":752,"context_line":"                         \u0027state\u0027: \u0027sharded\u0027, \u0027db_state\u0027: \u0027sharded\u0027,"},{"line_number":753,"context_line":"                         \u0027error\u0027: None},"},{"line_number":754,"context_line":"                        {\u0027object_count\u0027: 0, \u0027account\u0027: \u0027a\u0027, \u0027container\u0027: \u0027c1\u0027,"},{"line_number":755,"context_line":"                         \u0027meta_timestamp\u0027: mock.ANY,"}],"source_content_type":"text/x-python","patch_set":14,"id":"be9f8c2d_69bb66ee","line":752,"updated":"2021-02-26 16:17:24.000000000","message":"there it is!","commit_id":"1de98348167f9923710d1a6d634ef6d8008eea5d"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"683c1ef2008729897b682cb5db936557412ea097","unresolved":true,"context_lines":[{"line_number":765,"context_line":"            # one more cycle at recon_sharded_timeout seconds into the"},{"line_number":766,"context_line":"            # future to check that the completed broker is still reported"},{"line_number":767,"context_line":"            ts_now \u003d Timestamp(ts_now.timestamp +"},{"line_number":768,"context_line":"                               sharder.recon_sharded_timeout)"},{"line_number":769,"context_line":"            with mock.patch(\u0027eventlet.sleep\u0027), \\"},{"line_number":770,"context_line":"                    mock.patch.object(sharder, \u0027_process_broker\u0027) \\"},{"line_number":771,"context_line":"                    as mock_process_broker, mock_timestamp_now(ts_now):"}],"source_content_type":"text/x-python","patch_set":14,"id":"c25eae90_2c8776e4","line":768,"updated":"2021-02-26 16:17:24.000000000","message":"right on the border!","commit_id":"1de98348167f9923710d1a6d634ef6d8008eea5d"}],"tools/playbooks/common/install_dependencies.yaml":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"683c1ef2008729897b682cb5db936557412ea097","unresolved":true,"context_lines":[{"line_number":40,"context_line":"        - nose"},{"line_number":41,"context_line":"        - pyeclib"},{"line_number":42,"context_line":"        - python-swiftclient"},{"line_number":43,"context_line":"        - mock"}],"source_content_type":"text/x-yaml","patch_set":14,"id":"2c8e4ea4_b95551ba","line":43,"updated":"2021-02-26 16:17:24.000000000","message":"is it significant that mock is now a dependency of probetests?","commit_id":"1de98348167f9923710d1a6d634ef6d8008eea5d"}]}