)]}'
{"swift/common/db.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e466a32be2abfcf4be62f7902a5e15117cafcf87","unresolved":true,"context_lines":[{"line_number":228,"context_line":"    return conn"},{"line_number":229,"context_line":""},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"class TombstoneReclaimer(object):"},{"line_number":232,"context_line":"    \"\"\"Encapsulates reclamation of deleted rows in a database.\"\"\""},{"line_number":233,"context_line":"    def __init__(self, broker, age_timestamp):"},{"line_number":234,"context_line":"        self.broker \u003d broker"}],"source_content_type":"text/x-python","patch_set":2,"id":"181affb4_53c35a58","line":231,"range":{"start_line":231,"start_character":6,"end_line":231,"end_character":24},"updated":"2021-03-26 12:45:13.000000000","message":"I found I was having to pass around dicts to keep a tally on markers, definite and possible tombstones, and I didn\u0027t want to make those (transient) statistics attributes of the Broker, so it felt like a class would be useful.","commit_id":"da21d2858e4bbc67bbb78e439a17d494e0cff592"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e466a32be2abfcf4be62f7902a5e15117cafcf87","unresolved":true,"context_lines":[{"line_number":277,"context_line":"                self._reclaim(conn)"},{"line_number":278,"context_line":"                conn.commit()"},{"line_number":279,"context_line":""},{"line_number":280,"context_line":"    def refine_tombstone_estimate(self, threshold):"},{"line_number":281,"context_line":"        \"\"\""},{"line_number":282,"context_line":"        Refine the estimate of tombstones in db by iteratively querying the"},{"line_number":283,"context_line":"        namespace after the marker to confirm possible_tombstones as"}],"source_content_type":"text/x-python","patch_set":2,"id":"42c0c0ee_50be67b9","line":280,"range":{"start_line":280,"start_character":8,"end_line":280,"end_character":33},"updated":"2021-03-26 12:45:13.000000000","message":"this could prove unnecessary: the \u0027error\u0027 in tombstone estimate, i.e. possible_tombstones, is \u003c\u003d RECLAIM_BATCH_SIZE. If RECLAIM_BATCH_SIZE is \u003c\u003c shrink_threshold then we may not need to refine the estimate.\n\ne.g. if shrink_threshold is 100K and batch size is 10K, then without any refinement, we might estimate that objects + tombstones \u003d 100001 and not shrink, when in fact objects + tombstones \u003d 90001 and we could have shrunk.\n\nBut, it was fun writing so I\u0027m not ready to delete it yet :) And of course, if batch size increases or shrink threshold decreases then it becomes more useful to refine the tombstone estimate.","commit_id":"da21d2858e4bbc67bbb78e439a17d494e0cff592"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e466a32be2abfcf4be62f7902a5e15117cafcf87","unresolved":true,"context_lines":[{"line_number":1077,"context_line":"        tombstone_reclaimer \u003d TombstoneReclaimer(self, age_timestamp)"},{"line_number":1078,"context_line":"        tombstone_reclaimer.reclaim()"},{"line_number":1079,"context_line":"        with self.get() as conn:"},{"line_number":1080,"context_line":"            self._reclaim_other_stuff(conn, age_timestamp, sync_timestamp)"},{"line_number":1081,"context_line":"            conn.commit()"},{"line_number":1082,"context_line":""},{"line_number":1083,"context_line":"    def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp):"}],"source_content_type":"text/x-python","patch_set":2,"id":"7a5e2352_2c56db2b","line":1080,"range":{"start_line":1080,"start_character":12,"end_line":1080,"end_character":38},"updated":"2021-03-26 12:45:13.000000000","message":"NB we used to do this with the same conn as the final _reclaim() call, but now we get another conn.","commit_id":"da21d2858e4bbc67bbb78e439a17d494e0cff592"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f98c938d584bd71c44eb61f49483f26dd848ae8e","unresolved":true,"context_lines":[{"line_number":236,"context_line":"        self.remaining_tombstones \u003d self.reclaimed \u003d 0"},{"line_number":237,"context_line":"        self.finished \u003d False"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"    def _select_tombstone_batch(self, conn, marker, batch_size):"},{"line_number":240,"context_line":"        curs \u003d conn.execute(\u0027\u0027\u0027"},{"line_number":241,"context_line":"            SELECT name FROM %s WHERE deleted \u003d 1"},{"line_number":242,"context_line":"            AND name \u003e\u003d ?"}],"source_content_type":"text/x-python","patch_set":6,"id":"ca07a419_0fb11a96","line":239,"updated":"2021-04-10 00:09:33.000000000","message":"This seems to be unused.","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24f6e208b76ee56120a6da56a9dbe51754a8ed29","unresolved":true,"context_lines":[{"line_number":236,"context_line":"        self.remaining_tombstones \u003d self.reclaimed \u003d 0"},{"line_number":237,"context_line":"        self.finished \u003d False"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"    def _select_tombstone_batch(self, conn, marker, batch_size):"},{"line_number":240,"context_line":"        curs \u003d conn.execute(\u0027\u0027\u0027"},{"line_number":241,"context_line":"            SELECT name FROM %s WHERE deleted \u003d 1"},{"line_number":242,"context_line":"            AND name \u003e\u003d ?"}],"source_content_type":"text/x-python","patch_set":6,"id":"26ab2d44_96ad4247","line":239,"in_reply_to":"ca07a419_0fb11a96","updated":"2021-04-12 12:27:25.000000000","message":"yep, leftover from previous version, will fix","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f98c938d584bd71c44eb61f49483f26dd848ae8e","unresolved":true,"context_lines":[{"line_number":291,"context_line":"                SELECT COUNT(*) FROM %s WHERE deleted \u003d 1"},{"line_number":292,"context_line":"                AND name \u003e\u003d ?"},{"line_number":293,"context_line":"            \u0027\u0027\u0027 % (self.broker.db_contains_type,), (self.marker,))"},{"line_number":294,"context_line":"        tombstones \u003d curs.fetchone()[0]"},{"line_number":295,"context_line":"        self.remaining_tombstones +\u003d tombstones"},{"line_number":296,"context_line":"        return self.remaining_tombstones"},{"line_number":297,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"c1b70017_d85d5c75","line":294,"updated":"2021-04-10 00:09:33.000000000","message":"I wonder if we should just go ahead and do this up in _reclaim -- should be fairly cheap, right?","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24f6e208b76ee56120a6da56a9dbe51754a8ed29","unresolved":true,"context_lines":[{"line_number":291,"context_line":"                SELECT COUNT(*) FROM %s WHERE deleted \u003d 1"},{"line_number":292,"context_line":"                AND name \u003e\u003d ?"},{"line_number":293,"context_line":"            \u0027\u0027\u0027 % (self.broker.db_contains_type,), (self.marker,))"},{"line_number":294,"context_line":"        tombstones \u003d curs.fetchone()[0]"},{"line_number":295,"context_line":"        self.remaining_tombstones +\u003d tombstones"},{"line_number":296,"context_line":"        return self.remaining_tombstones"},{"line_number":297,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"512a071b_ee0b547f","line":294,"in_reply_to":"c1b70017_d85d5c75","updated":"2021-04-12 12:27:25.000000000","message":"I was planning to run some more performance measurements, but yes that\u0027s certainly an option","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5de6ab585c2b7b215fd3a399569d3e2fe055f97c","unresolved":true,"context_lines":[{"line_number":228,"context_line":""},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"class TombstoneReclaimer(object):"},{"line_number":231,"context_line":"    \"\"\"Encapsulates reclamation of deleted rows in a database.\"\"\""},{"line_number":232,"context_line":"    def __init__(self, broker, age_timestamp):"},{"line_number":233,"context_line":"        self.broker \u003d broker"},{"line_number":234,"context_line":"        self.age_timestamp \u003d age_timestamp"}],"source_content_type":"text/x-python","patch_set":7,"id":"08702159_01d038de","line":231,"updated":"2021-04-15 21:23:11.000000000","message":"DatabaseBroker reclaim is enhanced to count remaining tombstones\nafter rows have been reclaimed. A new TombstoneReclaimer class is\nadded to encapsulate the reclaim process and tombstone count.","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5de6ab585c2b7b215fd3a399569d3e2fe055f97c","unresolved":true,"context_lines":[{"line_number":240,"context_line":"        clean_batch_qry \u003d \u0027\u0027\u0027"},{"line_number":241,"context_line":"            DELETE FROM %s WHERE deleted \u003d 1"},{"line_number":242,"context_line":"            AND name \u003e\u003d ? AND %s \u003c ?"},{"line_number":243,"context_line":"        \u0027\u0027\u0027 % (self.broker.db_contains_type, self.broker.db_reclaim_timestamp)"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"        curs \u003d conn.execute(\u0027\u0027\u0027"},{"line_number":246,"context_line":"            SELECT name FROM %s WHERE deleted \u003d 1"}],"source_content_type":"text/x-python","patch_set":7,"id":"300f5129_0cf31340","line":243,"updated":"2021-04-15 21:23:11.000000000","message":"maybe better to define this once in __init__ than every time through the loop","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"40b7380d848463877bcc8bb2892af6c9def3a276","unresolved":false,"context_lines":[{"line_number":240,"context_line":"        clean_batch_qry \u003d \u0027\u0027\u0027"},{"line_number":241,"context_line":"            DELETE FROM %s WHERE deleted \u003d 1"},{"line_number":242,"context_line":"            AND name \u003e\u003d ? AND %s \u003c ?"},{"line_number":243,"context_line":"        \u0027\u0027\u0027 % (self.broker.db_contains_type, self.broker.db_reclaim_timestamp)"},{"line_number":244,"context_line":""},{"line_number":245,"context_line":"        curs \u003d conn.execute(\u0027\u0027\u0027"},{"line_number":246,"context_line":"            SELECT name FROM %s WHERE deleted \u003d 1"}],"source_content_type":"text/x-python","patch_set":7,"id":"588f7d2b_2d95a53d","line":243,"in_reply_to":"300f5129_0cf31340","updated":"2021-04-19 10:25:51.000000000","message":"Done","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5de6ab585c2b7b215fd3a399569d3e2fe055f97c","unresolved":true,"context_lines":[{"line_number":246,"context_line":"            SELECT name FROM %s WHERE deleted \u003d 1"},{"line_number":247,"context_line":"            AND name \u003e\u003d ?"},{"line_number":248,"context_line":"            ORDER BY NAME LIMIT 1 OFFSET ?"},{"line_number":249,"context_line":"        \u0027\u0027\u0027 % (self.broker.db_contains_type,), (self.marker, RECLAIM_PAGE_SIZE)"},{"line_number":250,"context_line":"        )"},{"line_number":251,"context_line":"        row \u003d curs.fetchone()"},{"line_number":252,"context_line":"        end_marker \u003d row[0] if row else \u0027\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"2f36b2ce_a7dc7056","line":249,"updated":"2021-04-15 21:23:11.000000000","message":"i guess this could be a template too - whatever looks cleanest I guess...\n\na reader has to grok bothe and see there\u0027s TWO conn.execute each time this method is executed","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"40b7380d848463877bcc8bb2892af6c9def3a276","unresolved":false,"context_lines":[{"line_number":246,"context_line":"            SELECT name FROM %s WHERE deleted \u003d 1"},{"line_number":247,"context_line":"            AND name \u003e\u003d ?"},{"line_number":248,"context_line":"            ORDER BY NAME LIMIT 1 OFFSET ?"},{"line_number":249,"context_line":"        \u0027\u0027\u0027 % (self.broker.db_contains_type,), (self.marker, RECLAIM_PAGE_SIZE)"},{"line_number":250,"context_line":"        )"},{"line_number":251,"context_line":"        row \u003d curs.fetchone()"},{"line_number":252,"context_line":"        end_marker \u003d row[0] if row else \u0027\u0027"}],"source_content_type":"text/x-python","patch_set":7,"id":"e1329900_d9ce41de","line":249,"in_reply_to":"2f36b2ce_a7dc7056","updated":"2021-04-19 10:25:51.000000000","message":"Done","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5de6ab585c2b7b215fd3a399569d3e2fe055f97c","unresolved":true,"context_lines":[{"line_number":249,"context_line":"        \u0027\u0027\u0027 % (self.broker.db_contains_type,), (self.marker, RECLAIM_PAGE_SIZE)"},{"line_number":250,"context_line":"        )"},{"line_number":251,"context_line":"        row \u003d curs.fetchone()"},{"line_number":252,"context_line":"        end_marker \u003d row[0] if row else \u0027\u0027"},{"line_number":253,"context_line":"        if end_marker:"},{"line_number":254,"context_line":"            # do a single book-ended DELETE and bounce out"},{"line_number":255,"context_line":"            curs \u003d conn.execute(clean_batch_qry + \u0027 AND name \u003c ?\u0027, ("}],"source_content_type":"text/x-python","patch_set":7,"id":"c851fd4b_9b283523","line":252,"updated":"2021-04-15 21:23:11.000000000","message":"the behavior of offset here to \"push the query off the end\" as it were is kind of surprising - I wonder if we should try and put a comment around that query","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"40b7380d848463877bcc8bb2892af6c9def3a276","unresolved":false,"context_lines":[{"line_number":249,"context_line":"        \u0027\u0027\u0027 % (self.broker.db_contains_type,), (self.marker, RECLAIM_PAGE_SIZE)"},{"line_number":250,"context_line":"        )"},{"line_number":251,"context_line":"        row \u003d curs.fetchone()"},{"line_number":252,"context_line":"        end_marker \u003d row[0] if row else \u0027\u0027"},{"line_number":253,"context_line":"        if end_marker:"},{"line_number":254,"context_line":"            # do a single book-ended DELETE and bounce out"},{"line_number":255,"context_line":"            curs \u003d conn.execute(clean_batch_qry + \u0027 AND name \u003c ?\u0027, ("}],"source_content_type":"text/x-python","patch_set":7,"id":"9acb39d3_9bb545e7","line":252,"in_reply_to":"c851fd4b_9b283523","updated":"2021-04-19 10:25:51.000000000","message":"Done","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5de6ab585c2b7b215fd3a399569d3e2fe055f97c","unresolved":true,"context_lines":[{"line_number":256,"context_line":"                self.marker, self.age_timestamp, end_marker))"},{"line_number":257,"context_line":"            self.marker \u003d end_marker"},{"line_number":258,"context_line":"            self.reclaimed +\u003d curs.rowcount"},{"line_number":259,"context_line":"            self.remaining_tombstones +\u003d RECLAIM_PAGE_SIZE - curs.rowcount"},{"line_number":260,"context_line":"        else:"},{"line_number":261,"context_line":"            # delete off the end"},{"line_number":262,"context_line":"            curs \u003d conn.execute(clean_batch_qry,"}],"source_content_type":"text/x-python","patch_set":7,"id":"884840dd_e3c79c92","line":259,"updated":"2021-04-15 21:23:11.000000000","message":"number_of_tombstones_we_leave_behind \u003d maximum_number_of_tombstones_selected_in_the_batch - the_number_that_our_query_actually_deleted","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5de6ab585c2b7b215fd3a399569d3e2fe055f97c","unresolved":true,"context_lines":[{"line_number":273,"context_line":"                self._reclaim(conn)"},{"line_number":274,"context_line":"                conn.commit()"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"    def get_tombstone_count(self):"},{"line_number":277,"context_line":"        if not self.finished:"},{"line_number":278,"context_line":"            self.reclaim()"},{"line_number":279,"context_line":"        with self.broker.get() as conn:"}],"source_content_type":"text/x-python","patch_set":7,"id":"29d73ffb_d7219857","line":276,"updated":"2021-04-15 21:23:11.000000000","message":"as 1/2 of the public interface of this class; I bet we\u0027d appreciate a docstring.","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"40b7380d848463877bcc8bb2892af6c9def3a276","unresolved":false,"context_lines":[{"line_number":273,"context_line":"                self._reclaim(conn)"},{"line_number":274,"context_line":"                conn.commit()"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"    def get_tombstone_count(self):"},{"line_number":277,"context_line":"        if not self.finished:"},{"line_number":278,"context_line":"            self.reclaim()"},{"line_number":279,"context_line":"        with self.broker.get() as conn:"}],"source_content_type":"text/x-python","patch_set":7,"id":"863c3b7e_19ddeaff","line":276,"in_reply_to":"29d73ffb_d7219857","updated":"2021-04-19 10:25:51.000000000","message":"Done","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0058274a29de2e119f9af410c3171e72e89a9200","unresolved":true,"context_lines":[{"line_number":280,"context_line":"            curs \u003d conn.execute(\u0027\u0027\u0027"},{"line_number":281,"context_line":"                SELECT COUNT(*) FROM %s WHERE deleted \u003d 1"},{"line_number":282,"context_line":"                AND name \u003e\u003d ?"},{"line_number":283,"context_line":"            \u0027\u0027\u0027 % (self.broker.db_contains_type,), (self.marker,))"},{"line_number":284,"context_line":"        tombstones \u003d curs.fetchone()[0]"},{"line_number":285,"context_line":"        self.remaining_tombstones +\u003d tombstones"},{"line_number":286,"context_line":"        return self.remaining_tombstones"}],"source_content_type":"text/x-python","patch_set":7,"id":"41f99f5e_5d0b13fc","line":283,"updated":"2021-04-19 21:38:48.000000000","message":"What do you think about moving this into _reclaim? It\u0027d be a little more expensive in the replicator (where we don\u0027t care about the tombstone count), but I bet sqlite could answer that from what it\u0027s still got in memory after that final delete off the end, so IDK that it\u0027s *that* expensive.","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5de6ab585c2b7b215fd3a399569d3e2fe055f97c","unresolved":true,"context_lines":[{"line_number":1052,"context_line":"        tombstone_reclaimer.reclaim()"},{"line_number":1053,"context_line":"        with self.get() as conn:"},{"line_number":1054,"context_line":"            self._reclaim_other_stuff(conn, age_timestamp, sync_timestamp)"},{"line_number":1055,"context_line":"            conn.commit()"},{"line_number":1056,"context_line":"        return tombstone_reclaimer"},{"line_number":1057,"context_line":""},{"line_number":1058,"context_line":"    def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp):"}],"source_content_type":"text/x-python","patch_set":7,"id":"8869dfd1_55b695a2","line":1055,"updated":"2021-04-15 21:23:11.000000000","message":"this little bit here surrounded by the reclaimer looked off at first...\n\ni mean the TombstoneReclaimer *has* self/broker - it must be opening and closing connectoins in .reclaim(), so obviously he could do all this at the end of reclaim using the same connection as the last batch... \n\nbut, I think TombstoneReclaimer ends up looking more focused ... at the cost of an extra get() as conn","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"40b7380d848463877bcc8bb2892af6c9def3a276","unresolved":true,"context_lines":[{"line_number":1052,"context_line":"        tombstone_reclaimer.reclaim()"},{"line_number":1053,"context_line":"        with self.get() as conn:"},{"line_number":1054,"context_line":"            self._reclaim_other_stuff(conn, age_timestamp, sync_timestamp)"},{"line_number":1055,"context_line":"            conn.commit()"},{"line_number":1056,"context_line":"        return tombstone_reclaimer"},{"line_number":1057,"context_line":""},{"line_number":1058,"context_line":"    def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp):"}],"source_content_type":"text/x-python","patch_set":7,"id":"ffa5a1a3_3bd9a9a6","line":1055,"in_reply_to":"8869dfd1_55b695a2","updated":"2021-04-19 10:25:51.000000000","message":"I agree, there\u0027s potential to encapsulate more in TombstoneReclaimer. I kept it focussed because when used in the sharder I only care about tombstones getting reclaimed and counted (not other stuff). But perhaps there\u0027s no harm reclaiming everything in the sharder call too? The trade off is \u0027only do what I need to do\u0027 vs \u0027more encapsulation\u0027.\n\nI guess TombstoneReclaimer could have:\n\n  reclaim_all (which includes a call to reclaim_tombstones)\n  reclaim_tombstones\n\n(and the class would be renamed simply Reclaimer.) But then we\u0027d be passing sync_timestamp into the class only for it to pass it back to broker.reclaim_other_stuff.\n\nI\u0027ll leave it as it is, but I\u0027m open to further discussion.","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"0058274a29de2e119f9af410c3171e72e89a9200","unresolved":true,"context_lines":[{"line_number":1052,"context_line":"        tombstone_reclaimer.reclaim()"},{"line_number":1053,"context_line":"        with self.get() as conn:"},{"line_number":1054,"context_line":"            self._reclaim_other_stuff(conn, age_timestamp, sync_timestamp)"},{"line_number":1055,"context_line":"            conn.commit()"},{"line_number":1056,"context_line":"        return tombstone_reclaimer"},{"line_number":1057,"context_line":""},{"line_number":1058,"context_line":"    def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp):"}],"source_content_type":"text/x-python","patch_set":7,"id":"1d590d4d_d0a23a0a","line":1055,"in_reply_to":"ffa5a1a3_3bd9a9a6","updated":"2021-04-19 21:38:48.000000000","message":"I think I prefer the encapsulation -- the alternative seems (to me) to be accepting a callback, like\n\n tombstone_reclaimer.reclaim(lambda conn: self._reclaim_other_stuff(\n     conn, age_timestamp, sync_timestamp))\n\nand that feels kinda gross.","commit_id":"fd2e7aa8db5f9f81e90481d87231283451b20704"}],"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f98c938d584bd71c44eb61f49483f26dd848ae8e","unresolved":true,"context_lines":[{"line_number":5288,"context_line":"    @property"},{"line_number":5289,"context_line":"    def row_count(self):"},{"line_number":5290,"context_line":"        \"\"\""},{"line_number":5291,"context_line":"        Returns the total number if rows in the shard range i.e. the sum of"},{"line_number":5292,"context_line":"        objects and tombstones."},{"line_number":5293,"context_line":""},{"line_number":5294,"context_line":"        :return: the row count"}],"source_content_type":"text/x-python","patch_set":6,"id":"a4a9aff2_78fa67b6","line":5291,"range":{"start_line":5291,"start_character":33,"end_line":5291,"end_character":35},"updated":"2021-04-10 00:09:33.000000000","message":"of","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24f6e208b76ee56120a6da56a9dbe51754a8ed29","unresolved":false,"context_lines":[{"line_number":5288,"context_line":"    @property"},{"line_number":5289,"context_line":"    def row_count(self):"},{"line_number":5290,"context_line":"        \"\"\""},{"line_number":5291,"context_line":"        Returns the total number if rows in the shard range i.e. the sum of"},{"line_number":5292,"context_line":"        objects and tombstones."},{"line_number":5293,"context_line":""},{"line_number":5294,"context_line":"        :return: the row count"}],"source_content_type":"text/x-python","patch_set":6,"id":"d3f852c9_bb2d8806","line":5291,"range":{"start_line":5291,"start_character":33,"end_line":5291,"end_character":35},"in_reply_to":"a4a9aff2_78fa67b6","updated":"2021-04-12 12:27:25.000000000","message":"Done","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f98c938d584bd71c44eb61f49483f26dd848ae8e","unresolved":true,"context_lines":[{"line_number":5295,"context_line":"        \"\"\""},{"line_number":5296,"context_line":"        if self.tombstones \u003e\u003d 0:"},{"line_number":5297,"context_line":"            return self.object_count + self.tombstones"},{"line_number":5298,"context_line":"        return self.object_count"},{"line_number":5299,"context_line":""},{"line_number":5300,"context_line":"    def update_meta(self, object_count, bytes_used, meta_timestamp\u003dNone):"},{"line_number":5301,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":6,"id":"58947fa2_76bc5880","line":5298,"updated":"2021-04-10 00:09:33.000000000","message":"Maybe\n\n return self.object_count + max(self.tombstones, 0)\n\n? *shrug*","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24f6e208b76ee56120a6da56a9dbe51754a8ed29","unresolved":false,"context_lines":[{"line_number":5295,"context_line":"        \"\"\""},{"line_number":5296,"context_line":"        if self.tombstones \u003e\u003d 0:"},{"line_number":5297,"context_line":"            return self.object_count + self.tombstones"},{"line_number":5298,"context_line":"        return self.object_count"},{"line_number":5299,"context_line":""},{"line_number":5300,"context_line":"    def update_meta(self, object_count, bytes_used, meta_timestamp\u003dNone):"},{"line_number":5301,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":6,"id":"1741ff7b_ff8302a2","line":5298,"in_reply_to":"58947fa2_76bc5880","updated":"2021-04-12 12:27:25.000000000","message":"Done","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f98c938d584bd71c44eb61f49483f26dd848ae8e","unresolved":true,"context_lines":[{"line_number":5344,"context_line":"        else:"},{"line_number":5345,"context_line":"            self.meta_timestamp \u003d meta_timestamp"},{"line_number":5346,"context_line":""},{"line_number":5347,"context_line":"    def increment_meta(self, object_count, bytes_used):"},{"line_number":5348,"context_line":"        \"\"\""},{"line_number":5349,"context_line":"        Increment the object stats metadata by the given values and update the"},{"line_number":5350,"context_line":"        meta_timestamp to the current time."}],"source_content_type":"text/x-python","patch_set":6,"id":"5e56ba2b_a34e3392","line":5347,"updated":"2021-04-10 00:09:33.000000000","message":"Should this pick up a tombstones arg?","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"24f6e208b76ee56120a6da56a9dbe51754a8ed29","unresolved":true,"context_lines":[{"line_number":5344,"context_line":"        else:"},{"line_number":5345,"context_line":"            self.meta_timestamp \u003d meta_timestamp"},{"line_number":5346,"context_line":""},{"line_number":5347,"context_line":"    def increment_meta(self, object_count, bytes_used):"},{"line_number":5348,"context_line":"        \"\"\""},{"line_number":5349,"context_line":"        Increment the object stats metadata by the given values and update the"},{"line_number":5350,"context_line":"        meta_timestamp to the current time."}],"source_content_type":"text/x-python","patch_set":6,"id":"3d7bf989_78358964","line":5347,"in_reply_to":"5e56ba2b_a34e3392","updated":"2021-04-12 12:27:25.000000000","message":"I found it wasn\u0027t needed: meta is updated as a side-effect of getting \na broker\u0027s own_shard_range, whereas tombstones is updated by the sharder after the reclaim","commit_id":"548dd9f266a49f377808328d03823159a267818b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"268acd0a2a874b711bd341922fcb8a4fea88eb2d","unresolved":true,"context_lines":[{"line_number":5340,"context_line":"        if meta_timestamp is None:"},{"line_number":5341,"context_line":"            self.meta_timestamp \u003d Timestamp.now()"},{"line_number":5342,"context_line":"        else:"},{"line_number":5343,"context_line":"            self.meta_timestamp \u003d meta_timestamp"},{"line_number":5344,"context_line":""},{"line_number":5345,"context_line":"    def increment_meta(self, object_count, bytes_used):"},{"line_number":5346,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"a90c7372_ec171d70","line":5343,"updated":"2021-05-07 00:30:06.000000000","message":"So meta_timestamp becomes the last time we *either* called update_meta() *or* update_tombstones() -- weirds me out a little that we don\u0027t know which...","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c6c9807f0d3221fa712bc66e6fe6a3918390d440","unresolved":true,"context_lines":[{"line_number":5340,"context_line":"        if meta_timestamp is None:"},{"line_number":5341,"context_line":"            self.meta_timestamp \u003d Timestamp.now()"},{"line_number":5342,"context_line":"        else:"},{"line_number":5343,"context_line":"            self.meta_timestamp \u003d meta_timestamp"},{"line_number":5344,"context_line":""},{"line_number":5345,"context_line":"    def increment_meta(self, object_count, bytes_used):"},{"line_number":5346,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"84b9c682_66d97757","line":5343,"in_reply_to":"a90c7372_ec171d70","updated":"2021-05-07 18:56:02.000000000","message":"Al pointed out that the only time we call update_tombstones should be shortly after calling update_meta, so the difference in practice should be pretty negligible.","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"268acd0a2a874b711bd341922fcb8a4fea88eb2d","unresolved":true,"context_lines":[{"line_number":5593,"context_line":"            params[\u0027upper\u0027], params[\u0027object_count\u0027], params[\u0027bytes_used\u0027],"},{"line_number":5594,"context_line":"            params[\u0027meta_timestamp\u0027], params[\u0027deleted\u0027], params[\u0027state\u0027],"},{"line_number":5595,"context_line":"            params[\u0027state_timestamp\u0027], params[\u0027epoch\u0027],"},{"line_number":5596,"context_line":"            params.get(\u0027reported\u0027, 0), params.get(\u0027tombstones\u0027, -1))"},{"line_number":5597,"context_line":""},{"line_number":5598,"context_line":"    def expand(self, donors):"},{"line_number":5599,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":9,"id":"9ba907d1_6b4df2fd","line":5596,"updated":"2021-05-07 00:30:06.000000000","message":"I\u0027m so glad we didn\u0027t try to do something like\n\n params.setdefault(\u0027reported\u0027, 0)\n return cls(**params)\n\nbefore!","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"}],"swift/container/backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"268acd0a2a874b711bd341922fcb8a4fea88eb2d","unresolved":true,"context_lines":[{"line_number":1454,"context_line":"            conn.commit()"},{"line_number":1455,"context_line":""},{"line_number":1456,"context_line":"        with self.get() as conn:"},{"line_number":1457,"context_line":"            attempts \u003d 4"},{"line_number":1458,"context_line":"            while attempts:"},{"line_number":1459,"context_line":"                attempts -\u003d 1"},{"line_number":1460,"context_line":"                try:"}],"source_content_type":"text/x-python","patch_set":9,"id":"f8fdbccd_6e1ef16d","line":1457,"range":{"start_line":1457,"start_character":23,"end_line":1457,"end_character":24},"updated":"2021-05-07 00:30:06.000000000","message":"I\u0027m not feeling the magic number here... http://paste.openstack.org/show/805015/ might be better?","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9cd0fe0ba0fcd0d9cf0f78f135def4ddf7a7debf","unresolved":true,"context_lines":[{"line_number":1454,"context_line":"            conn.commit()"},{"line_number":1455,"context_line":""},{"line_number":1456,"context_line":"        with self.get() as conn:"},{"line_number":1457,"context_line":"            attempts \u003d 4"},{"line_number":1458,"context_line":"            while attempts:"},{"line_number":1459,"context_line":"                attempts -\u003d 1"},{"line_number":1460,"context_line":"                try:"}],"source_content_type":"text/x-python","patch_set":9,"id":"cffafe16_794c083b","line":1457,"range":{"start_line":1457,"start_character":23,"end_line":1457,"end_character":24},"in_reply_to":"f8fdbccd_6e1ef16d","updated":"2021-05-07 14:53:46.000000000","message":"ok, but you need to break out of the for loop ;)","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"268acd0a2a874b711bd341922fcb8a4fea88eb2d","unresolved":true,"context_lines":[{"line_number":1732,"context_line":""},{"line_number":1733,"context_line":"        with self.maybe_get(connection) as conn:"},{"line_number":1734,"context_line":"            defaults \u003d set()"},{"line_number":1735,"context_line":"            attempts \u003d 3"},{"line_number":1736,"context_line":"            while attempts:"},{"line_number":1737,"context_line":"                attempts -\u003d 1"},{"line_number":1738,"context_line":"                try:"}],"source_content_type":"text/x-python","patch_set":9,"id":"3a43c2e1_ee1c8c61","line":1735,"range":{"start_line":1735,"start_character":23,"end_line":1735,"end_character":24},"updated":"2021-05-07 00:30:06.000000000","message":"Better as\n\n len(default_values) + 1\n\n?","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9cd0fe0ba0fcd0d9cf0f78f135def4ddf7a7debf","unresolved":false,"context_lines":[{"line_number":1732,"context_line":""},{"line_number":1733,"context_line":"        with self.maybe_get(connection) as conn:"},{"line_number":1734,"context_line":"            defaults \u003d set()"},{"line_number":1735,"context_line":"            attempts \u003d 3"},{"line_number":1736,"context_line":"            while attempts:"},{"line_number":1737,"context_line":"                attempts -\u003d 1"},{"line_number":1738,"context_line":"                try:"}],"source_content_type":"text/x-python","patch_set":9,"id":"f589b9b9_34313b9b","line":1735,"range":{"start_line":1735,"start_character":23,"end_line":1735,"end_character":24},"in_reply_to":"3a43c2e1_ee1c8c61","updated":"2021-05-07 14:53:46.000000000","message":"Done","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b1e0a5557f1d36a6541f0217e521fd67d36b89f5","unresolved":true,"context_lines":[{"line_number":1753,"context_line":""},{"line_number":1754,"context_line":"            self.assert_container_listing([alpha])"},{"line_number":1755,"context_line":""},{"line_number":1756,"context_line":"            # second shard updates root stats; reclaim tombstones so that the"},{"line_number":1757,"context_line":"            # shard is shrinkable"},{"line_number":1758,"context_line":"            shard_1_part \u003d self.get_part_and_node_numbers("},{"line_number":1759,"context_line":"                orig_shard_ranges[1])[0]"},{"line_number":1760,"context_line":"            for conf_index in self.configs[\u0027container-sharder\u0027].keys():"},{"line_number":1761,"context_line":"                self.run_custom_sharder(conf_index, {\u0027reclaim_age\u0027: 0},"},{"line_number":1762,"context_line":"                                        override_partitions\u003d[shard_1_part])"},{"line_number":1763,"context_line":"            self.assert_container_listing([alpha])"},{"line_number":1764,"context_line":"            # runs sharders so second range shrinks away, requires up to 2"},{"line_number":1765,"context_line":"            # cycles"},{"line_number":1766,"context_line":"            self.sharders.once()  # root finds shrinkable shard"}],"source_content_type":"text/x-python","patch_set":9,"id":"414216c5_2cecc9ef","line":1763,"range":{"start_line":1756,"start_character":0,"end_line":1763,"end_character":3},"updated":"2021-05-06 16:52:30.000000000","message":"in at least one of these cases let\u0027s run the vanilla sharders first and check things did not get shrunk, then run the custom sharder","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c6c9807f0d3221fa712bc66e6fe6a3918390d440","unresolved":true,"context_lines":[{"line_number":1765,"context_line":"            # the acceptor shard is intact..."},{"line_number":1766,"context_line":"            shard_nodes_data \u003d self.direct_get_container_shard_ranges("},{"line_number":1767,"context_line":"                orig_shard_ranges[1].account, orig_shard_ranges[1].container)"},{"line_number":1768,"context_line":"            obj_count, bytes_used \u003d check_shard_nodes_data(shard_nodes_data)"},{"line_number":1769,"context_line":"            self.assertEqual(1, obj_count)"},{"line_number":1770,"context_line":""},{"line_number":1771,"context_line":"            # run sharders to reclaim tombstones so that the second shard is"}],"source_content_type":"text/x-python","patch_set":11,"id":"65baeb73_54f6cfca","line":1768,"updated":"2021-05-07 18:56:02.000000000","message":"Cool -- backing out the changes under swift/, this pops like\n\n FAIL: test_shrinking (test.probe.test_sharder.TestContainerSharding)\n ----------------------------------------------------------------------\n Traceback (most recent call last):\n   File \".../probe/test_sharder.py\", line 1829, in test_shrinking\n     do_shard_then_shrink()\n   File \".../probe/test_sharder.py\", line 1768, in do_shard_then_shrink\n     obj_count, bytes_used \u003d check_shard_nodes_data(shard_nodes_data)\n   File \".../probe/test_sharder.py\", line 1521, in check_shard_nodes_data\n     check_node_data(\n   File \".../probe/test_sharder.py\", line 1489, in check_node_data\n     self.assert_dict_contains(exp_hdrs, hdrs)\n   File \".../probe/test_sharder.py\", line 263, in assert_dict_contains\n     self.assertEqual(expected_items, filtered_actual)\n AssertionError: Node id 3. Failed with {\u0027X-Container-Sysmeta-Shard-Quoted-Root\u0027: \u0027AU[92 chars]ded\u0027} !\u003d {\u0027X-Backend-Sharding-State\u0027: \u0027sharded\u0027, \u0027X-Co[90 chars]49b\u0027}\n - {\u0027X-Backend-Sharding-State\u0027: \u0027unsharded\u0027,\n ?                               --\n \n + {\u0027X-Backend-Sharding-State\u0027: \u0027sharded\u0027,\n    \u0027X-Container-Sysmeta-Shard-Quoted-Root\u0027: \u0027AUTH_test/container-c44a17f7-8aa8-4e3b-8063-f181a4c7a49b\u0027}","commit_id":"bcecddd517229c9671a2494c9efed257c0833525"}],"test/unit/common/test_db.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"268acd0a2a874b711bd341922fcb8a4fea88eb2d","unresolved":true,"context_lines":[{"line_number":1631,"context_line":"        #  a-* has 70 \u0027active\u0027 tombstones followed by 70 reclaimable"},{"line_number":1632,"context_line":"        #  b-* has 70 \u0027active\u0027 tombstones followed by 70 reclaimable"},{"line_number":1633,"context_line":"        for i in range(0, 560, 4):"},{"line_number":1634,"context_line":"            self._make_object(broker, \u0027a_%3d\u0027 % i,"},{"line_number":1635,"context_line":"                              Timestamp(top_of_the_minute - (i * 60)),"},{"line_number":1636,"context_line":"                              True)"},{"line_number":1637,"context_line":"            self._make_object(broker, \u0027a_%3d\u0027 % (i + 1),"}],"source_content_type":"text/x-python","patch_set":9,"id":"6b989cb7_ca992577","line":1634,"range":{"start_line":1634,"start_character":48,"end_line":1634,"end_character":49},"updated":"2021-05-07 00:30:06.000000000","message":"So this is the main difference from _setup_reclaimable_active? The object names?\n\nWDYT about http://paste.openstack.org/show/805013/ ?","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9cd0fe0ba0fcd0d9cf0f78f135def4ddf7a7debf","unresolved":true,"context_lines":[{"line_number":1631,"context_line":"        #  a-* has 70 \u0027active\u0027 tombstones followed by 70 reclaimable"},{"line_number":1632,"context_line":"        #  b-* has 70 \u0027active\u0027 tombstones followed by 70 reclaimable"},{"line_number":1633,"context_line":"        for i in range(0, 560, 4):"},{"line_number":1634,"context_line":"            self._make_object(broker, \u0027a_%3d\u0027 % i,"},{"line_number":1635,"context_line":"                              Timestamp(top_of_the_minute - (i * 60)),"},{"line_number":1636,"context_line":"                              True)"},{"line_number":1637,"context_line":"            self._make_object(broker, \u0027a_%3d\u0027 % (i + 1),"}],"source_content_type":"text/x-python","patch_set":9,"id":"a38cb6f0_3fd4130b","line":1634,"range":{"start_line":1634,"start_character":48,"end_line":1634,"end_character":49},"in_reply_to":"6b989cb7_ca992577","updated":"2021-05-07 14:53:46.000000000","message":"that\u0027s much nicer, thanks","commit_id":"1e28dfcfc9aa85e2e62a8eed5cf68b65d69ab215"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c6c9807f0d3221fa712bc66e6fe6a3918390d440","unresolved":true,"context_lines":[{"line_number":1594,"context_line":"        now \u003d time.time()"},{"line_number":1595,"context_line":"        top_of_the_minute \u003d now - (now % 60)"},{"line_number":1596,"context_line":""},{"line_number":1597,"context_line":"        # namespace if reverse:"},{"line_number":1598,"context_line":"        #  a-* has 70 \u0027active\u0027 tombstones followed by 70 reclaimable"},{"line_number":1599,"context_line":"        #  b-* has 70 \u0027active\u0027 tombstones followed by 70 reclaimable"},{"line_number":1600,"context_line":"        # else:"}],"source_content_type":"text/x-python","patch_set":11,"id":"127e10d3_3330e959","line":1597,"range":{"start_line":1597,"start_character":20,"end_line":1597,"end_character":30},"updated":"2021-05-07 18:56:02.000000000","message":"Isn\u0027t the `if not reverse_names` case?","commit_id":"bcecddd517229c9671a2494c9efed257c0833525"}]}