)]}'
{"doc/source/overview_expiring_objects.rst":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1349a1123c4d6d768635e8da6c51bc972a075477","unresolved":false,"context_lines":[{"line_number":46,"context_line":"    use \u003d egg:swift#catch_errors"},{"line_number":47,"context_line":"    # See proxy-server.conf-sample for options"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"The daemon needs to run on a machine with access to all the backend servers in the cluster, but does not need proxy server or public access. The daemon will use its own internal proxy code instance to access the backend servers."}],"source_content_type":"text/x-rst","patch_set":5,"id":"fa69d971_30e313db","side":"PARENT","line":49,"updated":"2016-01-11 19:44:55.000000000","message":"Rather than remove this documentation you should add an explanation of the current implementation and explain how to upgrade.","commit_id":"02ecd51b906c8734585cf5d101e99a005e8134f3"}],"swift/container/backend.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":216,"context_line":"            BEGIN"},{"line_number":217,"context_line":"                DELETE FROM expired WHERE obj_row_id \u003d old.rowid;"},{"line_number":218,"context_line":"            END;"},{"line_number":219,"context_line":"        \"\"\")"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":"    def create_container_info_table(self, conn, put_timestamp,"},{"line_number":222,"context_line":"                                    storage_policy_index):"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_f79f145d","line":219,"updated":"2015-12-01 22:49:53.000000000","message":"Where is the migration to create this table for existing containers?","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":305,"context_line":"        (name, timestamp, size, content_type,"},{"line_number":306,"context_line":"         etag, deleted, expired_at) \u003d data[:7]"},{"line_number":307,"context_line":"        if len(data) \u003e 7:"},{"line_number":308,"context_line":"            storage_policy_index \u003d data[7]"},{"line_number":309,"context_line":"        else:"},{"line_number":310,"context_line":"            storage_policy_index \u003d 0"},{"line_number":311,"context_line":"        item_list.append({\u0027name\u0027: name,"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_d777709d","line":308,"updated":"2015-12-01 22:49:53.000000000","message":"this looks like if there\u0027s a current .pending we\u0027ll put the storage_policy_index in as an expired_at?","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":350,"context_line":"                    WHERE expired_at \u003c %s"},{"line_number":351,"context_line":"                )\u0027\u0027\u0027 % (timestamp))"},{"line_number":352,"context_line":"            expired_count \u003d cursor.rowcount"},{"line_number":353,"context_line":"            conn.commit()"},{"line_number":354,"context_line":"        return expired_count"},{"line_number":355,"context_line":""},{"line_number":356,"context_line":"    def delete_object(self, name, timestamp, storage_policy_index\u003d0):"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_97b7a8f4","line":353,"updated":"2015-12-01 22:49:53.000000000","message":"what about reaping rows from the expired table?","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":739,"context_line":""},{"line_number":740,"context_line":"        :param item_list: list of dictionaries of {\u0027name\u0027, \u0027created_at\u0027,"},{"line_number":741,"context_line":"                          \u0027size\u0027, \u0027content_type\u0027, \u0027etag\u0027,"},{"line_number":742,"context_line":"                          \u0027deleted\u0027, \u0027storage_policy_index\u0027}"},{"line_number":743,"context_line":"        :param source: if defined, update incoming_sync with the source"},{"line_number":744,"context_line":"        \"\"\""},{"line_number":745,"context_line":"        for item in item_list:"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_b2267ab8","line":742,"updated":"2015-12-01 22:49:53.000000000","message":"and expired_at yeah?","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":804,"context_line":"                                              \u0027deleted\u0027: rec[\u0027deleted\u0027]})"},{"line_number":805,"context_line":"                        rowid \u003d query.fetchone()[0]"},{"line_number":806,"context_line":"                        curs.execute(\u0027INSERT INTO expired VALUES (?, ?)\u0027,"},{"line_number":807,"context_line":"                                     (rowid, rec[\u0027expired_at\u0027]))"},{"line_number":808,"context_line":"            if source:"},{"line_number":809,"context_line":"                # for replication we rely on the remote end sending merges in"},{"line_number":810,"context_line":"                # order with no gaps to increment sync_points"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_37e83ce2","line":807,"updated":"2015-12-01 22:49:53.000000000","message":"this could be a lot of queries if you\u0027re merging a large .pending\n\nIt doesn\u0027t look like replication currently sends records with their expired_at keys (which I\u0027m somewhat dubious of), but if it did there could be as many as 10K items in to_add and we\u0027re doing 2 queries for each.","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1349a1123c4d6d768635e8da6c51bc972a075477","unresolved":false,"context_lines":[{"line_number":306,"context_line":"        (name, timestamp, size, content_type,"},{"line_number":307,"context_line":"         etag, deleted, expired_at) \u003d data[:7]"},{"line_number":308,"context_line":"        if len(data) \u003e 7:"},{"line_number":309,"context_line":"            storage_policy_index \u003d data[7]"},{"line_number":310,"context_line":"        else:"},{"line_number":311,"context_line":"            storage_policy_index \u003d 0"},{"line_number":312,"context_line":"        item_list.append({\u0027name\u0027: name,"}],"source_content_type":"text/x-python","patch_set":5,"id":"fa69d971_d0539714","line":309,"updated":"2016-01-11 19:44:55.000000000","message":"This is not correct.\n\nIf a .pending file is on-disk right now with 7 columns the last one is storage_policy_index - not expired_at.\n\nIf an old .pending file is on-disk right now with 6 columns this blows up with a ValueError.","commit_id":"46b6417b7362ec63387b97573ca23805f7616536"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1349a1123c4d6d768635e8da6c51bc972a075477","unresolved":false,"context_lines":[{"line_number":794,"context_line":"                    else:"},{"line_number":795,"context_line":"                        to_add \u003d _check_existance(item,"},{"line_number":796,"context_line":"                                                  item_ident,"},{"line_number":797,"context_line":"                                                  to_add)"},{"line_number":798,"context_line":"            if to_delete:"},{"line_number":799,"context_line":"                curs.executemany("},{"line_number":800,"context_line":"                    \u0027DELETE FROM object WHERE \u0027 + query_mod +"}],"source_content_type":"text/x-python","patch_set":5,"id":"fa69d971_7070ab40","line":797,"updated":"2016-01-11 19:44:55.000000000","message":"is it possible that an item_ident might end up in both to_expired and to_add?\n\nIt seems that the existing code was handling the case of multiple ident\u0027s with different timestamps in the .pending file - if the older one has no expired, and the newer does - which dict(s) would have the ident for the obj?","commit_id":"46b6417b7362ec63387b97573ca23805f7616536"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"1349a1123c4d6d768635e8da6c51bc972a075477","unresolved":false,"context_lines":[{"line_number":812,"context_line":"                         rec[\u0027content_type\u0027], rec[\u0027etag\u0027],"},{"line_number":813,"context_line":"                         rec[\u0027deleted\u0027], rec[\u0027storage_policy_index\u0027])))"},{"line_number":814,"context_line":"                    curs.execute(\u0027INSERT INTO expired VALUES (?, ?)\u0027,"},{"line_number":815,"context_line":"                                 (curs.lastrowid, rec[\u0027expired_at\u0027]))"},{"line_number":816,"context_line":"            if to_add:"},{"line_number":817,"context_line":"                curs.executemany("},{"line_number":818,"context_line":"                    \u0027INSERT INTO object (name, created_at, size, content_type,\u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"fa69d971_10a28fae","line":815,"updated":"2016-01-11 19:44:55.000000000","message":"... considering the implications of potentially many to_expired rows the .pending - use of the lastrowid might make it difficult to get this into an executemany, or it might not make a difference.","commit_id":"46b6417b7362ec63387b97573ca23805f7616536"}],"swift/container/replicator.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":190,"context_line":"        expired_count \u003d broker.remove_expired_objects()"},{"line_number":191,"context_line":"        if expired_count:"},{"line_number":192,"context_line":"            self.logger.info(\"%s objects expired and removed.\" %"},{"line_number":193,"context_line":"                             expired_count)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"    def _post_replicate_hook(self, broker, info, responses):"},{"line_number":196,"context_line":"        if info[\u0027account\u0027] \u003d\u003d MISPLACED_OBJECTS_ACCOUNT:"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_9729e839","line":193,"updated":"2015-12-01 22:49:53.000000000","message":"is the expirery data not replicated?","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"}],"swift/obj/auditor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":222,"context_line":"                    self.total_bytes_processed +\u003d chunk_len"},{"line_number":223,"context_line":"        except DiskFileExpired:"},{"line_number":224,"context_line":"            df \u003d self.diskfile_mgr.get_diskfile_from_audit_location(location)"},{"line_number":225,"context_line":"            df.delete(normalize_timestamp(time.time()))"},{"line_number":226,"context_line":"            diskfile.invalidate_hash(dirname(dirname(df._datadir)))"},{"line_number":227,"context_line":"            self.expired +\u003d 1"},{"line_number":228,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_d7e0d0ce","line":225,"updated":"2015-12-01 22:49:53.000000000","message":"shouldn\u0027t we use the delete at time?","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"00ad4f3e0a4d47f4549e959fb939cfc9be56b148","unresolved":false,"context_lines":[{"line_number":223,"context_line":"        except DiskFileExpired:"},{"line_number":224,"context_line":"            df \u003d self.diskfile_mgr.get_diskfile_from_audit_location(location)"},{"line_number":225,"context_line":"            df.delete(normalize_timestamp(time.time()))"},{"line_number":226,"context_line":"            diskfile.invalidate_hash(dirname(dirname(df._datadir)))"},{"line_number":227,"context_line":"            self.expired +\u003d 1"},{"line_number":228,"context_line":"            return"},{"line_number":229,"context_line":"        except DiskFileNotExist:"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_f72074f9","line":226,"updated":"2015-12-01 22:49:53.000000000","message":"I\u0027m not sure this is strictly needed, but it may be more safe.  Nonetheless, I think the analogy when all object nodes should independently discover the same end-state is reclaim - which does not currently cause the entire suffix to be re-hashed.  Of course currently a DELETE from the expirer causes a re-hash so maybe it\u0027s a future enhancement.","commit_id":"5d50740c2b71591479c432ae45957fdaeef63c18"}]}