)]}'
{"gnocchi/storage/_carbonara.py":[{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"b3e1df71edcf00dc472c7e3111e1c745b2fad336","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                               oldest_mutable_timestamp):"},{"line_number":226,"context_line":"        # NOTE(jd) We write the full split only if the driver works that way"},{"line_number":227,"context_line":"        # (self.WRITE_FULL) or if the oldest_mutable_timestamp is out of range."},{"line_number":228,"context_line":"        write_full \u003d self.WRITE_FULL or oldest_mutable_timestamp \u003e\u003d next(key)"},{"line_number":229,"context_line":"        key_as_str \u003d str(key)"},{"line_number":230,"context_line":"        if write_full:"},{"line_number":231,"context_line":"            offset \u003d 0"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_298175fb","line":228,"updated":"2016-08-30 14:44:18.000000000","message":"do we still need this oldest_mutable check if we already rewrite older obj below?","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"c1bd26b4fc5ff246b5d05924b2177a3b74edcbcd","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                               oldest_mutable_timestamp):"},{"line_number":226,"context_line":"        # NOTE(jd) We write the full split only if the driver works that way"},{"line_number":227,"context_line":"        # (self.WRITE_FULL) or if the oldest_mutable_timestamp is out of range."},{"line_number":228,"context_line":"        write_full \u003d self.WRITE_FULL or oldest_mutable_timestamp \u003e\u003d next(key)"},{"line_number":229,"context_line":"        key_as_str \u003d str(key)"},{"line_number":230,"context_line":"        if write_full:"},{"line_number":231,"context_line":"            offset \u003d 0"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_bada2515","line":228,"in_reply_to":"fa7ab95a_298175fb","updated":"2016-08-30 15:20:56.000000000","message":"Yes: the rewrite is done before any new storage, so if a new split is being stored and is before oldest_mutable_timestamp, then we want to have it store compressed directly.","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"c1bd26b4fc5ff246b5d05924b2177a3b74edcbcd","unresolved":false,"context_lines":[{"line_number":291,"context_line":"        # happens if `previous_oldest_mutable_timestamp\u0027 exists, which means we"},{"line_number":292,"context_line":"        # already wrote some splits at some point – so this is not the first"},{"line_number":293,"context_line":"        # time we treat this timeserie."},{"line_number":294,"context_line":"        if already_have_splits:"},{"line_number":295,"context_line":"            previous_oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":296,"context_line":"                previous_oldest_mutable_timestamp))"},{"line_number":297,"context_line":"            oldest_mutable_key \u003d str(ts.get_split_key("}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_bafb0546","line":294,"updated":"2016-08-30 15:20:56.000000000","message":"I forgot to check: this is useless if self.WRITE_FULL is True","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"b3e1df71edcf00dc472c7e3111e1c745b2fad336","unresolved":false,"context_lines":[{"line_number":292,"context_line":"        # already wrote some splits at some point – so this is not the first"},{"line_number":293,"context_line":"        # time we treat this timeserie."},{"line_number":294,"context_line":"        if already_have_splits:"},{"line_number":295,"context_line":"            previous_oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":296,"context_line":"                previous_oldest_mutable_timestamp))"},{"line_number":297,"context_line":"            oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":298,"context_line":"                oldest_mutable_timestamp))"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"            for key in existing_keys:"},{"line_number":301,"context_line":"                if previous_oldest_mutable_key \u003c\u003d key \u003c oldest_mutable_key:"},{"line_number":302,"context_line":"                    # NOTE(jd) Rewrite it entirely for fun (and later for"},{"line_number":303,"context_line":"                    # compression). For that, we just pass None as split."},{"line_number":304,"context_line":"                    self._store_timeserie_split("},{"line_number":305,"context_line":"                        metric, carbonara.SplitKey.from_key_string("},{"line_number":306,"context_line":"                            key, archive_policy_def.granularity),"},{"line_number":307,"context_line":"                        None, aggregation, archive_policy_def,"},{"line_number":308,"context_line":"                        oldest_mutable_timestamp)"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"        for key, split in ts.split():"},{"line_number":311,"context_line":"            if key \u003e\u003d oldest_key_to_keep:"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_89ad21eb","line":308,"range":{"start_line":295,"start_character":0,"end_line":308,"end_character":49},"updated":"2016-08-30 14:44:18.000000000","message":"can we do this stuff after truncate? i was thinking we can keep all the add_measures stuff relatively similar. just write_full or write_with_offset.\n\nand then after we finished, we have a compress step that does something like abiove: check if previous oldest mutable split key is different from the truncated old mutable split key, if yes, go back to previous obj and read+compress+write.","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"c1bd26b4fc5ff246b5d05924b2177a3b74edcbcd","unresolved":false,"context_lines":[{"line_number":292,"context_line":"        # already wrote some splits at some point – so this is not the first"},{"line_number":293,"context_line":"        # time we treat this timeserie."},{"line_number":294,"context_line":"        if already_have_splits:"},{"line_number":295,"context_line":"            previous_oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":296,"context_line":"                previous_oldest_mutable_timestamp))"},{"line_number":297,"context_line":"            oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":298,"context_line":"                oldest_mutable_timestamp))"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"            for key in existing_keys:"},{"line_number":301,"context_line":"                if previous_oldest_mutable_key \u003c\u003d key \u003c oldest_mutable_key:"},{"line_number":302,"context_line":"                    # NOTE(jd) Rewrite it entirely for fun (and later for"},{"line_number":303,"context_line":"                    # compression). For that, we just pass None as split."},{"line_number":304,"context_line":"                    self._store_timeserie_split("},{"line_number":305,"context_line":"                        metric, carbonara.SplitKey.from_key_string("},{"line_number":306,"context_line":"                            key, archive_policy_def.granularity),"},{"line_number":307,"context_line":"                        None, aggregation, archive_policy_def,"},{"line_number":308,"context_line":"                        oldest_mutable_timestamp)"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"        for key, split in ts.split():"},{"line_number":311,"context_line":"            if key \u003e\u003d oldest_key_to_keep:"}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_faf02d69","line":308,"range":{"start_line":295,"start_character":0,"end_line":308,"end_character":49},"in_reply_to":"fa7ab95a_89ad21eb","updated":"2016-08-30 15:20:56.000000000","message":"We could do that I guess but I don\u0027t see the gain?","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"1e120e2dcebcbb98cbda3607d8086984fed14591","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":496,"context_line":"                                      carbonara.round_timestamp("},{"line_number":497,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":498,"context_line":"                                  current_first_block_timestamp,"},{"line_number":499,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":500,"context_line":"                                 for aggregation in agg_methods"},{"line_number":501,"context_line":"                                 for d in metric.archive_policy.definition))"},{"line_number":502,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_c9f6495c","line":499,"range":{"start_line":498,"start_character":0,"end_line":499,"end_character":73},"updated":"2016-08-30 14:46:17.000000000","message":"i think this might end up being the same timestamp since this happens before truncate?","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"7ec712b2ebd310260cae9ae4675ec99758cf2917","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":496,"context_line":"                                      carbonara.round_timestamp("},{"line_number":497,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":498,"context_line":"                                  current_first_block_timestamp,"},{"line_number":499,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":500,"context_line":"                                 for aggregation in agg_methods"},{"line_number":501,"context_line":"                                 for d in metric.archive_policy.definition))"},{"line_number":502,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_e6940f38","line":499,"range":{"start_line":498,"start_character":0,"end_line":499,"end_character":73},"in_reply_to":"fa7ab95a_901c5d6b","updated":"2016-08-30 16:37:44.000000000","message":"right, but i think it should be no? if my first_block_timestamp is always related to largest block, i might have a split for one of the smaller timespans but they won\u0027t get rewritten\u0026compressed because the longer series doesn\u0027t need to?","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"5f12c35e13bd779903fe6cf4f95064b9e2144430","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":496,"context_line":"                                      carbonara.round_timestamp("},{"line_number":497,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":498,"context_line":"                                  current_first_block_timestamp,"},{"line_number":499,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":500,"context_line":"                                 for aggregation in agg_methods"},{"line_number":501,"context_line":"                                 for d in metric.archive_policy.definition))"},{"line_number":502,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_a0c2f282","line":499,"range":{"start_line":498,"start_character":0,"end_line":499,"end_character":73},"in_reply_to":"fa7ab95a_9aae412c","updated":"2016-08-30 15:39:37.000000000","message":"right, so the first blockstamp will still be from the unaggregated series? if or backwindow series is [(t1,1),(t2,2)] and we add (t3,3), the first_block_timestamp will still be t1... or maybe i understand this wrong.","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"94558011dd3b089718a5154e3e68d164fd5e91ac","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":496,"context_line":"                                      carbonara.round_timestamp("},{"line_number":497,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":498,"context_line":"                                  current_first_block_timestamp,"},{"line_number":499,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":500,"context_line":"                                 for aggregation in agg_methods"},{"line_number":501,"context_line":"                                 for d in metric.archive_policy.definition))"},{"line_number":502,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_c080d68d","line":499,"range":{"start_line":498,"start_character":0,"end_line":499,"end_character":73},"in_reply_to":"fa7ab95a_a0c2f282","updated":"2016-08-30 15:46:20.000000000","message":"actually i just thought of another issue, first_block_timestamp is only valid for series with longest back_window. if we have multiple granularities, their oldest_mutable_timestamp will not be the same.","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"7e9835ade4dcd61cfbeb31ad4380911a9dbf3450","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":496,"context_line":"                                      carbonara.round_timestamp("},{"line_number":497,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":498,"context_line":"                                  current_first_block_timestamp,"},{"line_number":499,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":500,"context_line":"                                 for aggregation in agg_methods"},{"line_number":501,"context_line":"                                 for d in metric.archive_policy.definition))"},{"line_number":502,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_901c5d6b","line":499,"range":{"start_line":498,"start_character":0,"end_line":499,"end_character":73},"in_reply_to":"fa7ab95a_c080d68d","updated":"2016-08-30 16:08:36.000000000","message":"The block size for a BoundTimeserie is always the larger block of an archive policy definition. So if you\u0027re largest granularity is 1 hour, the block will be one hour max (with a back_window of 0). If the back_window is 2, the BoundTimeSerie will be 3 hours max).\n\nThe first block timestamp has nothing to do with split (let\u0027s be clear about that).\n\nI hope that\u0027ll help you think about it :)","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"c1bd26b4fc5ff246b5d05924b2177a3b74edcbcd","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":496,"context_line":"                                      carbonara.round_timestamp("},{"line_number":497,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":498,"context_line":"                                  current_first_block_timestamp,"},{"line_number":499,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":500,"context_line":"                                 for aggregation in agg_methods"},{"line_number":501,"context_line":"                                 for d in metric.archive_policy.definition))"},{"line_number":502,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_9aae412c","line":499,"range":{"start_line":498,"start_character":0,"end_line":499,"end_character":73},"in_reply_to":"fa7ab95a_c9f6495c","updated":"2016-08-30 15:20:56.000000000","message":"Nop because the new measures has been added this time, so the new one is likely to be more in the future than the old one.","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"ac51bd7aab1a15bc1bc371c1ea2193f8c82f0eec","unresolved":false,"context_lines":[{"line_number":495,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":496,"context_line":"                                      carbonara.round_timestamp("},{"line_number":497,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":498,"context_line":"                                  current_first_block_timestamp,"},{"line_number":499,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":500,"context_line":"                                 for aggregation in agg_methods"},{"line_number":501,"context_line":"                                 for d in metric.archive_policy.definition))"},{"line_number":502,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"fa7ab95a_e114f9a0","line":499,"range":{"start_line":498,"start_character":0,"end_line":499,"end_character":73},"in_reply_to":"fa7ab95a_e6940f38","updated":"2016-08-30 16:51:59.000000000","message":"They will be rewritten\u0026compressed if they are (the key is) between old_first_block_timestamp and current_first_block_timestamp.\n\nSo let\u0027s take an example: your BoundTimeSeries is one hour long.\nYou have a point for 11:58 in it. So your back window is one hour long and can go back to 11:00.\n\nYou wrote the 11:58 point for an aggregate of 5 minutes in a 11:55 point, in a split that covers up to 12:55. That split is not compressed yet, as maybe later you\u0027ll get a point for 12:23 and they\u0027ll be in the split for the 12:20 aggregation.\n\nBut what you get ispoint for 15:40.\n\n`current_first_block_timestamp\u0027 is 11:00 and the new one (L499) is going to be 15:00.\nSo any split that contains data between 11:00 and 15:00 need to be rewritten. Which means your split with the 11:55 aggregate need compression!\n\nThat\u0027s what L295 and after do: they compute the split key for 11:00 and 15:00 and recompress every splits that matches that. :)","commit_id":"13281d78bf7f22db7131b5aa87591692f0898286"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"58ff7976936bd33a8f7e7c6794df8665feebdbe8","unresolved":false,"context_lines":[{"line_number":230,"context_line":"                               oldest_mutable_timestamp):"},{"line_number":231,"context_line":"        # NOTE(jd) We write the full split only if the driver works that way"},{"line_number":232,"context_line":"        # (self.WRITE_FULL) or if the oldest_mutable_timestamp is out of range."},{"line_number":233,"context_line":"        write_full \u003d self.WRITE_FULL or oldest_mutable_timestamp \u003e\u003d next(key)"},{"line_number":234,"context_line":"        key_as_str \u003d str(key)"},{"line_number":235,"context_line":"        if write_full:"},{"line_number":236,"context_line":"            offset \u003d 0"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_08e9f9b6","line":233,"range":{"start_line":233,"start_character":37,"end_line":233,"end_character":77},"updated":"2016-08-30 19:09:03.000000000","message":"i still don\u0027t think we need this. i think we can just check to see if a split is None. if it\u0027s none, we will write_full if not, we will just write offset (or full if no offset is applicable)","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"4cbc3b17bb428ab8bbfbd8b5a4f1ff074f50810a","unresolved":false,"context_lines":[{"line_number":230,"context_line":"                               oldest_mutable_timestamp):"},{"line_number":231,"context_line":"        # NOTE(jd) We write the full split only if the driver works that way"},{"line_number":232,"context_line":"        # (self.WRITE_FULL) or if the oldest_mutable_timestamp is out of range."},{"line_number":233,"context_line":"        write_full \u003d self.WRITE_FULL or oldest_mutable_timestamp \u003e\u003d next(key)"},{"line_number":234,"context_line":"        key_as_str \u003d str(key)"},{"line_number":235,"context_line":"        if write_full:"},{"line_number":236,"context_line":"            offset \u003d 0"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_900fd62d","line":233,"range":{"start_line":233,"start_character":37,"end_line":233,"end_character":77},"in_reply_to":"fa7ab95a_08e9f9b6","updated":"2016-08-31 07:42:30.000000000","message":"Nop, we still need it. If your BoundTimeserie is full of 3 days of data (while your back window size is 1 day) and you generate aggregation for 1 sec, you\u0027ll end up with a lot splits that are going to be generated on L320 that are going to be readonly directly so you want to compress them right now.","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"3c3d43b494c4b00ec9b79bc6c779cd7da24dde28","unresolved":false,"context_lines":[{"line_number":230,"context_line":"                               oldest_mutable_timestamp):"},{"line_number":231,"context_line":"        # NOTE(jd) We write the full split only if the driver works that way"},{"line_number":232,"context_line":"        # (self.WRITE_FULL) or if the oldest_mutable_timestamp is out of range."},{"line_number":233,"context_line":"        write_full \u003d self.WRITE_FULL or oldest_mutable_timestamp \u003e\u003d next(key)"},{"line_number":234,"context_line":"        key_as_str \u003d str(key)"},{"line_number":235,"context_line":"        if write_full:"},{"line_number":236,"context_line":"            offset \u003d 0"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_d84a5127","line":233,"range":{"start_line":233,"start_character":37,"end_line":233,"end_character":77},"in_reply_to":"fa7ab95a_900fd62d","updated":"2016-08-31 12:58:06.000000000","message":"i see. nm it\u0027s because i keep thinking store should come before delete/compress. :)","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"58ff7976936bd33a8f7e7c6794df8665feebdbe8","unresolved":false,"context_lines":[{"line_number":274,"context_line":"            and previous_oldest_mutable_timestamp is not None"},{"line_number":275,"context_line":"        )"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"        if need_to_delete or already_have_splits:"},{"line_number":278,"context_line":"            existing_keys \u003d self._list_split_keys_for_metric("},{"line_number":279,"context_line":"                metric, aggregation, archive_policy_def.granularity)"},{"line_number":280,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_28287d99","line":277,"range":{"start_line":277,"start_character":29,"end_line":277,"end_character":48},"updated":"2016-08-30 19:09:03.000000000","message":"doesn\u0027t exist anywhere.","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"4cbc3b17bb428ab8bbfbd8b5a4f1ff074f50810a","unresolved":false,"context_lines":[{"line_number":274,"context_line":"            and previous_oldest_mutable_timestamp is not None"},{"line_number":275,"context_line":"        )"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"        if need_to_delete or already_have_splits:"},{"line_number":278,"context_line":"            existing_keys \u003d self._list_split_keys_for_metric("},{"line_number":279,"context_line":"                metric, aggregation, archive_policy_def.granularity)"},{"line_number":280,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_ce29c0f4","line":277,"range":{"start_line":277,"start_character":29,"end_line":277,"end_character":48},"in_reply_to":"fa7ab95a_28287d99","updated":"2016-08-31 07:42:30.000000000","message":"Done","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"58ff7976936bd33a8f7e7c6794df8665feebdbe8","unresolved":false,"context_lines":[{"line_number":280,"context_line":""},{"line_number":281,"context_line":"        # First delete old splits"},{"line_number":282,"context_line":"        if need_to_delete:"},{"line_number":283,"context_line":"            oldest_point_to_keep \u003d ts.last - datetime.timedelta("},{"line_number":284,"context_line":"                seconds\u003darchive_policy_def.timespan)"},{"line_number":285,"context_line":"            oldest_key_to_keep \u003d ts.get_split_key(oldest_point_to_keep)"},{"line_number":286,"context_line":"            oldest_key_to_keep_s \u003d str(oldest_key_to_keep)"},{"line_number":287,"context_line":"            for key in list(existing_keys):"},{"line_number":288,"context_line":"                # NOTE(jd) Only delete if the key is strictly inferior to"},{"line_number":289,"context_line":"                # the timestamp; we don\u0027t delete any timeserie split that"},{"line_number":290,"context_line":"                # contains our timestamp, so we prefer to keep a bit more"},{"line_number":291,"context_line":"                # than deleting too much"},{"line_number":292,"context_line":"                if key \u003c oldest_key_to_keep_s:"},{"line_number":293,"context_line":"                    self._delete_metric_measures("},{"line_number":294,"context_line":"                        metric, key, aggregation,"},{"line_number":295,"context_line":"                        archive_policy_def.granularity)"},{"line_number":296,"context_line":"                    existing_keys.remove(key)"},{"line_number":297,"context_line":"        else:"},{"line_number":298,"context_line":"            oldest_key_to_keep \u003d carbonara.SplitKey(0)"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"        # Rewrite all read-only splits just for fun (and compression) This only"},{"line_number":301,"context_line":"        # happens if `previous_oldest_mutable_timestamp\u0027 exists, which means we"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_43112251","line":298,"range":{"start_line":283,"start_character":0,"end_line":298,"end_character":54},"updated":"2016-08-30 19:09:03.000000000","message":"why did we need to change this? seems like nothing is using _delete_metric_measures_before anymore","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"4cbc3b17bb428ab8bbfbd8b5a4f1ff074f50810a","unresolved":false,"context_lines":[{"line_number":280,"context_line":""},{"line_number":281,"context_line":"        # First delete old splits"},{"line_number":282,"context_line":"        if need_to_delete:"},{"line_number":283,"context_line":"            oldest_point_to_keep \u003d ts.last - datetime.timedelta("},{"line_number":284,"context_line":"                seconds\u003darchive_policy_def.timespan)"},{"line_number":285,"context_line":"            oldest_key_to_keep \u003d ts.get_split_key(oldest_point_to_keep)"},{"line_number":286,"context_line":"            oldest_key_to_keep_s \u003d str(oldest_key_to_keep)"},{"line_number":287,"context_line":"            for key in list(existing_keys):"},{"line_number":288,"context_line":"                # NOTE(jd) Only delete if the key is strictly inferior to"},{"line_number":289,"context_line":"                # the timestamp; we don\u0027t delete any timeserie split that"},{"line_number":290,"context_line":"                # contains our timestamp, so we prefer to keep a bit more"},{"line_number":291,"context_line":"                # than deleting too much"},{"line_number":292,"context_line":"                if key \u003c oldest_key_to_keep_s:"},{"line_number":293,"context_line":"                    self._delete_metric_measures("},{"line_number":294,"context_line":"                        metric, key, aggregation,"},{"line_number":295,"context_line":"                        archive_policy_def.granularity)"},{"line_number":296,"context_line":"                    existing_keys.remove(key)"},{"line_number":297,"context_line":"        else:"},{"line_number":298,"context_line":"            oldest_key_to_keep \u003d carbonara.SplitKey(0)"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"        # Rewrite all read-only splits just for fun (and compression) This only"},{"line_number":301,"context_line":"        # happens if `previous_oldest_mutable_timestamp\u0027 exists, which means we"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_4e25b008","line":298,"range":{"start_line":283,"start_character":0,"end_line":298,"end_character":54},"in_reply_to":"fa7ab95a_43112251","updated":"2016-08-31 07:42:30.000000000","message":"I don\u0027t follow. I just unwrapped _delete_metric_measures_before here so we have to list keys only once and save us a request.","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"58ff7976936bd33a8f7e7c6794df8665feebdbe8","unresolved":false,"context_lines":[{"line_number":302,"context_line":"        # already wrote some splits at some point – so this is not the first"},{"line_number":303,"context_line":"        # time we treat this timeserie."},{"line_number":304,"context_line":"        if need_rewrite:"},{"line_number":305,"context_line":"            previous_oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":306,"context_line":"                previous_oldest_mutable_timestamp))"},{"line_number":307,"context_line":"            oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":308,"context_line":"                oldest_mutable_timestamp))"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"            for key in existing_keys:"},{"line_number":311,"context_line":"                if previous_oldest_mutable_key \u003c\u003d key \u003c oldest_mutable_key:"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_433842d3","line":308,"range":{"start_line":305,"start_character":0,"end_line":308,"end_character":42},"updated":"2016-08-30 19:09:03.000000000","message":"we should check to see if this two end up the same. if not, we can skip everything below.","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"4cbc3b17bb428ab8bbfbd8b5a4f1ff074f50810a","unresolved":false,"context_lines":[{"line_number":302,"context_line":"        # already wrote some splits at some point – so this is not the first"},{"line_number":303,"context_line":"        # time we treat this timeserie."},{"line_number":304,"context_line":"        if need_rewrite:"},{"line_number":305,"context_line":"            previous_oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":306,"context_line":"                previous_oldest_mutable_timestamp))"},{"line_number":307,"context_line":"            oldest_mutable_key \u003d str(ts.get_split_key("},{"line_number":308,"context_line":"                oldest_mutable_timestamp))"},{"line_number":309,"context_line":""},{"line_number":310,"context_line":"            for key in existing_keys:"},{"line_number":311,"context_line":"                if previous_oldest_mutable_key \u003c\u003d key \u003c oldest_mutable_key:"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_6eac3436","line":308,"range":{"start_line":305,"start_character":0,"end_line":308,"end_character":42},"in_reply_to":"fa7ab95a_433842d3","updated":"2016-08-31 07:42:30.000000000","message":"I hate this kind of check because they\u0027re an optimization for the compiler lol. but ok :)","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"58ff7976936bd33a8f7e7c6794df8665feebdbe8","unresolved":false,"context_lines":[{"line_number":318,"context_line":"                        oldest_mutable_timestamp)"},{"line_number":319,"context_line":""},{"line_number":320,"context_line":"        for key, split in ts.split():"},{"line_number":321,"context_line":"            if key \u003e\u003d oldest_key_to_keep:"},{"line_number":322,"context_line":"                self._store_timeserie_split("},{"line_number":323,"context_line":"                    metric, key, split, aggregation, archive_policy_def,"},{"line_number":324,"context_line":"                    oldest_mutable_timestamp)"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_a384ae5d","line":321,"range":{"start_line":321,"start_character":1,"end_line":321,"end_character":40},"updated":"2016-08-30 19:09:03.000000000","message":"can key actually be older than oldest point to keep? unless our entire series is part of backwindow? if so can\u0027t we just keep the delete step as last step?","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"3c3d43b494c4b00ec9b79bc6c779cd7da24dde28","unresolved":false,"context_lines":[{"line_number":318,"context_line":"                        oldest_mutable_timestamp)"},{"line_number":319,"context_line":""},{"line_number":320,"context_line":"        for key, split in ts.split():"},{"line_number":321,"context_line":"            if key \u003e\u003d oldest_key_to_keep:"},{"line_number":322,"context_line":"                self._store_timeserie_split("},{"line_number":323,"context_line":"                    metric, key, split, aggregation, archive_policy_def,"},{"line_number":324,"context_line":"                    oldest_mutable_timestamp)"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_98c9a9af","line":321,"range":{"start_line":321,"start_character":1,"end_line":321,"end_character":40},"in_reply_to":"fa7ab95a_2e486c87","updated":"2016-08-31 12:58:06.000000000","message":"ack, got it. it\u0027s because you\u0027re considering scenario where backlog is way bigger than backwindow.","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"4cbc3b17bb428ab8bbfbd8b5a4f1ff074f50810a","unresolved":false,"context_lines":[{"line_number":318,"context_line":"                        oldest_mutable_timestamp)"},{"line_number":319,"context_line":""},{"line_number":320,"context_line":"        for key, split in ts.split():"},{"line_number":321,"context_line":"            if key \u003e\u003d oldest_key_to_keep:"},{"line_number":322,"context_line":"                self._store_timeserie_split("},{"line_number":323,"context_line":"                    metric, key, split, aggregation, archive_policy_def,"},{"line_number":324,"context_line":"                    oldest_mutable_timestamp)"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_2e486c87","line":321,"range":{"start_line":321,"start_character":1,"end_line":321,"end_character":40},"in_reply_to":"fa7ab95a_a384ae5d","updated":"2016-08-31 07:42:30.000000000","message":"If you have a back_window of 1 day and one of your timeserie is just keeping 1 point of 5 min aggregate, when you might end up aggregating tons of data generating a lot of splits that are older than your oldest_key_to_keep…\n\nI don\u0027t think storing + then deleting is smart.","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"58ff7976936bd33a8f7e7c6794df8665feebdbe8","unresolved":false,"context_lines":[{"line_number":505,"context_line":"                                  carbonara.TimeSerie(bound_timeserie.ts["},{"line_number":506,"context_line":"                                      carbonara.round_timestamp("},{"line_number":507,"context_line":"                                          tstamp, d.granularity * 10e8):]),"},{"line_number":508,"context_line":"                                  current_first_block_timestamp,"},{"line_number":509,"context_line":"                                  bound_timeserie.first_block_timestamp())"},{"line_number":510,"context_line":"                                 for aggregation in agg_methods"},{"line_number":511,"context_line":"                                 for d in metric.archive_policy.definition))"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa7ab95a_0859f934","line":508,"updated":"2016-08-30 19:09:03.000000000","message":"got it. this makes sense to me now :)","commit_id":"1705efccfeb77f2326e8bd3a6ce78c8077f47ecc"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"54b1cdd454ff10ec4de2e88e2068c1143c86ce4a","unresolved":false,"context_lines":[{"line_number":279,"context_line":"                metric, aggregation, archive_policy_def.granularity)"},{"line_number":280,"context_line":""},{"line_number":281,"context_line":"        # First delete old splits"},{"line_number":282,"context_line":"        if need_to_delete:"},{"line_number":283,"context_line":"            oldest_point_to_keep \u003d ts.last - datetime.timedelta("},{"line_number":284,"context_line":"                seconds\u003darchive_policy_def.timespan)"},{"line_number":285,"context_line":"            oldest_key_to_keep \u003d ts.get_split_key(oldest_point_to_keep)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa7ab95a_d225d00a","line":282,"updated":"2016-08-31 12:59:26.000000000","message":"i see, 1 call makes sense. let\u0027s remove _delete_metric_measures_before then since it\u0027s not used anywhere anymore?\n\ni guess another question is do we need this check? archive_policy will always have a timespan and this code is skipped if ts is empty so won\u0027t it always be need_to_delete?","commit_id":"961d3bb77988ff2e44259fda1ce0ec268456a06a"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"523d1d0da4e88decf1ef0b00b87edcfa721cbe65","unresolved":false,"context_lines":[{"line_number":279,"context_line":"                metric, aggregation, archive_policy_def.granularity)"},{"line_number":280,"context_line":""},{"line_number":281,"context_line":"        # First delete old splits"},{"line_number":282,"context_line":"        if need_to_delete:"},{"line_number":283,"context_line":"            oldest_point_to_keep \u003d ts.last - datetime.timedelta("},{"line_number":284,"context_line":"                seconds\u003darchive_policy_def.timespan)"},{"line_number":285,"context_line":"            oldest_key_to_keep \u003d ts.get_split_key(oldest_point_to_keep)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa7ab95a_85ce9db2","line":282,"in_reply_to":"fa7ab95a_b8390d44","updated":"2016-08-31 13:27:47.000000000","message":"nevermind i was grepping wrong code :/\n\nwhat? you can create limitless series? /me starts to look at code again.","commit_id":"961d3bb77988ff2e44259fda1ce0ec268456a06a"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"052bb9b6e58c88bdc5d91e28b9a1580b2ddf57a8","unresolved":false,"context_lines":[{"line_number":279,"context_line":"                metric, aggregation, archive_policy_def.granularity)"},{"line_number":280,"context_line":""},{"line_number":281,"context_line":"        # First delete old splits"},{"line_number":282,"context_line":"        if need_to_delete:"},{"line_number":283,"context_line":"            oldest_point_to_keep \u003d ts.last - datetime.timedelta("},{"line_number":284,"context_line":"                seconds\u003darchive_policy_def.timespan)"},{"line_number":285,"context_line":"            oldest_key_to_keep \u003d ts.get_split_key(oldest_point_to_keep)"}],"source_content_type":"text/x-python","patch_set":4,"id":"fa7ab95a_b8390d44","line":282,"in_reply_to":"fa7ab95a_d225d00a","updated":"2016-08-31 13:04:23.000000000","message":"Hum where did you see _delete_metric_measures_before? I\u0027m grepping like crazy and I see no trace of it anymore.\n\nBut you have a point that checking for ts.last is not useless. But an archive policy can have no timespan if it has no limit on retention. I\u0027ll simplify a bit.","commit_id":"961d3bb77988ff2e44259fda1ce0ec268456a06a"}]}