)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"3a69f42f62d2db5a0a27bb1a95fd25480f419f33","unresolved":false,"context_lines":[{"line_number":9,"context_line":"We used a back window of 2 block sizes before, but we should by default"},{"line_number":10,"context_line":"use a one block back window, meaning it\u0027s not possible to send data"},{"line_number":11,"context_line":"backward. We now allow this back window to be configured to be any"},{"line_number":12,"context_line":"timespan the user wants."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: Ib934dc6f1cbce8e7595a20626a5d3a0b5b4678e6"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"5a890539_806dd69a","line":12,"updated":"2014-11-13 13:28:08.000000000","message":"Sorry, I\u0027m being a bit blind today, I can\u0027t see how the user could configure this back window.\n\nI was expected to see something exposed in the archive policy to allow this span to be controlled either (a) per-entity, or (b) per-granularity/retention pair for each entity.\n\nSeems that option (a) would be more convenient for the storage driver (since the full-res timeseries could be maintained by the driver once for all archives in the entity).\n\nIIUC, basing the back_window on the block_size is effectively the same thing, as the block size for the TimeSerieArchive is the *coarsest* granularity for the corresponding entity, amiright?\n\nAlso does switching to a single block-size by default mean that we keep full-res data for a full block *in addition* to current period being filled? (or that we just keep full-res data for the current period being filled?)","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"aa42f0d75469bdccfc5f2ad3be4d26a9cbff1baa","unresolved":false,"context_lines":[{"line_number":9,"context_line":"We used a back window of 2 block sizes before, but we should by default"},{"line_number":10,"context_line":"use a one block back window, meaning it\u0027s not possible to send data"},{"line_number":11,"context_line":"backward. We now allow this back window to be configured to be any"},{"line_number":12,"context_line":"timespan the user wants."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: Ib934dc6f1cbce8e7595a20626a5d3a0b5b4678e6"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"5a890539_2696fa4c","line":12,"in_reply_to":"5a890539_806dd69a","updated":"2014-11-13 14:03:52.000000000","message":"This is a carbonara only fix, so it\u0027s exposed to the user of Carbonara, not to Gnocchi user itself yet.\n\nThat would be finally exposed by archive policy in the Gnocchi REST API – hoping other drivers will have some kind of ability like that. So my plan is option (a).\n\nYou\u0027re right for the block size being the coarsest granularity.\n\nAnd finally, that means we keep a full-res data for a full block for the current period being filled, that\u0027s it. So you can\u0027t go back the current period being filled (unless you request more blocks for back window).","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"}],"gnocchi/carbonara.py":[{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"3a69f42f62d2db5a0a27bb1a95fd25480f419f33","unresolved":false,"context_lines":[{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_ab755399","line":153,"updated":"2014-11-13 13:28:08.000000000","message":"Why do we need to copy and resample the timeseries here, when we haven\u0027t actually insert any new data as yet?","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"e56e5bc5cf5e5fdd0291f009f30268ec80cc273e","unresolved":false,"context_lines":[{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_f6873fd5","line":153,"in_reply_to":"5a890539_0c491d8b","updated":"2014-11-13 15:44:03.000000000","message":"This is a bound time serie, there\u0027s no notion of resampling elsewhere in this class. It\u0027s not an aggregated time serie.","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"52022b2664de99e2f321e7201803502473187d33","unresolved":false,"context_lines":[{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_0c491d8b","line":153,"in_reply_to":"5a890539_66dc8213","updated":"2014-11-13 14:48:38.000000000","message":"A-ha, so the effect of the resample is just to round the last literal timestamp to the start of the period?\n\nWould it be a lot cheaper to just do that rounding directly on the last timestamp, without having to re-run the aggregation function?\n\nAlso, would that resampling have already been done after the last set_values insertion, so would we even need to do it again here?","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"aa42f0d75469bdccfc5f2ad3be4d26a9cbff1baa","unresolved":false,"context_lines":[{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_66dc8213","line":153,"in_reply_to":"5a890539_ab755399","updated":"2014-11-13 14:03:52.000000000","message":"This is to compute the timestamp of the last block (ts.index[-1]).\n\nBut actually there\u0027s no need to copy before, you\u0027re right. Fixing.","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"3a69f42f62d2db5a0a27bb1a95fd25480f419f33","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_cb51bf0f","line":154,"updated":"2014-11-13 13:28:08.000000000","message":"The subtraction of 1 from back_window confuses me, as back_window defaults to 1, so this will default to the timestamp of the most recent item in the timeseries.\n\n(i.e. effectively no back_window?)","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"e56e5bc5cf5e5fdd0291f009f30268ec80cc273e","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_f6481f89","line":154,"in_reply_to":"5a890539_0c5cbd24","updated":"2014-11-13 15:44:03.000000000","message":"It is definitely determenistic. You cannot go back in time or you have to keep all the datapoints for ever. You have to drop the non-aggregated datapoints at some points.\n\nBy default this point of no return is the timestamp of the last aggregation window (e.g. 14:00 in a 1 hour aggregation window).\n\nIf you want your scenario to work, you\u0027ll have to double the back window so the points are kept until that back_window + 1, e.g. 13:00 in that case.\n\n(or buy a Delorean).","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"da8ad3e9e7152908552e0e82605142408420d663","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_aa0b6828","line":154,"in_reply_to":"5a890539_2ae5b8dc","updated":"2014-11-14 11:34:07.000000000","message":"No you are totally right, now I think you are arguing about the default value.\n\nI think the default value of 0 is fine within Carbonara. I don\u0027t have yet exposed the back window parameter in the REST API for the archive policies, but I think having a default value of 1 so that you are sure you can handle points a little late when you just switched from an aggregation window to a next one is a sane default.","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"383f713903933f723c6fe4b9b110cf9293255aee","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_7ac31789","line":154,"in_reply_to":"5a890539_32b2a694","updated":"2014-11-14 08:41:45.000000000","message":"I definitely understand what you mean, you were just misusing \"deterministic\". :)","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"fc43fe8eded38811c4808e430459f1af35154739","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_8a430c41","line":154,"in_reply_to":"5a890539_604d263e","updated":"2014-11-14 11:08:11.000000000","message":"As I said, by default there\u0027s no assumption you are able to go back in time. If you want to do so, you have to increase the back window. You focus on your example, but you could also make the same argument with points arriving out of order over 24 hours and complains that it may not work. Yeah, with no back window, it may not work. :)\n\n(a) is setting back_window\u003d1 (with latest patchset), so that\u0027s just an option if you want to store more points\n\n(b) that\u0027s not possible, you need the complete block to compute correct aggregation","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"880d00e79321062bf65a1f5cea568cf7bbb1ab8f","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_604d263e","line":154,"in_reply_to":"5a890539_7ac31789","updated":"2014-11-14 10:42:11.000000000","message":"Fair enough - a deterministic outcome in gnocchi for a particular ordering of datapoints received.\n\nCan we agree though that a non-deterministic ordering of the *submission* of those datapoints is possible, with multiple collectors pulling off the metering topic and submitting to gnocchi?\n\nHere\u0027s an idea to consider ... instead of the back_window just being the timespan since the current coarsest period started to be filled, with edge effects as current time traverses a period boundary (i.e. start of day, start of hour etc.) how about either:\n\n(a) setting the default back window to be one entire block *plus* the current period being filled\n\nor:\n\n(b) interpreting the back_window as a sliding window equivalent in length to one block, but not anchored at a period boundary (instead anchored on current time)","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"52022b2664de99e2f321e7201803502473187d33","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_0c5cbd24","line":154,"in_reply_to":"5a890539_8645eeb4","updated":"2014-11-13 14:48:38.000000000","message":"So, seems that this strict interpretation of the back window could have awkward \"edge effects\".\n\nSay two close datapoints arrive out-of-order at the gnocchi storage driver, with timestamps 13:59:55 and 14:00:05.\n\nIf processed in timestamp-order, both datapoints would be registered in an entity with 1hr granularity.\n\nHowever if processed in the order received, the earlier datapoint would be discarded.\n\nThis seems like potentially surprising non-deterministic behaviour.\n\n(BTW with the current release of InfluxDB, that would be the only behaviour that we could support, as backfilled data that arrives after a downsampling continuous query is run cannot be fed into a down-sample already computed ... however I\u0027ve been pushing pauldix to support continuous queries that are aware of lagging data)","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"575ad77d5cb183520b981478c2ffd464ded7d89c","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_2ae5b8dc","line":154,"in_reply_to":"5a890539_8a430c41","updated":"2014-11-14 11:22:44.000000000","message":"A-ha, yes, of course the aggregation issue ... so scratch option (b) :)\n\nSo the reason I\u0027m laboring this point to death is that I\u0027m much more concerned with what happens by default with slightly out-of-order datapoints arriving in the very recent past (not a full 24 hours into the past).\n\nMost of the time, that \"very recent past\" is totally tolerant of this slight mis-ordering (because current time is usually not near to the boundary of the coarsest period for the entity).\n\nBut for a short timespan after the period switches from the prev to the next (e.g shortly after midnight if that granularity is 1day) we end up on a knife-edge, with slight mis-orderings of incoming datapoints having a different result to a similar slight misordering occurring in the middle of the period.\n\nAm I over-stating or misunderstanding that potential issue in the default case?","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"751459a05b72f571fd99a0f8f49bf780fc896114","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_6d3fdad6","line":154,"in_reply_to":"5a890539_aa0b6828","updated":"2014-11-14 12:00:52.000000000","message":"Cool, I think we\u0027re ad idem :)\n\nAs mentioned above, the InfluxDB driver will only be able to support that default carbonara behavior initially (i.e. back_window\u003d0) until pauldix fixes the laggy-data-awareness issue for continuous queries.\n\nBut once that\u0027s done, and we expose this mechanism via the REST API, then agreed that the suggested switch to an overarching default of 1 full period (in addition to the current in-progress period) would be sane.","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"aa42f0d75469bdccfc5f2ad3be4d26a9cbff1baa","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_8645eeb4","line":154,"in_reply_to":"5a890539_cb51bf0f","updated":"2014-11-13 14:03:52.000000000","message":"yeah no back_window by default meaning we only keep one block \u003d the current block being aggregated. You can\u0027t go back.\n\ne.g. if the block size is one hour and the last point sent is 14:52, you can\u0027t send points before 14:00.","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"},{"author":{"_account_id":2284,"name":"Eoghan Glynn","email":"eglynn@redhat.com","username":"eglynn"},"change_message_id":"55be792a93322d929b92fca0985e0ee06fe44964","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def _first_block_timestamp(self):"},{"line_number":152,"context_line":"        ts \u003d self.ts.copy()"},{"line_number":153,"context_line":"        ts \u003d ts.resample(self.block_size)"},{"line_number":154,"context_line":"        return (ts.index[-1] - (self.block_size * (self.back_window - 1)))"},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"    def _truncate(self):"},{"line_number":157,"context_line":"        \"\"\"Truncate the timeserie.\"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"5a890539_32b2a694","line":154,"in_reply_to":"5a890539_f6481f89","updated":"2014-11-13 21:54:53.000000000","message":"Sure, it\u0027s deterministic if the datapoints arrive in the expected order.\n\nThe problem occurs when datapoints are slightly out-of-order near a period boundary. Which is possible with multiple collectors and gnocchi servers. \n\nSee the modified test in https://review.openstack.org/134368","commit_id":"2bfa92f2471b5db02d236214627577675547dedd"}]}