)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4062d7b3d5e249379e13e243943fbd1cc4b294d9","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"f71f7902_0d0be04c","updated":"2021-11-12 10:11:50.000000000","message":"@Matt thanks for making a start on this. I put some sketchy ideas inline.\n\nWhat\u0027s the requirements? This applies fixed per-attr rate limiting, but is there a requirement for adaptive rate limiting?\n\nCould this be used for 3 levels of rate limiting i.e. in context of object updater, per container, per node and overall?","commit_id":"c7b7423a8beff3b6201c545a638068361f047e2a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"ec57da6860e2c5b38642d4863ac5d6e813ffc32a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"21d0b251_03429ff8","updated":"2021-11-12 06:09:12.000000000","message":"Here\u0027s a really dodgy script to kinda test it: https://paste.opendev.org/show/810958/","commit_id":"c7b7423a8beff3b6201c545a638068361f047e2a"}],"swift/common/utils.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4062d7b3d5e249379e13e243943fbd1cc4b294d9","unresolved":true,"context_lines":[{"line_number":1873,"context_line":"    def add_value_to_iters(self, next_value):"},{"line_number":1874,"context_line":"        attr \u003d getattr(next_value, self.attribute, None)"},{"line_number":1875,"context_line":"        if attr and attr not in self.per_attribute_iters:"},{"line_number":1876,"context_line":"            self.per_attribute_iters[attr] \u003d ("},{"line_number":1877,"context_line":"                RateLimitedIterator([], self.elements_per_second,"},{"line_number":1878,"context_line":"                                    self.limit_after), [])"},{"line_number":1879,"context_line":"        self.per_attribute_iters[attr][-1].append(next_value)"}],"source_content_type":"text/x-python","patch_set":1,"id":"c19d9094_56427b67","line":1876,"range":{"start_line":1876,"start_character":12,"end_line":1876,"end_character":42},"updated":"2021-11-12 10:11:50.000000000","message":"when does anything get removed from per_attribute_iters? i.e. what bounds the size of per_attribute_iters?\n\nIn later comment I sketch an idea for removing per_attr_iters that have seen no action for a period of time.","commit_id":"c7b7423a8beff3b6201c545a638068361f047e2a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4062d7b3d5e249379e13e243943fbd1cc4b294d9","unresolved":true,"context_lines":[{"line_number":1902,"context_line":"        next_time \u003d min(map(get_running_time,"},{"line_number":1903,"context_line":"                            self.per_attribute_iters.values()))"},{"line_number":1904,"context_line":"        sleep_interval \u003d next_time - time.time() / 1000.0"},{"line_number":1905,"context_line":"        return sleep_interval if sleep_interval \u003e 0 else 0"},{"line_number":1906,"context_line":""},{"line_number":1907,"context_line":"    def next(self):"},{"line_number":1908,"context_line":"        wrapped_iter_done \u003d False"}],"source_content_type":"text/x-python","patch_set":1,"id":"4fc290e7_ae2aa364","line":1905,"updated":"2021-11-12 10:11:50.000000000","message":"or max(0, sleep_interval)","commit_id":"c7b7423a8beff3b6201c545a638068361f047e2a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4062d7b3d5e249379e13e243943fbd1cc4b294d9","unresolved":true,"context_lines":[{"line_number":1909,"context_line":"        while True:"},{"line_number":1910,"context_line":"            try:"},{"line_number":1911,"context_line":"                # if there is something available in our attr iters use it"},{"line_number":1912,"context_line":"                next_value \u003d next(self.rebuild_iters())"},{"line_number":1913,"context_line":"                self.yielded +\u003d 1"},{"line_number":1914,"context_line":"                self.remove_value_from_iters(next_value)"},{"line_number":1915,"context_line":"                return next_value"}],"source_content_type":"text/x-python","patch_set":1,"id":"601e5ced_36f808de","line":1912,"range":{"start_line":1912,"start_character":34,"end_line":1912,"end_character":54},"updated":"2021-11-12 10:11:50.000000000","message":"IIUC this generator is constructed but the instance only ever yields one value? Does it need to be a generator?\n\nWRT my comment below (i.e. with some additions to the RateLimitedIterator interface), could we have something like (this is very sketchy!!):\n\n  min_sleep \u003d 0\n  for attr, it in self.per_attribute_iters():  # maybe shuffle them for fairness\n      it_sleep \u003d it.would_sleep()\n      if it_sleep \u003c 0:\n          try:\n              return next(it)\n          except StopIteration:\n             if it_sleep \u003c -N:  \n                 # i.e. nothing yielded in last N secs\n                 # this iter really is empty and not ratelimited\n                 # so remove it - we may never have items for \n                 # this attr again, and if we do then we can create a\n                 # fresh RateLimitedIterator\n                 self.per_attribute_iters.pop(attr)\n      elif min_sleep \u003c 0 or it_sleep \u003c min_sleep:\n          min_sleeper \u003d attr\n          min_sleep \u003d it_sleep\n\n   # fill per_attribute_iters from self.iterator...\n   if wrapped_iter_done:\n       if self.yielded \u003d\u003d self.processed:\n           raise StopIteration \n       eventlet.sleep(min_sleep)  # we learnt min_sleep in the loop","commit_id":"c7b7423a8beff3b6201c545a638068361f047e2a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"77dc5de04f71a6f10ae21e27d3ed540c99d45543","unresolved":true,"context_lines":[{"line_number":1909,"context_line":"        while True:"},{"line_number":1910,"context_line":"            try:"},{"line_number":1911,"context_line":"                # if there is something available in our attr iters use it"},{"line_number":1912,"context_line":"                next_value \u003d next(self.rebuild_iters())"},{"line_number":1913,"context_line":"                self.yielded +\u003d 1"},{"line_number":1914,"context_line":"                self.remove_value_from_iters(next_value)"},{"line_number":1915,"context_line":"                return next_value"}],"source_content_type":"text/x-python","patch_set":1,"id":"28981b2b_c0eefeed","line":1912,"range":{"start_line":1912,"start_character":34,"end_line":1912,"end_character":54},"in_reply_to":"601e5ced_36f808de","updated":"2021-11-15 04:12:50.000000000","message":"Cool thanks Al, some good tips, and seems much cleaner then attempting to recreate iterators because I want to add things.\n\nGone a little futher. We can\u0027t pop unused iters in this loop because of:\n\n  RuntimeError: dictionary changed size during iteration\n\nSo added some garbage collection, so if it\u0027s a long running thing we will be garbage collectiong unused iterators.\n\nAlso the \"time to next ratelimit event\" isn\u0027t since the last event, so cant simply use it for tracking GC. Anyway, uploading a new version. Still need to add some tests and fix up the commit message and docs.. more of a.. how about this version 😊","commit_id":"c7b7423a8beff3b6201c545a638068361f047e2a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4062d7b3d5e249379e13e243943fbd1cc4b294d9","unresolved":true,"context_lines":[{"line_number":3579,"context_line":"                        but better average accuracy. Must be \u003e 0 to engage"},{"line_number":3580,"context_line":"                        rate-limiting behavior."},{"line_number":3581,"context_line":"    :param throw: Instead of sleep, throw the given throwable. This is used"},{"line_number":3582,"context_line":"                  in the attributedRateLimiter."},{"line_number":3583,"context_line":"    \"\"\""},{"line_number":3584,"context_line":"    if max_rate \u003c\u003d 0 or incr_by \u003c\u003d 0:"},{"line_number":3585,"context_line":"        return running_time"}],"source_content_type":"text/x-python","patch_set":1,"id":"416247f9_f139f417","line":3582,"updated":"2021-11-12 10:11:50.000000000","message":"At the risk of being accused of LBYL, I wonder whether it might be simpler for the RateLimitedIterator to have a would_sleep() method so that it doesn\u0027t have to throw StopIteration and then, IIUC, be replaced with a fresh iter? \n\nWe could refactor this function to break out a separate helper:\n  \n  sleep_time \u003d would_sleep(running_time, max_rate, incr_by\u003d1, rate_buffer\u003d5)\n\nthat can be called from ratelimit_sleep() and also proxied by RateLimitedIterator","commit_id":"c7b7423a8beff3b6201c545a638068361f047e2a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"34f231c73547ead1c638523b72287e7268c2c16f","unresolved":true,"context_lines":[{"line_number":1820,"context_line":"                 │  RateLimittedIter* │A │A │A │  │"},{"line_number":1821,"context_line":"                 │                    ├──┼──┼──┼──┤"},{"line_number":1822,"context_line":"                 │  RateLimittedIter* │B │  │  │  │"},{"line_number":1823,"context_line":"    ◄── Chain ◄──┤                    ├──┼──┼──┼──┤"},{"line_number":1824,"context_line":"                 │  RateLimittedIter* │C │C │  │  │"},{"line_number":1825,"context_line":"                 │                    ├──┼──┼──┼──┤"},{"line_number":1826,"context_line":"                 │  RateLimittedIter* │D │  │  │  │"}],"source_content_type":"text/x-python","patch_set":3,"id":"5800963b_5a3ca42a","line":1823,"range":{"start_line":1823,"start_character":3,"end_line":1823,"end_character":14},"updated":"2021-11-15 22:01:33.000000000","message":"Chain iterator isn\u0027t a thing anymore.","commit_id":"783365687a0a5802c095537d36e5f6a2bcc678d9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"34f231c73547ead1c638523b72287e7268c2c16f","unresolved":true,"context_lines":[{"line_number":1847,"context_line":"    5. if the wrapped gets depleated and there are still items waiting, then"},{"line_number":1848,"context_line":"       we start sleeping the minimum sleep until something should be ready."},{"line_number":1849,"context_line":""},{"line_number":1850,"context_line":"    Caveats:"},{"line_number":1851,"context_line":"      Because we chain and call StopIteration on the RateLimiters, we need to"},{"line_number":1852,"context_line":"      reset the iterators everytime, which we need to do anyway because the"},{"line_number":1853,"context_line":"      rate attribute bucket/lists might have grown from the wrapped iterator."},{"line_number":1854,"context_line":"      This means popping yielded values from the bucket is important or we\u0027ll"},{"line_number":1855,"context_line":"      call the same objects again and again :("},{"line_number":1856,"context_line":"    \"\"\""},{"line_number":1857,"context_line":"    def __init__(self, iterable, attribute, elements_per_second,"},{"line_number":1858,"context_line":"                 limit_after\u003d0, gc_time\u003d5):"}],"source_content_type":"text/x-python","patch_set":3,"id":"29a744c2_c4bcee2b","line":1855,"range":{"start_line":1850,"start_character":4,"end_line":1855,"end_character":46},"updated":"2021-11-15 22:01:33.000000000","message":"Not correct anymore.","commit_id":"783365687a0a5802c095537d36e5f6a2bcc678d9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b9f4be986897ba6270e0585fad5c1683d17a1259","unresolved":true,"context_lines":[{"line_number":1774,"context_line":"        raise ValueError(\u0027Invalid partition: %s\u0027 % quote(partition or \u0027\u0027))"},{"line_number":1775,"context_line":""},{"line_number":1776,"context_line":""},{"line_number":1777,"context_line":"class AdaptiveRateLimitedBase(object):"},{"line_number":1778,"context_line":"    feedback_effected_attribute \u003d None"},{"line_number":1779,"context_line":""},{"line_number":1780,"context_line":"    def adjust_rate(self, value, **kwargs):"}],"source_content_type":"text/x-python","patch_set":4,"id":"c0b531ce_3970b894","line":1777,"updated":"2021-12-02 05:51:56.000000000","message":"Playing with a potential generic way of adjusing ratelimtting feedback.\n\nThis is so my test token and the existing time based ratelimiter can \"start\" to be adaptive.","commit_id":"c60425890696acb987b693bfd4b8898bb037e4cf"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b9f4be986897ba6270e0585fad5c1683d17a1259","unresolved":true,"context_lines":[{"line_number":1821,"context_line":"    __next__ \u003d next"},{"line_number":1822,"context_line":""},{"line_number":1823,"context_line":""},{"line_number":1824,"context_line":"class TokenRateLimitedIterator(AdaptiveRateLimitedBase):"},{"line_number":1825,"context_line":"    feedback_effected_attribute \u003d \u0027refill_rate\u0027"},{"line_number":1826,"context_line":""},{"line_number":1827,"context_line":"    def __init__(self, interable, high_water_mark, refill_rate, sleep\u003dTrue,"}],"source_content_type":"text/x-python","patch_set":4,"id":"3b9b4303_15a83cfa","line":1824,"updated":"2021-12-02 05:51:56.000000000","message":"Just playing with token based rate limiter","commit_id":"c60425890696acb987b693bfd4b8898bb037e4cf"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b9f4be986897ba6270e0585fad5c1683d17a1259","unresolved":true,"context_lines":[{"line_number":1965,"context_line":"                 unique_ratelimiter_kwargs, limit_after\u003d0, gc_time\u003d5):"},{"line_number":1966,"context_line":"        self.iterator \u003d iter(iterable)"},{"line_number":1967,"context_line":"        self.ratelimiter_class \u003d ratelimiter_class"},{"line_number":1968,"context_line":"        self.ratelimiter_kwargs \u003d unique_ratelimiter_kwargs"},{"line_number":1969,"context_line":"        self.limit_after \u003d limit_after"},{"line_number":1970,"context_line":"        self.attribute \u003d attribute"},{"line_number":1971,"context_line":"        self.gc_time \u003d gc_time"}],"source_content_type":"text/x-python","patch_set":4,"id":"b1dd0b94_064c2315","line":1968,"updated":"2021-12-02 05:51:56.000000000","message":"Make this more pluggable. I can now use it with the test token and time based rate limiters. And use adjustments on the rate.","commit_id":"c60425890696acb987b693bfd4b8898bb037e4cf"}]}