)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"c253b8c8714625f0e113ff043070f43bb2378af9","unresolved":false,"context_lines":[{"line_number":10,"context_line":"them:"},{"line_number":11,"context_line":"* Make sure we always indicate why some readiness or liveness probe"},{"line_number":12,"context_line":"  fail."},{"line_number":13,"context_line":"* Supress INFO logs from werkzeug (so that we don\u0027t see every probe"},{"line_number":14,"context_line":"  call on INFO level)."},{"line_number":15,"context_line":"* Remove logging of successful probe checks."},{"line_number":16,"context_line":"* Make watcher restart logs less scary and include more cases."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"ff570b3c_de898775","line":13,"range":{"start_line":13,"start_character":2,"end_line":13,"end_character":9},"updated":"2020-06-01 11:59:46.000000000","message":"Suppress.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"c253b8c8714625f0e113ff043070f43bb2378af9","unresolved":false,"context_lines":[{"line_number":17,"context_line":"* Add backoff to watcher restarts so that we don\u0027t spam logs when K8s"},{"line_number":18,"context_line":"  API is briefly unavailable."},{"line_number":19,"context_line":"* Add warnings for low quotas."},{"line_number":20,"context_line":"* Supress some long logs on K8s healthz failures - we don\u0027t need full"},{"line_number":21,"context_line":"  message from K8s printed twice."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"I also refactored CNI and controller health probes servers to make sure"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"ff570b3c_1e801f46","line":20,"range":{"start_line":20,"start_character":2,"end_line":20,"end_character":9},"updated":"2020-06-01 11:59:46.000000000","message":"ditto.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"}],"kuryr_kubernetes/health.py":[{"author":{"_account_id":27032,"name":"Maysa de Macedo Souza","email":"maysa.macedo95@gmail.com","username":"maysa"},"change_message_id":"c7985e4b0500e2d603fc2da294cdd50c59137cc0","unresolved":false,"context_lines":[{"line_number":34,"context_line":"            \u0027/ready\u0027, methods\u003d[\u0027GET\u0027], view_func\u003dself.readiness_status)"},{"line_number":35,"context_line":"        self.application.add_url_rule("},{"line_number":36,"context_line":"            \u0027/alive\u0027, methods\u003d[\u0027GET\u0027], view_func\u003dself.liveness_status)"},{"line_number":37,"context_line":"        self.headers \u003d {\u0027Connection\u0027: \u0027close\u0027}"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"        def apply_conn_close(response):"},{"line_number":40,"context_line":"            response.headers[\u0027Connection\u0027] \u003d \u0027close\u0027"}],"source_content_type":"text/x-python","patch_set":4,"id":"bf51134e_26bc5168","line":37,"range":{"start_line":37,"start_character":8,"end_line":37,"end_character":46},"updated":"2020-07-03 10:08:05.000000000","message":"Looks like this can be removed.","commit_id":"0effeb716feea94c8fe37fffa96ed40ce462c81d"}],"kuryr_kubernetes/k8s_client.py":[{"author":{"_account_id":27032,"name":"Maysa de Macedo Souza","email":"maysa.macedo95@gmail.com","username":"maysa"},"change_message_id":"823ce83cd683f848be21a7f8378b0e0cee68dabf","unresolved":false,"context_lines":[{"line_number":305,"context_line":"                log(\u0027Connection error when watching %s. Retrying in %ds with \u0027"},{"line_number":306,"context_line":"                    \u0027resourceVersion\u003d%s\u0027, path, t,"},{"line_number":307,"context_line":"                    params.get(\u0027resourceVersion\u0027))"},{"line_number":308,"context_line":"                time.sleep(t)"},{"line_number":309,"context_line":"                attempt +\u003d 1"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_f94e8194","line":308,"range":{"start_line":308,"start_character":16,"end_line":308,"end_character":29},"updated":"2020-06-10 14:01:26.000000000","message":"How hard will this affect our response on handling events?","commit_id":"98889ad88bb026b49e54f9e0112e63a4c5302b71"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0e1df10c650da965854130cf19e3883d1c21607d","unresolved":false,"context_lines":[{"line_number":305,"context_line":"                log(\u0027Connection error when watching %s. Retrying in %ds with \u0027"},{"line_number":306,"context_line":"                    \u0027resourceVersion\u003d%s\u0027, path, t,"},{"line_number":307,"context_line":"                    params.get(\u0027resourceVersion\u0027))"},{"line_number":308,"context_line":"                time.sleep(t)"},{"line_number":309,"context_line":"                attempt +\u003d 1"}],"source_content_type":"text/x-python","patch_set":3,"id":"ff570b3c_4e6ebec9","line":308,"range":{"start_line":308,"start_character":16,"end_line":308,"end_character":29},"in_reply_to":"ff570b3c_f94e8194","updated":"2020-06-10 14:34:15.000000000","message":"I bet not much, if it\u0027s just a disconnect due to data timeout, we reconnect immediately (min_backoff\u003d0). If it\u0027s a longer outage of Kubernetes API, we start just a normal backoff instead of retrying in a loop.","commit_id":"98889ad88bb026b49e54f9e0112e63a4c5302b71"},{"author":{"_account_id":23567,"name":"Luis Tomas Bolivar","email":"ltomasbo@redhat.com","username":"ltomasbo"},"change_message_id":"11ac5e9be5d692c5689c8c065b79664529fb02f5","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":4,"id":"bf51134e_441b6472","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"updated":"2020-07-01 07:24:23.000000000","message":"this is not only log civilizing. Do we want to wait up to 1 min to reconnect? why that much?","commit_id":"0effeb716feea94c8fe37fffa96ed40ce462c81d"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"d9bc01c2b902d2a31bdc8439c0e4afce850333de","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":4,"id":"bf51134e_bbba4cde","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"in_reply_to":"bf51134e_441b6472","updated":"2020-07-02 07:56:16.000000000","message":"That\u0027s a valid point. I\u0027d opt for spliting the patch to civilize logging and improving code regarding backoff mechanisms.","commit_id":"0effeb716feea94c8fe37fffa96ed40ce462c81d"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"5c9b073190298dd28674f9e6d9fc2a7d9247ae8c","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":4,"id":"bf51134e_86ff2833","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"in_reply_to":"bf51134e_a696ac93","updated":"2020-07-02 16:23:49.000000000","message":"Very well.","commit_id":"0effeb716feea94c8fe37fffa96ed40ce462c81d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"3e84b2c5f195bca8feede53b79beee9fe1812e78","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":4,"id":"bf51134e_a696ac93","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"in_reply_to":"bf51134e_bbba4cde","updated":"2020-07-02 15:02:41.000000000","message":"It is civilizing logging - without backoff we just try reconnecting several times a second, spamming logs with hundreds of lines when API is down for just a brief period.\n\nUp to a minute seems like a fair number when the API was down for several minutes, I\u0027ll try tweaking this.","commit_id":"0effeb716feea94c8fe37fffa96ed40ce462c81d"},{"author":{"_account_id":23567,"name":"Luis Tomas Bolivar","email":"ltomasbo@redhat.com","username":"ltomasbo"},"change_message_id":"fd3390464fae63920c60e192682dff767420bf82","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf51134e_5daca48b","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"updated":"2020-07-06 06:59:34.000000000","message":"we don\u0027t need the if/raise anymore?","commit_id":"d8892d2e727241c5b157b2206101f3b7a738c39f"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"145d3c3d2ea259d3bfee3f73aa09d377bd4327e5","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf51134e_a92ab47a","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"in_reply_to":"bf51134e_50b910e7","updated":"2020-07-08 13:40:36.000000000","message":"The idea is that way broader range of errors will be caught here, so that first occurrence will only produce a debug LOG. \n\nI don\u0027t want to just reraise exception after specified number of tries as it\u0027ll be caught in the upper layer (watcher), logged and retried there. It\u0027s better that we just retry with a reasonable backoff of 60 seconds forever - just as K8s services like kubelet do.","commit_id":"d8892d2e727241c5b157b2206101f3b7a738c39f"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"d40a4906198ca8d7df96565241a57d74504c821c","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf51134e_50b910e7","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"in_reply_to":"bf51134e_5daca48b","updated":"2020-07-07 12:07:13.000000000","message":"I guess, that we need to raise the exception after\n\n  attempt \u003e utils.MAX_ATTEMPTS\n\nOtherwise we would end up in endless loop (or until k8s api start to respond).","commit_id":"d8892d2e727241c5b157b2206101f3b7a738c39f"},{"author":{"_account_id":23567,"name":"Luis Tomas Bolivar","email":"ltomasbo@redhat.com","username":"ltomasbo"},"change_message_id":"0a4d4c43f3812959b33f2e945ec1b2a5d54bce61","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf51134e_283970d6","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"in_reply_to":"bf51134e_7ff32a9f","updated":"2020-07-09 07:36:17.000000000","message":"ok","commit_id":"d8892d2e727241c5b157b2206101f3b7a738c39f"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"b7f737398329c53621a8be94b4c919161b18e02d","unresolved":false,"context_lines":[{"line_number":298,"context_line":"                            resource_version \u003d m.get(\u0027resourceVersion\u0027, None)"},{"line_number":299,"context_line":"            except (requests.ReadTimeout, requests.ConnectionError,"},{"line_number":300,"context_line":"                    ssl.SSLError, requests.exceptions.ChunkedEncodingError):"},{"line_number":301,"context_line":"                t \u003d utils.exponential_backoff(attempt, min_backoff\u003d0,"},{"line_number":302,"context_line":"                                              max_backoff\u003dMAX_BACKOFF)"},{"line_number":303,"context_line":"                log \u003d LOG.debug"},{"line_number":304,"context_line":"                if attempt \u003e 0:"},{"line_number":305,"context_line":"                    # Only make it a warning if it\u0027s happening again, no need"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf51134e_7ff32a9f","line":302,"range":{"start_line":301,"start_character":0,"end_line":302,"end_character":70},"in_reply_to":"bf51134e_a92ab47a","updated":"2020-07-08 15:10:02.000000000","message":"Oh. I see. Very well.","commit_id":"d8892d2e727241c5b157b2206101f3b7a738c39f"}],"kuryr_kubernetes/utils.py":[{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"c253b8c8714625f0e113ff043070f43bb2378af9","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"def exponential_backoff(attempt, interval\u003dDEFAULT_INTERVAL, min_backoff\u003d1,"},{"line_number":146,"context_line":"                        max_backoff\u003dNone):"},{"line_number":147,"context_line":"    if attempt \u003e\u003d MAX_ATTEMPTS:"},{"line_number":148,"context_line":"        # No need to calculate very long intervals"},{"line_number":149,"context_line":"        attempt \u003d MAX_ATTEMPTS"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    backoff \u003d random.randint(min_backoff, 2 ** attempt - 1) * interval"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    if max_backoff is not None and backoff \u003e max_backoff:"},{"line_number":154,"context_line":"        backoff \u003d max_backoff"}],"source_content_type":"text/x-python","patch_set":2,"id":"ff570b3c_3eafe37d","line":151,"range":{"start_line":147,"start_character":0,"end_line":151,"end_character":70},"updated":"2020-06-01 11:59:46.000000000","message":"How long do you plan to wait on this exponential thing in k8s client? Why using random? Did you know, that at max attempts, most likely you\u0027ll get values expressed in years? Maaaany years. Like millions.\n\nHow about using simple equation, like:\n\n    backoff \u003d attempt ** 3\n\nand reduce MAX_ATTEMPTS to 20, which gives you around two hours and thirteen seconds.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"d9bc01c2b902d2a31bdc8439c0e4afce850333de","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"def exponential_backoff(attempt, interval\u003dDEFAULT_INTERVAL, min_backoff\u003d1,"},{"line_number":146,"context_line":"                        max_backoff\u003dNone):"},{"line_number":147,"context_line":"    if attempt \u003e\u003d MAX_ATTEMPTS:"},{"line_number":148,"context_line":"        # No need to calculate very long intervals"},{"line_number":149,"context_line":"        attempt \u003d MAX_ATTEMPTS"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    backoff \u003d random.randint(min_backoff, 2 ** attempt - 1) * interval"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    if max_backoff is not None and backoff \u003e max_backoff:"},{"line_number":154,"context_line":"        backoff \u003d max_backoff"}],"source_content_type":"text/x-python","patch_set":2,"id":"bf51134e_d553ab6b","line":151,"range":{"start_line":147,"start_character":0,"end_line":151,"end_character":70},"in_reply_to":"bf51134e_f5450f45","updated":"2020-07-02 07:56:16.000000000","message":"Yup. I guess so.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c6966757885c278291135c53b75a073406012c68","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"def exponential_backoff(attempt, interval\u003dDEFAULT_INTERVAL, min_backoff\u003d1,"},{"line_number":146,"context_line":"                        max_backoff\u003dNone):"},{"line_number":147,"context_line":"    if attempt \u003e\u003d MAX_ATTEMPTS:"},{"line_number":148,"context_line":"        # No need to calculate very long intervals"},{"line_number":149,"context_line":"        attempt \u003d MAX_ATTEMPTS"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    backoff \u003d random.randint(min_backoff, 2 ** attempt - 1) * interval"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    if max_backoff is not None and backoff \u003e max_backoff:"},{"line_number":154,"context_line":"        backoff \u003d max_backoff"}],"source_content_type":"text/x-python","patch_set":2,"id":"ff570b3c_9e22cbb9","line":151,"range":{"start_line":147,"start_character":0,"end_line":151,"end_character":70},"in_reply_to":"ff570b3c_3eafe37d","updated":"2020-06-10 13:29:15.000000000","message":"My assumption was not to change what was already implemented in exponential_sleep, the algorithm\u0027s source is explained there. I expect the random there is to make sure all the watchers won\u0027t get restarted in the same second when a persistent connection failure occurred.\n\nI\u0027ve checked implementation from retrying library and it seems pretty similar to this one, so I\u0027d rather stick with it, just reducing MAX_ATTEMPTS to 10.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"c3be5ca9b7aea038023ec9f22f54e024487bacdc","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"def exponential_backoff(attempt, interval\u003dDEFAULT_INTERVAL, min_backoff\u003d1,"},{"line_number":146,"context_line":"                        max_backoff\u003dNone):"},{"line_number":147,"context_line":"    if attempt \u003e\u003d MAX_ATTEMPTS:"},{"line_number":148,"context_line":"        # No need to calculate very long intervals"},{"line_number":149,"context_line":"        attempt \u003d MAX_ATTEMPTS"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    backoff \u003d random.randint(min_backoff, 2 ** attempt - 1) * interval"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    if max_backoff is not None and backoff \u003e max_backoff:"},{"line_number":154,"context_line":"        backoff \u003d max_backoff"}],"source_content_type":"text/x-python","patch_set":2,"id":"bf51134e_f5450f45","line":151,"range":{"start_line":147,"start_character":0,"end_line":151,"end_character":70},"in_reply_to":"ff570b3c_6e92a2a1","updated":"2020-06-30 14:12:22.000000000","message":"Alright, I\u0027ve just put 60 seconds there, I see no scenario when we want to wait longer. And we reach 60 seconds only on 5th retry, meaning that API was down for around 39 seconds already, which seems reasonable.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"},{"author":{"_account_id":13692,"name":"Roman Dobosz","email":"gryf73@gmail.com","username":"gryf"},"change_message_id":"3846bf3a461dd7d74262e9232b00ed5ee983434e","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"def exponential_backoff(attempt, interval\u003dDEFAULT_INTERVAL, min_backoff\u003d1,"},{"line_number":146,"context_line":"                        max_backoff\u003dNone):"},{"line_number":147,"context_line":"    if attempt \u003e\u003d MAX_ATTEMPTS:"},{"line_number":148,"context_line":"        # No need to calculate very long intervals"},{"line_number":149,"context_line":"        attempt \u003d MAX_ATTEMPTS"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    backoff \u003d random.randint(min_backoff, 2 ** attempt - 1) * interval"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    if max_backoff is not None and backoff \u003e max_backoff:"},{"line_number":154,"context_line":"        backoff \u003d max_backoff"}],"source_content_type":"text/x-python","patch_set":2,"id":"ff570b3c_ee2a327c","line":151,"range":{"start_line":147,"start_character":0,"end_line":151,"end_character":70},"in_reply_to":"ff570b3c_9e22cbb9","updated":"2020-06-10 14:27:17.000000000","message":"I see.\n\nAs for adding some randomness, I guess current implementation can breake contract of exponential backoff. Maybe something like this could be considered as a solution, so we make sure, the sleep is long enough for each attempt, yet it provides an amount of randomness:\n\n  backoff \u003d random.randint(2 ** (attempt - 1), 2 ** attempt -1) * interval\n\nWith DEFAULT_INTERNAL\u003d3 and MAX_ATTEMPTS\u003d10, that would gives max 51 minutes and 9 seconds, if my math is right, so my question remains, how long do we plan to wait before making another attempt? My wild guess would be tens of seconds, not minutes, but I might be wrong.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0e1df10c650da965854130cf19e3883d1c21607d","unresolved":false,"context_lines":[{"line_number":144,"context_line":""},{"line_number":145,"context_line":"def exponential_backoff(attempt, interval\u003dDEFAULT_INTERVAL, min_backoff\u003d1,"},{"line_number":146,"context_line":"                        max_backoff\u003dNone):"},{"line_number":147,"context_line":"    if attempt \u003e\u003d MAX_ATTEMPTS:"},{"line_number":148,"context_line":"        # No need to calculate very long intervals"},{"line_number":149,"context_line":"        attempt \u003d MAX_ATTEMPTS"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"    backoff \u003d random.randint(min_backoff, 2 ** attempt - 1) * interval"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    if max_backoff is not None and backoff \u003e max_backoff:"},{"line_number":154,"context_line":"        backoff \u003d max_backoff"}],"source_content_type":"text/x-python","patch_set":2,"id":"ff570b3c_6e92a2a1","line":151,"range":{"start_line":147,"start_character":0,"end_line":151,"end_character":70},"in_reply_to":"ff570b3c_ee2a327c","updated":"2020-06-10 14:34:15.000000000","message":"Hm, true, I\u0027m using [kubernetes]watch_retry_timeout as a max_backoff value, which defaults to 60 seconds, but I see we recommend even setting it to an hour. That\u0027s way too long. Okay, let me revisit this.","commit_id":"3f5257cf7ba896e6087dcf307d18d8828e6ef43a"},{"author":{"_account_id":23567,"name":"Luis Tomas Bolivar","email":"ltomasbo@redhat.com","username":"ltomasbo"},"change_message_id":"0a4d4c43f3812959b33f2e945ec1b2a5d54bce61","unresolved":false,"context_lines":[{"line_number":251,"context_line":"    if availability \u003c\u003d 3:"},{"line_number":252,"context_line":"        LOG.warning(\"Neutron quota low for %s. Used %d out of %d limit.\","},{"line_number":253,"context_line":"                    resource, resource_quota[\u0027limit\u0027], resource_quota[\u0027used\u0027])"},{"line_number":254,"context_line":"    elif availability \u003c\u003d 0:"},{"line_number":255,"context_line":"        LOG.error(\"Neutron quota exceeded for %s. Used %d out of %d limit.\","},{"line_number":256,"context_line":"                  resource, resource_quota[\u0027limit\u0027], resource_quota[\u0027used\u0027])"},{"line_number":257,"context_line":"        return False"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf51134e_e8581832","line":254,"range":{"start_line":254,"start_character":9,"end_line":254,"end_character":26},"updated":"2020-07-09 07:36:17.000000000","message":"should this be \u003d\u003d 0 only? probably this is not call when quota is -1, but just in case. Quota -1 means infinite quota","commit_id":"d8892d2e727241c5b157b2206101f3b7a738c39f"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"34feb781e52b2f464c5ed52d73f4552ed1ebca11","unresolved":false,"context_lines":[{"line_number":251,"context_line":"    if availability \u003c\u003d 3:"},{"line_number":252,"context_line":"        LOG.warning(\"Neutron quota low for %s. Used %d out of %d limit.\","},{"line_number":253,"context_line":"                    resource, resource_quota[\u0027limit\u0027], resource_quota[\u0027used\u0027])"},{"line_number":254,"context_line":"    elif availability \u003c\u003d 0:"},{"line_number":255,"context_line":"        LOG.error(\"Neutron quota exceeded for %s. Used %d out of %d limit.\","},{"line_number":256,"context_line":"                  resource, resource_quota[\u0027limit\u0027], resource_quota[\u0027used\u0027])"},{"line_number":257,"context_line":"        return False"}],"source_content_type":"text/x-python","patch_set":5,"id":"bf51134e_f3662ebb","line":254,"range":{"start_line":254,"start_character":9,"end_line":254,"end_character":26},"in_reply_to":"bf51134e_e8581832","updated":"2020-07-10 13:45:11.000000000","message":"Well, if quota is -1, then availability will end up as wrong result anyway and this is always checked with `has_limit()`, so it doesn\u0027t matter. But very good catch, this could be reworked a bit in a follow up patch.","commit_id":"d8892d2e727241c5b157b2206101f3b7a738c39f"}]}