)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"0a684ed1b327c1307f10d05bddc6d0d8628cfa5f","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"NOTE: This code assumes port cleanup happens quickly. If it doesn\u0027t, the"},{"line_number":10,"context_line":"OpenStack driver could appear to hang while the port cleanup process"},{"line_number":11,"context_line":"runs."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Change-Id: I81b57d6f6142e64dd0ebf31531ca6489d6c46583"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"3f79a3b5_916a0eea","line":11,"updated":"2018-10-19 19:26:15.000000000","message":"This is probably a good throttle anyway since the problem this alleviates is new hosts not booting because ports are leaking and we run out of quota.","commit_id":"07e5d5420743918a47b5900fa0e923f5f1370088"}],"nodepool/driver/openstack/provider.py":[{"author":{"_account_id":7118,"name":"Ian Wienand","email":"iwienand@redhat.com","username":"iwienand"},"change_message_id":"c617672d52fc298aa0858cfa58185b81e26dbecd","unresolved":false,"context_lines":[{"line_number":504,"context_line":"        if not self.down_ports:"},{"line_number":505,"context_line":"            self.down_ports \u003d self.listDownPorts()"},{"line_number":506,"context_line":""},{"line_number":507,"context_line":"        # if not enough time (3m?) has passed between calls to check down ports"},{"line_number":508,"context_line":"        #    return"},{"line_number":509,"context_line":"        # down \u003d self.listDownPorts()"},{"line_number":510,"context_line":"        # remove_ports \u003d intersection(self.down, down)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_c79cf35c","line":507,"updated":"2018-10-12 00:01:48.000000000","message":"I used this pause+compare externally in OSC, but since we\u0027re inside nodepool, we could actually correlate the port -\u003e instance mapping against active hosts, and then just purge any DOWN ports that are against instances we don\u0027t know about?\n\npossibly to make this easier the handler should put the associated port in after the create in [1], and we should track any DOWN ports no seen by that?\n\nI feel like it would be good to emit stats on this; i feel like we want to graph this because it indicates something is going wrong\n\nThe other thing is -- if this turns out to be a neutron bug in a very specific release/situation, either fixing it there and not worrying about it in nodepool -- or doing something with a cleanup function in OSC directly so everyone can use it (like delete_unattached_floating_ips) might be more appropriate.\n\n[1] https://git.openstack.org/cgit/openstack-infra/nodepool/tree/nodepool/driver/openstack/handler.py#n125","commit_id":"e54a506a53d5fe72979611b533345a287738c79c"},{"author":{"_account_id":3099,"name":"David Shrewsbury","email":"dshrewsb@redhat.com","username":"dshrews"},"change_message_id":"1b627ac3ebeb28d9461a0315b8cbddc0f12e05e2","unresolved":false,"context_lines":[{"line_number":504,"context_line":"        if not self.down_ports:"},{"line_number":505,"context_line":"            self.down_ports \u003d self.listDownPorts()"},{"line_number":506,"context_line":""},{"line_number":507,"context_line":"        # if not enough time (3m?) has passed between calls to check down ports"},{"line_number":508,"context_line":"        #    return"},{"line_number":509,"context_line":"        # down \u003d self.listDownPorts()"},{"line_number":510,"context_line":"        # remove_ports \u003d intersection(self.down, down)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3f79a3b5_a9d6de25","line":507,"in_reply_to":"3f79a3b5_c79cf35c","updated":"2018-10-18 18:48:33.000000000","message":"In order to compare a port against known nodes, we would have to iterate through all nodes from ZK, then compare the port IPs with all IPs from the ZK nodes. That\u0027s kind of a heavy operation to do every 60s (default interval for the cleanup thread). I feel like spacing this out a bit might be better.\n\nAs for putting the port into the ZK node after server create, if the server data returned from the create operation contained that info, I might be on board with it. But it doesn\u0027t have it. We would need additional queries to determine the port, and whatever lag time that required for it to appear.\n\nAnd we can totally emit stats.","commit_id":"e54a506a53d5fe72979611b533345a287738c79c"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"0a684ed1b327c1307f10d05bddc6d0d8628cfa5f","unresolved":false,"context_lines":[{"line_number":514,"context_line":"    def cleanupLeakedPorts(self):"},{"line_number":515,"context_line":"        if not self._last_port_cleanup:"},{"line_number":516,"context_line":"            self._last_port_cleanup \u003d time.monotonic()"},{"line_number":517,"context_line":"            return"},{"line_number":518,"context_line":""},{"line_number":519,"context_line":"        # Return if not enough time has passed between cleanup"},{"line_number":520,"context_line":"        last_check_in_secs \u003d int(time.monotonic() - self._last_port_cleanup)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_b12b8a99","line":517,"updated":"2018-10-19 19:26:15.000000000","message":"This check will cause us to go through two cleanup intervals before deleting any ports. The first to set _last_port_cleanup and the second to set _down_ports.\n\nWould be nice to set both _last_port_cleanup and _down_ports on the first pass through so that we start cleaning up as quickly as possible.","commit_id":"07e5d5420743918a47b5900fa0e923f5f1370088"},{"author":{"_account_id":3099,"name":"David Shrewsbury","email":"dshrewsb@redhat.com","username":"dshrews"},"change_message_id":"5dc43321538509982875d748cdbfcb70a7f036a0","unresolved":false,"context_lines":[{"line_number":514,"context_line":"    def cleanupLeakedPorts(self):"},{"line_number":515,"context_line":"        if not self._last_port_cleanup:"},{"line_number":516,"context_line":"            self._last_port_cleanup \u003d time.monotonic()"},{"line_number":517,"context_line":"            return"},{"line_number":518,"context_line":""},{"line_number":519,"context_line":"        # Return if not enough time has passed between cleanup"},{"line_number":520,"context_line":"        last_check_in_secs \u003d int(time.monotonic() - self._last_port_cleanup)"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_a940e933","line":517,"in_reply_to":"3f79a3b5_b12b8a99","updated":"2018-10-22 14:44:18.000000000","message":"Done","commit_id":"07e5d5420743918a47b5900fa0e923f5f1370088"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"0a684ed1b327c1307f10d05bddc6d0d8628cfa5f","unresolved":false,"context_lines":[{"line_number":532,"context_line":""},{"line_number":533,"context_line":"        for port in remove_set:"},{"line_number":534,"context_line":"            self.log.debug(\"Removing DOWN port %s\", port[0])"},{"line_number":535,"context_line":"            self.deletePort(port[0])"},{"line_number":536,"context_line":"            self._down_ports \u003d self._down_ports - set([port])"},{"line_number":537,"context_line":""},{"line_number":538,"context_line":"    def cleanupLeakedResources(self):"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_b114ea51","line":535,"updated":"2018-10-19 19:26:15.000000000","message":"If this raises we probably want to keep trying to delete the other ports. Should probably be wrapped in a try: block to do that?","commit_id":"07e5d5420743918a47b5900fa0e923f5f1370088"},{"author":{"_account_id":3099,"name":"David Shrewsbury","email":"dshrewsb@redhat.com","username":"dshrews"},"change_message_id":"5dc43321538509982875d748cdbfcb70a7f036a0","unresolved":false,"context_lines":[{"line_number":532,"context_line":""},{"line_number":533,"context_line":"        for port in remove_set:"},{"line_number":534,"context_line":"            self.log.debug(\"Removing DOWN port %s\", port[0])"},{"line_number":535,"context_line":"            self.deletePort(port[0])"},{"line_number":536,"context_line":"            self._down_ports \u003d self._down_ports - set([port])"},{"line_number":537,"context_line":""},{"line_number":538,"context_line":"    def cleanupLeakedResources(self):"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_09c79d9a","line":535,"in_reply_to":"3f79a3b5_b114ea51","updated":"2018-10-22 14:44:18.000000000","message":"Done","commit_id":"07e5d5420743918a47b5900fa0e923f5f1370088"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"0a684ed1b327c1307f10d05bddc6d0d8628cfa5f","unresolved":false,"context_lines":[{"line_number":533,"context_line":"        for port in remove_set:"},{"line_number":534,"context_line":"            self.log.debug(\"Removing DOWN port %s\", port[0])"},{"line_number":535,"context_line":"            self.deletePort(port[0])"},{"line_number":536,"context_line":"            self._down_ports \u003d self._down_ports - set([port])"},{"line_number":537,"context_line":""},{"line_number":538,"context_line":"    def cleanupLeakedResources(self):"},{"line_number":539,"context_line":"        self.cleanupLeakedInstances()"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_f1282299","line":536,"updated":"2018-10-19 19:26:15.000000000","message":"This assumes that the port is actually deleted. I\u0027m skeptical we can rely on that because yay openstack. Maybe do another self.listPorts(status\u003d\u0027DOWN\u0027) after this loop instead?","commit_id":"07e5d5420743918a47b5900fa0e923f5f1370088"},{"author":{"_account_id":3099,"name":"David Shrewsbury","email":"dshrewsb@redhat.com","username":"dshrews"},"change_message_id":"5dc43321538509982875d748cdbfcb70a7f036a0","unresolved":false,"context_lines":[{"line_number":533,"context_line":"        for port in remove_set:"},{"line_number":534,"context_line":"            self.log.debug(\"Removing DOWN port %s\", port[0])"},{"line_number":535,"context_line":"            self.deletePort(port[0])"},{"line_number":536,"context_line":"            self._down_ports \u003d self._down_ports - set([port])"},{"line_number":537,"context_line":""},{"line_number":538,"context_line":"    def cleanupLeakedResources(self):"},{"line_number":539,"context_line":"        self.cleanupLeakedInstances()"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_090a3dd0","line":536,"in_reply_to":"3f79a3b5_f1282299","updated":"2018-10-22 14:44:18.000000000","message":"Done","commit_id":"07e5d5420743918a47b5900fa0e923f5f1370088"},{"author":{"_account_id":7118,"name":"Ian Wienand","email":"iwienand@redhat.com","username":"iwienand"},"change_message_id":"b45bc7f493c7a7212d26de725369abe21cbe019e","unresolved":false,"context_lines":[{"line_number":528,"context_line":""},{"line_number":529,"context_line":"        for (port_id, port_status) in remove_set:"},{"line_number":530,"context_line":"            try:"},{"line_number":531,"context_line":"                self.deletePort(port_id)"},{"line_number":532,"context_line":"            except Exception:"},{"line_number":533,"context_line":"                self.log.exception(\"Exception deleting port %s:\", port_id)"},{"line_number":534,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_6493afa5","line":531,"updated":"2018-10-23 23:34:39.000000000","message":"I think it\u0027s fairly important we add a stat output at this point (and update our tracing graphs for it after).  We really don\u0027t have a concrete idea of where this is happening, it seems to be a multiple provider issue but we\u0027re not 100% sure.  Seeing this triggering in grafana is going to help I think.","commit_id":"078f97a91bff6999ee0bfdd3b155b3fa7f6bcc33"},{"author":{"_account_id":3099,"name":"David Shrewsbury","email":"dshrewsb@redhat.com","username":"dshrews"},"change_message_id":"0edd19b0847662f2716f097bc3f713e5cd36baa2","unresolved":false,"context_lines":[{"line_number":528,"context_line":""},{"line_number":529,"context_line":"        for (port_id, port_status) in remove_set:"},{"line_number":530,"context_line":"            try:"},{"line_number":531,"context_line":"                self.deletePort(port_id)"},{"line_number":532,"context_line":"            except Exception:"},{"line_number":533,"context_line":"                self.log.exception(\"Exception deleting port %s:\", port_id)"},{"line_number":534,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_92ae029f","line":531,"in_reply_to":"3f79a3b5_6493afa5","updated":"2018-10-26 17:43:28.000000000","message":"Done","commit_id":"078f97a91bff6999ee0bfdd3b155b3fa7f6bcc33"},{"author":{"_account_id":7118,"name":"Ian Wienand","email":"iwienand@redhat.com","username":"iwienand"},"change_message_id":"b45bc7f493c7a7212d26de725369abe21cbe019e","unresolved":false,"context_lines":[{"line_number":532,"context_line":"            except Exception:"},{"line_number":533,"context_line":"                self.log.exception(\"Exception deleting port %s:\", port_id)"},{"line_number":534,"context_line":"            else:"},{"line_number":535,"context_line":"                self.log.debug(\"Removed DOWN port %s\", port_id)"},{"line_number":536,"context_line":""},{"line_number":537,"context_line":"        self._last_port_cleanup \u003d time.monotonic()"},{"line_number":538,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_a4754788","line":535,"updated":"2018-10-23 23:34:39.000000000","message":"I think overall we need a better approach for separating logs out per-provider, because trying to trace things through on something with a lot of providers like on nl04 is a nightmare.\n\nCould we add the provider to this debug message?  it would make it a lot more greppable","commit_id":"078f97a91bff6999ee0bfdd3b155b3fa7f6bcc33"},{"author":{"_account_id":3099,"name":"David Shrewsbury","email":"dshrewsb@redhat.com","username":"dshrews"},"change_message_id":"0edd19b0847662f2716f097bc3f713e5cd36baa2","unresolved":false,"context_lines":[{"line_number":532,"context_line":"            except Exception:"},{"line_number":533,"context_line":"                self.log.exception(\"Exception deleting port %s:\", port_id)"},{"line_number":534,"context_line":"            else:"},{"line_number":535,"context_line":"                self.log.debug(\"Removed DOWN port %s\", port_id)"},{"line_number":536,"context_line":""},{"line_number":537,"context_line":"        self._last_port_cleanup \u003d time.monotonic()"},{"line_number":538,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"3f79a3b5_b2b1be02","line":535,"in_reply_to":"3f79a3b5_a4754788","updated":"2018-10-26 17:43:28.000000000","message":"Done","commit_id":"078f97a91bff6999ee0bfdd3b155b3fa7f6bcc33"},{"author":{"_account_id":2,"name":"Monty Taylor","email":"mordred@inaugust.com","username":"mordred"},"change_message_id":"123092d8c15e7ff6e5d956173bc466b3b52002df","unresolved":false,"context_lines":[{"line_number":516,"context_line":"        if not self._last_port_cleanup:"},{"line_number":517,"context_line":"            self._last_port_cleanup \u003d time.monotonic()"},{"line_number":518,"context_line":"            ports \u003d self.listPorts(status\u003d\u0027DOWN\u0027)"},{"line_number":519,"context_line":"            self._down_ports \u003d set([(p.id, p.status) for p in ports])"},{"line_number":520,"context_line":"            return"},{"line_number":521,"context_line":""},{"line_number":522,"context_line":"        # Return if not enough time has passed between cleanup"}],"source_content_type":"text/x-python","patch_set":6,"id":"3f79a3b5_d6c6a015","line":519,"range":{"start_line":519,"start_character":43,"end_line":519,"end_character":51},"updated":"2018-10-28 14:07:14.000000000","message":"why store status in the set? p.status shoujld always \u003d\u003d \u0027DOWN\u0027 based on the filter.","commit_id":"56987a6fc8087586ecddd57bb2b5b2b64729aba9"},{"author":{"_account_id":2,"name":"Monty Taylor","email":"mordred@inaugust.com","username":"mordred"},"change_message_id":"123092d8c15e7ff6e5d956173bc466b3b52002df","unresolved":false,"context_lines":[{"line_number":525,"context_line":"            return"},{"line_number":526,"context_line":""},{"line_number":527,"context_line":"        ports \u003d self.listPorts(status\u003d\u0027DOWN\u0027)"},{"line_number":528,"context_line":"        current_set \u003d set([(p.id, p.status) for p in ports])"},{"line_number":529,"context_line":"        remove_set \u003d current_set \u0026 self._down_ports"},{"line_number":530,"context_line":""},{"line_number":531,"context_line":"        removed_count \u003d 0"}],"source_content_type":"text/x-python","patch_set":6,"id":"3f79a3b5_76c9cc21","line":528,"range":{"start_line":528,"start_character":33,"end_line":528,"end_character":43},"updated":"2018-10-28 14:07:14.000000000","message":"same","commit_id":"56987a6fc8087586ecddd57bb2b5b2b64729aba9"},{"author":{"_account_id":2,"name":"Monty Taylor","email":"mordred@inaugust.com","username":"mordred"},"change_message_id":"123092d8c15e7ff6e5d956173bc466b3b52002df","unresolved":false,"context_lines":[{"line_number":529,"context_line":"        remove_set \u003d current_set \u0026 self._down_ports"},{"line_number":530,"context_line":""},{"line_number":531,"context_line":"        removed_count \u003d 0"},{"line_number":532,"context_line":"        for (port_id, port_status) in remove_set:"},{"line_number":533,"context_line":"            try:"},{"line_number":534,"context_line":"                self.deletePort(port_id)"},{"line_number":535,"context_line":"            except Exception:"}],"source_content_type":"text/x-python","patch_set":6,"id":"3f79a3b5_96cca833","line":532,"range":{"start_line":532,"start_character":22,"end_line":532,"end_character":33},"updated":"2018-10-28 14:07:14.000000000","message":"same - and port_status isn\u0027t consumed in this loop","commit_id":"56987a6fc8087586ecddd57bb2b5b2b64729aba9"},{"author":{"_account_id":2,"name":"Monty Taylor","email":"mordred@inaugust.com","username":"mordred"},"change_message_id":"123092d8c15e7ff6e5d956173bc466b3b52002df","unresolved":false,"context_lines":[{"line_number":549,"context_line":"        # Rely on OpenStack to tell us the down ports rather than doing our"},{"line_number":550,"context_line":"        # own set adjustment."},{"line_number":551,"context_line":"        ports \u003d self.listPorts(status\u003d\u0027DOWN\u0027)"},{"line_number":552,"context_line":"        self._down_ports \u003d set([(p.id, p.status) for p in ports])"},{"line_number":553,"context_line":""},{"line_number":554,"context_line":"    def cleanupLeakedResources(self):"},{"line_number":555,"context_line":"        self.cleanupLeakedInstances()"}],"source_content_type":"text/x-python","patch_set":6,"id":"3f79a3b5_36d7d4c2","line":552,"range":{"start_line":552,"start_character":39,"end_line":552,"end_character":47},"updated":"2018-10-28 14:07:14.000000000","message":"and same","commit_id":"56987a6fc8087586ecddd57bb2b5b2b64729aba9"},{"author":{"_account_id":16068,"name":"Tobias Henkel","email":"tobias.henkel@bmw.de","username":"tobias.henkel"},"change_message_id":"d2ca53785f8c822890e5a1eaa3f8ca53fe3759f2","unresolved":false,"context_lines":[{"line_number":537,"context_line":"                                   port_id, self.provider.name)"},{"line_number":538,"context_line":"            else:"},{"line_number":539,"context_line":"                removed_count +\u003d 1"},{"line_number":540,"context_line":"                self.log.debug(\"Removed DOWN port %s in %s\","},{"line_number":541,"context_line":"                               port_id, self.provider.name)"},{"line_number":542,"context_line":""},{"line_number":543,"context_line":"        if self._statsd and removed_count:"}],"source_content_type":"text/x-python","patch_set":7,"id":"3f79a3b5_2a43e798","line":540,"range":{"start_line":540,"start_character":25,"end_line":540,"end_character":30},"updated":"2018-10-29 06:35:03.000000000","message":"Maybe this is something we want to have as info or even warning as this might indicate a problem with the cloud?","commit_id":"d88ac00873400386b8229f47cfb7f9b73e1ed6b1"}]}