)]}'
{"ironicclient/common/http.py":[{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"e0d02dc548c693656dd9c9667e585148bbad7508","unresolved":false,"context_lines":[{"line_number":119,"context_line":"            else:"},{"line_number":120,"context_line":"                base_version \u003d API_VERSION"},{"line_number":121,"context_line":"            # We could check the response code here. The below method seems to"},{"line_number":122,"context_line":"            # be too simple..."},{"line_number":123,"context_line":"            return self._make_simple_request(conn, \u0027GET\u0027, base_version)"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"        version_overridden \u003d False"}],"source_content_type":"text/x-python","patch_set":1,"id":"7faddb67_6beae30e","line":122,"updated":"2019-08-12 16:26:27.000000000","message":"This would likely be problematic, as this is a pre-flight check, for the instantiation of the library and start of version negotiation. The thing is that can change with an upgrade so better to keep that code at a higher level.","commit_id":"323d4776d2596400158fa54cf7bb1ce94a869abe"},{"author":{"_account_id":14826,"name":"Mark Goddard","email":"markgoddard86@gmail.com","username":"mgoddard"},"change_message_id":"61613072ce32ae27f1d0d8fa6b0d10bd76abb5a8","unresolved":false,"context_lines":[{"line_number":119,"context_line":"            else:"},{"line_number":120,"context_line":"                base_version \u003d API_VERSION"},{"line_number":121,"context_line":"            # We could check the response code here. The below method seems to"},{"line_number":122,"context_line":"            # be too simple..."},{"line_number":123,"context_line":"            return self._make_simple_request(conn, \u0027GET\u0027, base_version)"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"        version_overridden \u003d False"}],"source_content_type":"text/x-python","patch_set":1,"id":"7faddb67_cba0b794","line":122,"in_reply_to":"7faddb67_6beae30e","updated":"2019-08-12 16:39:56.000000000","message":"I\u0027m not suggesting we use the higher level request methods here, only that we actually handle the response code rather than ignoring it.","commit_id":"323d4776d2596400158fa54cf7bb1ce94a869abe"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"e0d02dc548c693656dd9c9667e585148bbad7508","unresolved":false,"context_lines":[{"line_number":166,"context_line":"            # If the query failed for some reason, max_ver will be None."},{"line_number":167,"context_line":"            if not max_ver:"},{"line_number":168,"context_line":"                # Could raise an exception from this library at this point."},{"line_number":169,"context_line":"                pass"},{"line_number":170,"context_line":"        # Reset the maximum version that we permit"},{"line_number":171,"context_line":"        if StrictVersion(max_ver) \u003e StrictVersion(LATEST_VERSION):"},{"line_number":172,"context_line":"            LOG.debug(\"Remote API version %(max_ver)s is greater than the \""}],"source_content_type":"text/x-python","patch_set":1,"id":"7faddb67_cbac37f0","line":169,"updated":"2019-08-12 16:26:27.000000000","message":"Basically we do need to raise an error, however we have to be careful what we raise here because it may impact nova. Well... maybe not actually[0]. I think we ideallly should likely go ahead and error at this point. It would be a breaking change for people trying to connect to a 1.0 server, but... we likely should have been doing that for a while.\n\n\n[0]https://github.com/openstack/nova/blob/master/nova/virt/ironic/driver.py#L636","commit_id":"323d4776d2596400158fa54cf7bb1ce94a869abe"},{"author":{"_account_id":14826,"name":"Mark Goddard","email":"markgoddard86@gmail.com","username":"mgoddard"},"change_message_id":"61613072ce32ae27f1d0d8fa6b0d10bd76abb5a8","unresolved":false,"context_lines":[{"line_number":166,"context_line":"            # If the query failed for some reason, max_ver will be None."},{"line_number":167,"context_line":"            if not max_ver:"},{"line_number":168,"context_line":"                # Could raise an exception from this library at this point."},{"line_number":169,"context_line":"                pass"},{"line_number":170,"context_line":"        # Reset the maximum version that we permit"},{"line_number":171,"context_line":"        if StrictVersion(max_ver) \u003e StrictVersion(LATEST_VERSION):"},{"line_number":172,"context_line":"            LOG.debug(\"Remote API version %(max_ver)s is greater than the \""}],"source_content_type":"text/x-python","patch_set":1,"id":"7faddb67_6ba2e3af","line":169,"in_reply_to":"7faddb67_cbac37f0","updated":"2019-08-12 16:39:56.000000000","message":"Well we could easily differentiate between a 1.0 server and a failure when making the request.","commit_id":"323d4776d2596400158fa54cf7bb1ce94a869abe"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"71f7fba67cb006c096b638bc6432211b41189b1c","unresolved":false,"context_lines":[{"line_number":114,"context_line":"                                str(self.os_ironic_api_version).split(\u0027.\u0027)[0])"},{"line_number":115,"context_line":"            else:"},{"line_number":116,"context_line":"                base_version \u003d API_VERSION"},{"line_number":117,"context_line":"            # Raise exception on client or server error."},{"line_number":118,"context_line":"            resp \u003d self._make_simple_request(conn, \u0027GET\u0027, base_version)"},{"line_number":119,"context_line":"            if not resp.ok:"},{"line_number":120,"context_line":"                raise exc.from_response(resp, method\u003d\u0027GET\u0027, url\u003dbase_version)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5faad753_ebd6c1c3","line":117,"range":{"start_line":117,"start_character":0,"end_line":117,"end_character":56},"updated":"2019-09-11 12:03:19.000000000","message":"I wonder if we\u0027re actually achieving anything here. The /v1 endpoint only fails if ironic is completely down, in which case we don\u0027t get an HTTP code.","commit_id":"62d3d63392d67ffba8d343413f0f59d1d566a8e5"},{"author":{"_account_id":14826,"name":"Mark Goddard","email":"markgoddard86@gmail.com","username":"mgoddard"},"change_message_id":"4c628930872cb6e676f7886c5340c736395ebf43","unresolved":false,"context_lines":[{"line_number":114,"context_line":"                                str(self.os_ironic_api_version).split(\u0027.\u0027)[0])"},{"line_number":115,"context_line":"            else:"},{"line_number":116,"context_line":"                base_version \u003d API_VERSION"},{"line_number":117,"context_line":"            # Raise exception on client or server error."},{"line_number":118,"context_line":"            resp \u003d self._make_simple_request(conn, \u0027GET\u0027, base_version)"},{"line_number":119,"context_line":"            if not resp.ok:"},{"line_number":120,"context_line":"                raise exc.from_response(resp, method\u003d\u0027GET\u0027, url\u003dbase_version)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5faad753_f0ea2daa","line":117,"range":{"start_line":117,"start_character":0,"end_line":117,"end_character":56},"in_reply_to":"5faad753_a68d607c","updated":"2019-09-11 14:18:48.000000000","message":"Turns out there is a nested retry mechanism in inspector - one in the client (max_retries), and one in the dnsmasq pxe_filter which uses call_with_retries.","commit_id":"62d3d63392d67ffba8d343413f0f59d1d566a8e5"},{"author":{"_account_id":14826,"name":"Mark Goddard","email":"markgoddard86@gmail.com","username":"mgoddard"},"change_message_id":"30eb0c9cc6f056bb1ed1f186c2cd780bc22c251a","unresolved":false,"context_lines":[{"line_number":114,"context_line":"                                str(self.os_ironic_api_version).split(\u0027.\u0027)[0])"},{"line_number":115,"context_line":"            else:"},{"line_number":116,"context_line":"                base_version \u003d API_VERSION"},{"line_number":117,"context_line":"            # Raise exception on client or server error."},{"line_number":118,"context_line":"            resp \u003d self._make_simple_request(conn, \u0027GET\u0027, base_version)"},{"line_number":119,"context_line":"            if not resp.ok:"},{"line_number":120,"context_line":"                raise exc.from_response(resp, method\u003d\u0027GET\u0027, url\u003dbase_version)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5faad753_a68d607c","line":117,"range":{"start_line":117,"start_character":0,"end_line":117,"end_character":56},"in_reply_to":"5faad753_ebd6c1c3","updated":"2019-09-11 12:57:57.000000000","message":"That\u0027s a good question. The specific case I hit was with a load balancer in front of ironic API which was up, and giving a 503 because ironic was down.\n\nI tried taking down the load balancer (including keystone) and we fail during endpoint discovery:\n\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service [-] Error starting thread.: DiscoveryFailure: Could not find versioned identity endpoints when attempting to authenticate. Please check that your auth_url is correct. Unable to establish connection to http://10.60.253.190:35357: HTTPConnectionPool(host\u003d\u002710.60.253.190\u0027, port\u003d35357): Max retries exceeded with url: / (Caused by NewConnectionError(\u0027\u003curllib3.connection.HTTPConnection object at 0x7f84e6903a90\u003e: Failed to establish a new connection: [Errno 111] ECONNREFUSED\u0027,))\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service Traceback (most recent call last):\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/oslo_service/service.py\", line 796, in run_service\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     service.start()\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/common/rpc_service.py\", line 40, in start\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     self.manager.init_host()\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/conductor/manager.py\", line 80, in init_host\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     driver.init_filter()\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/pxe_filter/base.py\", line 81, in inner\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     return method(self, *args, **kwargs)\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/pxe_filter/dnsmasq.py\", line 143, in init_filter\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     ironic \u003d ir_utils.get_client()\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/common/ironic.py\", line 133, in get_client\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     session\u003dIRONIC_SESSION).get_endpoint()\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/adapter.py\", line 282, in get_endpoint\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     return self.session.get_endpoint(auth or self.auth, **kwargs)\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/session.py\", line 1200, in get_endpoint\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     return auth.get_endpoint(self, **kwargs)\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/identity/base.py\", line 380, in get_endpoint\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     allow_version_hack\u003dallow_version_hack, **kwargs)\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/identity/base.py\", line 271, in get_endpoint_data\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     service_catalog \u003d self.get_access(session).service_catalog\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/identity/base.py\", line 134, in get_access\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     self.auth_ref \u003d self.get_auth_ref(session)\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/identity/generic/base.py\", line 206, in get_auth_ref\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     self._plugin \u003d self._do_create_plugin(session)\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/identity/generic/base.py\", line 161, in _do_create_plugin\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service     \u0027auth_url is correct. %s\u0027 % e)\n2019-09-11 12:38:37.205 6 ERROR oslo_service.service DiscoveryFailure: Could not find versioned identity endpoints when attempting to authenticate. Please check that your auth_url is correct. Unable to establish connection to http://10.60.253.190:35357: HTTPConnectionPool(host\u003d\u002710.60.253.190\u0027, port\u003d35357): Max retries exceeded with url: / (Caused by NewConnectionError(\u0027\u003curllib3.connection.HTTPConnection object at 0x7f84e6903a90\u003e: Failed to establish a new connection: [Errno 111] ECONNREFUSED\u0027,))\n\nIn this case it claims to have hit max retries, but because we\u0027re not passing in max_retries here it actually returns immediately.\n\n\nI then tried again with keystone active but haproxy not not configured for ironic:\n\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service [-] Error starting thread.: ConnectFailure: Unable to establish connection to http://10.60.253.190:6385/v1: HTTPConnectionPool(host\u003d\u002710.60.253.190\u0027, port\u003d6385): Max retries exceeded with url: /v1 (Caused by NewConnectionError(\u0027\u003curllib3.connection.HTTPConnection object at 0x7f9ad95a7810\u003e: Failed to establish a new connection: [Errno 111] ECONNREFUSED\u0027,))\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service Traceback (most recent call last):\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/oslo_service/service.py\", line 796, in run_service\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     service.start()\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/common/rpc_service.py\", line 40, in start\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     self.manager.init_host()\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/conductor/manager.py\", line 80, in init_host\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     driver.init_filter()\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/pxe_filter/base.py\", line 81, in inner\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     return method(self, *args, **kwargs)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/pxe_filter/dnsmasq.py\", line 144, in init_filter\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     self._sync(ironic)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/pxe_filter/dnsmasq.py\", line 91, in _sync\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     fields\u003d[\u0027address\u0027]))\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/retrying.py\", line 68, in wrapped_f\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     return Retrying(*dargs, **dkw).call(f, *args, **kw)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/retrying.py\", line 223, in call\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     return attempt.get(self._wrap_exception)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/retrying.py\", line 261, in get\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     six.reraise(self.value[0], self.value[1], self.value[2])\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/retrying.py\", line 217, in call\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     attempt \u003d Attempt(fn(*args, **kwargs), attempt_number, False)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironic_inspector/common/ironic.py\", line 209, in call_with_retries\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     return func(*args, **kwargs)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/v1/port.py\", line 101, in list\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     limit\u003dlimit)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/common/base.py\", line 162, in _list_pagination\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     resp, body \u003d self.api.json_request(\u0027GET\u0027, url, **kwargs)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/common/http.py\", line 683, in json_request\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     resp \u003d self._http_request(url, method, **kwargs)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/common/http.py\", line 291, in wrapper\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     return func(self, url, method, **kwargs)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/common/http.py\", line 639, in _http_request\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     self.negotiate_version(self.session, None)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/common/http.py\", line 149, in negotiate_version\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     resp \u003d _query_server(conn)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/common/http.py\", line 121, in _query_server\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     return self._make_simple_request(conn, \u0027GET\u0027, base_version)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/ironicclient/common/http.py\", line 631, in _make_simple_request\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     endpoint_override\u003dself.endpoint_override)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/session.py\", line 888, in request\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     resp \u003d send(**kwargs)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service   File \"/usr/lib/python2.7/site-packages/keystoneauth1/session.py\", line 995, in _send_request\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service     raise exceptions.ConnectFailure(msg)\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service ConnectFailure: Unable to establish connection to http://10.60.253.190:6385/v1: HTTPConnectionPool(host\u003d\u002710.60.253.190\u0027, port\u003d6385): Max retries exceeded with url: /v1 (Caused by NewConnectionError(\u0027\u003curllib3.connection.HTTPConnection object at 0x7f9ad95a7810\u003e: Failed to establish a new connection: [Errno 111] ECONNREFUSED\u0027,))\n2019-09-11 12:43:12.508 6 ERROR oslo_service.service\n\nIt seems to use the retry mechanism, and times out after 60 seconds.","commit_id":"62d3d63392d67ffba8d343413f0f59d1d566a8e5"}]}