)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"b896648edfb9c2444cf9d799cfec70c65e42fa86","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"07755122_68347bc2","updated":"2022-02-01 17:16:21.000000000","message":"Looks good! Thanks!","commit_id":"a52183a3df4b9745eb46785b3b449a327589aec0"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"0c0d2f1db1118eeb07005c7a9fd35bcc4fc6c5bf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"1409eaba_93207150","updated":"2022-02-02 16:32:45.000000000","message":"Still seeing traces in nova-multi-cell run:\n\nhttps://zuul.opendev.org/t/openstack/build/2ffb2c2c1aff450e803264fe93bb77c8/log/controller/logs/screen-n-cpu.txt\nhttps://zuul.opendev.org/t/openstack/build/2ffb2c2c1aff450e803264fe93bb77c8/log/compute1/logs/screen-n-cpu.txt\n\nLooks like InstanceNotFound is not handled by the code initiating the info cache save. Going to look at this more.","commit_id":"a52183a3df4b9745eb46785b3b449a327589aec0"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"a1096e77bc47abe77453daeaaf5510b02282b66d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"eceaaf63_d1904273","updated":"2022-02-01 17:16:34.000000000","message":"recheck both failed job is unrelated","commit_id":"a52183a3df4b9745eb46785b3b449a327589aec0"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"5efa8452288a35f3be3f997439456d183857d9e1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"12274674_9b1ff11c","updated":"2022-02-03 03:45:56.000000000","message":"OK, so the red ERROR level tracebacks are gone, but I\u0027m still seeing (debug?) tracebacks. Not sure what\u0027s going on.\n\nExample:\n\nhttps://zuul.opendev.org/t/openstack/build/7c3c755868b848ed841d37457004d99b/log/controller/logs/screen-n-cpu.txt?severity\u003d0#10027","commit_id":"ee5e48f3c9a2e77dcfcb4a975b4b524aa3b23a43"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"fecfe8cb10c50acf5426f7b11b30dfc331861548","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"b20ec92a_fa7f8b8a","in_reply_to":"12274674_9b1ff11c","updated":"2022-02-03 11:23:04.000000000","message":"The InstanceNotFound is logged by the RPC layer as far as I see. And that exception is chained to the original DB exception hence the multiple backtrace. If you want to suppress the InstanceNotFound in the RPC layer then I think you need to decorate the InstanceInfoCache.save() with\n\n   @messaging.expected_exceptions(exception.InstanceNotFound)","commit_id":"ee5e48f3c9a2e77dcfcb4a975b4b524aa3b23a43"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"8cb53649748406fa390b898bd37f00ffbaba47e3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"a2ba745b_f3de60fc","updated":"2022-02-03 22:52:37.000000000","message":"Now I\u0027m getting [1]:\n\n oslo_messaging.rpc.client.RemoteError: Remote error: ExpectedException\n\nand I\u0027m guessing it\u0027s because the @messaging.expected_exceptions() decorator is executing on wrong (client) side? Aughhhhhh\n\n[1] https://zuul.opendev.org/t/openstack/build/2976279355174469ad9e8d633775fae2/log/controller/logs/screen-n-cpu.txt#9487","commit_id":"1f7632a4403ee095b62450a6da5462db7ed2c946"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"ae1c42022fd9ec01c8560aadee1f406343edf276","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"c06dad9b_52def624","in_reply_to":"a2ba745b_f3de60fc","updated":"2022-02-03 23:56:47.000000000","message":"Actually it looks like it is executing on the conductor side as wanted. But we\u0027re getting the oslo.messaging ExpectedException back and it\u0027s not being handled? I thought ExpectedException is supposed to get translated to the original exception when it replies? I must be missing something.","commit_id":"1f7632a4403ee095b62450a6da5462db7ed2c946"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e9656dbe2c8863c4f967a1cccdab167ce07b0e75","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"f1b6b2a5_0f83e16e","in_reply_to":"c06dad9b_52def624","updated":"2022-02-04 10:33:06.000000000","message":"Maybe it is different because we are calling remotable ovo methods. I only see this decorator on compute rpc methods but those are pure rpc not ovo. I\u0027m lost. Maybe Dan can help.","commit_id":"1f7632a4403ee095b62450a6da5462db7ed2c946"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"bba1d8cbf5f9deb2a6d286a728cb333fe1103cdf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"564f5986_5a4a093f","updated":"2022-02-05 17:02:48.000000000","message":"Finally got it working (!!) no more “foreign key” mega traces in screen-n-cpu.txt","commit_id":"103aa4f305452d7da404d123c1487bcffd26df2a"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"db606b1e4fda3b580d2c57cbeb04268b2657566a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"24ca8295_9564aae4","updated":"2022-02-07 10:20:51.000000000","message":"I have one request inline but otherwise looks OK.","commit_id":"103aa4f305452d7da404d123c1487bcffd26df2a"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"7c5cdf0f289c02ee0d5f465d34f43cd72186f9a2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":10,"id":"4cbe5bb1_4034acc4","updated":"2022-02-21 18:59:03.000000000","message":"Nice work. Took me a bit to grok but (mostly) makes sense :)","commit_id":"ecb5f1717191379b5ade160ae814f7da8b91e7e8"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"9ad94fec85b81613482a1f3da67152ec17fcfa69","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":10,"id":"3f1169bd_86480a89","updated":"2022-02-08 11:44:22.000000000","message":"Thanks! Looks goo","commit_id":"ecb5f1717191379b5ade160ae814f7da8b91e7e8"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"e9b1be6d1a92389927e5f12bb1e3e4b2e9715278","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":10,"id":"104e70b2_cffff735","in_reply_to":"3f1169bd_86480a89","updated":"2022-02-08 11:44:36.000000000","message":"good even :)","commit_id":"ecb5f1717191379b5ade160ae814f7da8b91e7e8"}],"nova/compute/manager.py":[{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"56c34872581d802ddbc87278dfbc03ef737c5525","unresolved":true,"context_lines":[{"line_number":10513,"context_line":"                return"},{"line_number":10514,"context_line":"        do_power_update()"},{"line_number":10515,"context_line":""},{"line_number":10516,"context_line":"    @wrap_exception()"},{"line_number":10517,"context_line":"    def external_instance_event(self, context, instances, events):"},{"line_number":10518,"context_line":"        # NOTE(danms): Some event types are handled by the manager, such"},{"line_number":10519,"context_line":"        # as when we\u0027re asked to update the instance\u0027s info_cache. If it\u0027s"}],"source_content_type":"text/x-python","patch_set":1,"id":"bcf294b3_51aaea3e","line":10516,"updated":"2022-02-02 18:29:28.000000000","message":"Realizing the traceback \"nova.exception_Remote.InstanceNotFound_Remote: Instance \u003cuuid\u003e\" is a remote exception coming from nova-conductor, I might need to add @messaging.expected_exceptions() with InstanceNotFound to this method. Going to try it.","commit_id":"a52183a3df4b9745eb46785b3b449a327589aec0"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"71d34bd505c9e7941a59b532e58cac2ed5d6163e","unresolved":false,"context_lines":[{"line_number":10513,"context_line":"                return"},{"line_number":10514,"context_line":"        do_power_update()"},{"line_number":10515,"context_line":""},{"line_number":10516,"context_line":"    @wrap_exception()"},{"line_number":10517,"context_line":"    def external_instance_event(self, context, instances, events):"},{"line_number":10518,"context_line":"        # NOTE(danms): Some event types are handled by the manager, such"},{"line_number":10519,"context_line":"        # as when we\u0027re asked to update the instance\u0027s info_cache. If it\u0027s"}],"source_content_type":"text/x-python","patch_set":1,"id":"e7cfe057_7324bf1d","line":10516,"in_reply_to":"bcf294b3_51aaea3e","updated":"2022-02-02 22:17:33.000000000","message":"That was not it.","commit_id":"a52183a3df4b9745eb46785b3b449a327589aec0"}],"nova/network/neutron.py":[{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"95698af845d7a1a2fd384b92bc94d99ec48eee4e","unresolved":true,"context_lines":[{"line_number":129,"context_line":"        # receive an external event from neutron. Avoid logging a traceback"},{"line_number":130,"context_line":"        # when it happens."},{"line_number":131,"context_line":"        msg \u003d str(e)"},{"line_number":132,"context_line":"        if e.__class__.__name__.endswith(\u0027_Remote\u0027):"},{"line_number":133,"context_line":"            # If this exception was raised remotely over RPC, the traceback(s)"},{"line_number":134,"context_line":"            # will be appended to the message. Truncate it in that case."},{"line_number":135,"context_line":"            msg \u003d utils.safe_truncate(msg.split(\u0027\\n\u0027, 1)[0], 255)"}],"source_content_type":"text/x-python","patch_set":9,"id":"cc85d2bc_79dc2932","line":132,"updated":"2022-02-05 17:18:37.000000000","message":"Note to reviewers: this is taken from oslo.messaging doc:\nhttps://docs.openstack.org/oslo.messaging/latest/reference/rpcclient.html#oslo_messaging.RPCClient.call","commit_id":"103aa4f305452d7da404d123c1487bcffd26df2a"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"db606b1e4fda3b580d2c57cbeb04268b2657566a","unresolved":true,"context_lines":[{"line_number":133,"context_line":"            # If this exception was raised remotely over RPC, the traceback(s)"},{"line_number":134,"context_line":"            # will be appended to the message. Truncate it in that case."},{"line_number":135,"context_line":"            msg \u003d utils.safe_truncate(msg.split(\u0027\\n\u0027, 1)[0], 255)"},{"line_number":136,"context_line":"        LOG.info(\u0027Failed storing info cache due to: %s\u0027, msg,"},{"line_number":137,"context_line":"                 instance\u003dinstance)"},{"line_number":138,"context_line":"        raise exception.InstanceNotFound(message\u003dmsg)"},{"line_number":139,"context_line":"    except Exception:"}],"source_content_type":"text/x-python","patch_set":9,"id":"35c0e221_4a2aabfc","line":136,"updated":"2022-02-07 10:20:51.000000000","message":"I would be a bit more specific here in the log. We know that the instance is moved cross cell, so we can mention that in the log as the reason.","commit_id":"103aa4f305452d7da404d123c1487bcffd26df2a"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"5aced28881ed22bfa6d68503a015f2ea36e708f0","unresolved":true,"context_lines":[{"line_number":133,"context_line":"            # If this exception was raised remotely over RPC, the traceback(s)"},{"line_number":134,"context_line":"            # will be appended to the message. Truncate it in that case."},{"line_number":135,"context_line":"            msg \u003d utils.safe_truncate(msg.split(\u0027\\n\u0027, 1)[0], 255)"},{"line_number":136,"context_line":"        LOG.info(\u0027Failed storing info cache due to: %s\u0027, msg,"},{"line_number":137,"context_line":"                 instance\u003dinstance)"},{"line_number":138,"context_line":"        raise exception.InstanceNotFound(message\u003dmsg)"},{"line_number":139,"context_line":"    except Exception:"}],"source_content_type":"text/x-python","patch_set":9,"id":"8b392038_9d78633f","line":136,"in_reply_to":"35c0e221_4a2aabfc","updated":"2022-02-07 16:13:08.000000000","message":"I was wary of doing that for worry of being misleading if it happens in the future due to some other bug or condition (saying it\u0027s because of a cross-cell migration when it wasn\u0027t). Maybe if I add to the message, \"The instance may have moved to another cell during a cross-cell resize or migration.\"?","commit_id":"103aa4f305452d7da404d123c1487bcffd26df2a"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"3803f035656ce60dd27fc10fc79320837d68db04","unresolved":false,"context_lines":[{"line_number":133,"context_line":"            # If this exception was raised remotely over RPC, the traceback(s)"},{"line_number":134,"context_line":"            # will be appended to the message. Truncate it in that case."},{"line_number":135,"context_line":"            msg \u003d utils.safe_truncate(msg.split(\u0027\\n\u0027, 1)[0], 255)"},{"line_number":136,"context_line":"        LOG.info(\u0027Failed storing info cache due to: %s\u0027, msg,"},{"line_number":137,"context_line":"                 instance\u003dinstance)"},{"line_number":138,"context_line":"        raise exception.InstanceNotFound(message\u003dmsg)"},{"line_number":139,"context_line":"    except Exception:"}],"source_content_type":"text/x-python","patch_set":9,"id":"bac0d633_4ff974fc","line":136,"in_reply_to":"8b392038_9d78633f","updated":"2022-02-07 20:13:32.000000000","message":"Done","commit_id":"103aa4f305452d7da404d123c1487bcffd26df2a"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"9ad94fec85b81613482a1f3da67152ec17fcfa69","unresolved":false,"context_lines":[{"line_number":133,"context_line":"            # If this exception was raised remotely over RPC, the traceback(s)"},{"line_number":134,"context_line":"            # will be appended to the message. Truncate it in that case."},{"line_number":135,"context_line":"            msg \u003d utils.safe_truncate(msg.split(\u0027\\n\u0027, 1)[0], 255)"},{"line_number":136,"context_line":"        LOG.info(\u0027Failed storing info cache due to: %s\u0027, msg,"},{"line_number":137,"context_line":"                 instance\u003dinstance)"},{"line_number":138,"context_line":"        raise exception.InstanceNotFound(message\u003dmsg)"},{"line_number":139,"context_line":"    except Exception:"}],"source_content_type":"text/x-python","patch_set":9,"id":"40537a9a_cc32e824","line":136,"in_reply_to":"bac0d633_4ff974fc","updated":"2022-02-08 11:44:22.000000000","message":"\"maybe\" sounds good. at least it hints the troubleshooter where to look next.","commit_id":"103aa4f305452d7da404d123c1487bcffd26df2a"}]}