)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":29775,"name":"Sandeep Yadav","email":"sandyada@redhat.com","username":"sandeepyadav93"},"change_message_id":"68e5fa6fce5607eee45d05daf57cc2d0499af71a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"6719be21_e302c3d2","updated":"2021-11-28 03:33:55.000000000","message":"Discussed with storage squad Abhishek/Rajat:-\n\nCurrent failure is:-      \n      \ntempest log\n \ntempest.exceptions.BuildErrorException: Server 17a76e89-3696-45aa-a06d-d2c71fd62848 failed to build and is in ERROR status\nDetails: {\u0027code\u0027: 500, \u0027created\u0027: \u00272021-11-23T08:59:56Z\u0027, \u0027message\u0027: \u0027Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 17a76e89-3696-45aa-a06d-d2c71fd62848.\u0027}\n \nnova compute logs\n \n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [req-922caa94-af0c-4482-bb3a-783632735e17 b0f6f21a547f48bea3a8e1842ebd2da5 1d5688ac5c3c45f6a662df3845fbb2ff - default default] [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848] Failed to build and run instance: nova.exception.ImageNotAuthorized: Not authorized for image 5191d7db-cb20-4c8e-a6d5-421a9ac740de.\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848] Traceback (most recent call last):\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/image/glance.py\", line 753, in image_import_copy\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     \u0027stores\u0027: stores})\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/image/glance.py\", line 190, in call\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     result \u003d getattr(controller, method)(*args, **kwargs)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/glanceclient/common/utils.py\", line 628, in inner\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     return RequestIdProxy(wrapped(*args, **kwargs))\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/glanceclient/v2/images.py\", line 372, in image_import\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     resp, body \u003d self.http_client.post(url, data\u003ddata, headers\u003dheaders)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/keystoneauth1/adapter.py\", line 401, in post\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     return self.request(url, \u0027POST\u0027, **kwargs)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/glanceclient/common/http.py\", line 380, in request\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     return self._handle_response(resp)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/glanceclient/common/http.py\", line 120, in _handle_response\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     raise exc.from_response(resp, resp.content)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848] glanceclient.exc.HTTPForbidden: HTTP 403 Forbidden: You are not authorized to complete copy_image action.\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848] During handling of the above exception, another exception occurred:\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848] Traceback (most recent call last):\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/compute/manager.py\", line 2402, in _build_and_run_instance\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     accel_info\u003daccel_info)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py\", line 4196, in spawn\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     block_device_info\u003dblock_device_info)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py\", line 4585, in _create_image\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     injection_info, fallback_from_host)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py\", line 4713, in _create_and_inject_local_root\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     instance, size, fallback_from_host)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py\", line 10368, in _try_fetch_image_cache\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     trusted_certs\u003dinstance.trusted_certs)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/imagebackend.py\", line 275, in cache\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     *args, **kwargs)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/imagebackend.py\", line 941, in create_image\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     prepare_template(target\u003dbase, *args, **kwargs)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/oslo_concurrency/lockutils.py\", line 391, in inner\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     return f(*args, **kwargs)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/imagebackend.py\", line 271, in fetch_func_sync\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     fetch_func(target\u003dtarget, *args, **kwargs)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py\", line 4690, in clone_fallback_to_fetch\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     backend.clone(context, disk_images[\u0027image_id\u0027])\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/imagebackend.py\", line 1052, in clone\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     self.copy_to_store(context, image_meta)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/virt/libvirt/imagebackend.py\", line 966, in copy_to_store\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     IMAGE_API.copy_image_to_store(context, image_id, store_name)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/image/glance.py\", line 1307, in copy_image_to_store\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     return session.image_import_copy(context, image_id, [store])\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]   File \"/usr/lib/python3.6/site-packages/nova/image/glance.py\", line 757, in image_import_copy\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]     raise exception.ImageNotAuthorized(image_id\u003dimage_id)\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848] nova.exception.ImageNotAuthorized: Not authorized for image 5191d7db-cb20-4c8e-a6d5-421a9ac740de.\n2021-11-23 08:59:54.841 7 ERROR nova.compute.manager [instance: 17a76e89-3696-45aa-a06d-d2c71fd62848]\n\n\nWe need something like this in policy:-\nhttps://opendev.org/openstack/nova/src/branch/master/playbooks/ceph/glance-copy-policy.yaml\n\n\"copy_image\": \"\"\n\nIn upstream nova-ceph-multistore job we do run this playbook to change the policy runtime to allow non admin users to copy the image","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"7915a8b010240827307b8425f2dd3c95cd7296f2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"107d2caf_50b505d9","updated":"2021-11-12 16:00:20.000000000","message":"I was looking at the failing test test_snapshot_pattern and it has requests to nova (create instance) and glance (create snapshot).\nI couldn\u0027t find anything problematic from glance side but there are a lot of Error logs on nova side saying ImageNotAuthorized error[1].\nI would recommend to verify the request ID of those errors with the request ID tempest used for the nova requests in that test.\n\n[1] https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_5d2/813959/1/check/tripleo-ci-centos-8-scenario001-standalone/5d2390b/logs/undercloud/var/log/containers/nova/nova-compute.log","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"},{"author":{"_account_id":29775,"name":"Sandeep Yadav","email":"sandyada@redhat.com","username":"sandeepyadav93"},"change_message_id":"8cd961bb10f72ba6fda63178620670096ae44456","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"0925dda3_ee43171b","updated":"2021-10-28 10:47:20.000000000","message":"recheck","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"},{"author":{"_account_id":19138,"name":"Pranali Deore","email":"pdeore@redhat.com","username":"PranaliD"},"change_message_id":"f7060e23ddc52fab75dd9fc7a27a02d4d0b88acf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"5eeca1b5_a6e658d4","updated":"2021-10-18 05:26:23.000000000","message":"recheck","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"},{"author":{"_account_id":29775,"name":"Sandeep Yadav","email":"sandyada@redhat.com","username":"sandeepyadav93"},"change_message_id":"a989785b312e46b0fadd350f53617b427c66bd73","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"493a8e30_c06f10f6","updated":"2021-10-14 17:31:39.000000000","message":"recheck\n\n\nFailed on tempest:-\n\nhttps://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_5d2/813959/1/check/tripleo-ci-centos-8-scenario001-standalone/5d2390b/logs/undercloud/var/log/tempest/failing_tests.log\n~~~\ntempest.scenario.test_snapshot_pattern.TestSnapshotPattern.test_snapshot_pattern[compute,id-608e604b-1d63-4a82-8e3e-91bc665c90b4,image,network,slow]\n~~~\n\nTraceback in Glance:-\n\nhttps://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_5d2/813959/1/check/tripleo-ci-centos-8-scenario001-standalone/5d2390b/logs/undercloud/var/log/containers/glance/api.log\n\n~~~\nStderr: \u0027\u0027 _process_cmd /usr/lib/python3.6/site-packages/oslo_privsep/daemon.py:482\nTraceback (most recent call last):\n  File \"/usr/lib/python3.6/site-packages/oslo_privsep/daemon.py\", line 477, in _process_cmd\n    ret \u003d func(*f_args, **f_kwargs)\n  File \"/usr/lib/python3.6/site-packages/oslo_privsep/priv_context.py\", line 274, in _wrap\n    return func(*args, **kwargs)\n  File \"/usr/lib/python3.6/site-packages/os_brick/privileged/rootwrap.py\", line 197, in execute_root\n    return custom_execute(*cmd, shell\u003dFalse, run_as_root\u003dFalse, **kwargs)\n  File \"/usr/lib/python3.6/site-packages/os_brick/privileged/rootwrap.py\", line 146, in custom_execute\n    on_completion\u003don_completion, *cmd, **kwargs)\n  File \"/usr/lib/python3.6/site-packages/oslo_concurrency/processutils.py\", line 441, in execute\n    cmd\u003dsanitized_cmd)\noslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.\n~~~","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"},{"author":{"_account_id":9303,"name":"Abhishek Kekane","email":"akekane@redhat.com","username":"abhishekkekane"},"change_message_id":"d95f3609317ab6497805387de4a302399a744eb3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"48fe5fc6_b8679bc1","updated":"2021-11-23 06:35:58.000000000","message":"recheck to generate logs again","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"817ce97a70e61c263fa0c413a94bfdafd5c52847","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"0d1d6064_42f7e3cc","in_reply_to":"493a8e30_c06f10f6","updated":"2021-11-12 15:14:17.000000000","message":"That is a DEBUG log (not an ERROR) coming from the nvme driver[1] during getting connector info. os-brick traverses through all possible connectors and this was a known issue in nvme connector logging this but i guess the error trace is fixed now. Although it\u0027s nothing fatal and doesn\u0027t break anything as it just says your connector is not nvme.\n\n[1] https://github.com/openstack/os-brick/blob/master/os_brick/initiator/connectors/nvmeof.py#L121-L122","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"}],"ci/environments/scenario001-standalone.yaml":[{"author":{"_account_id":9303,"name":"Abhishek Kekane","email":"akekane@redhat.com","username":"abhishekkekane"},"change_message_id":"a446fe8207945f94786f2a852532e8438f7b94d0","unresolved":true,"context_lines":[{"line_number":127,"context_line":"      CinderRbdAvailabilityZone: ceph3-AZ"},{"line_number":128,"context_line":"  CinderBackupBackend: ceph"},{"line_number":129,"context_line":"  GlanceEnabledImportMethods: \u0027glance-direct,web-download,copy-image\u0027"},{"line_number":130,"context_line":"  GlanceBackend: cinder"},{"line_number":131,"context_line":"  GlanceStoreDescription: \u0027Cinder glance store\u0027"},{"line_number":132,"context_line":"  GlanceMultistoreConfig:"},{"line_number":133,"context_line":"    rbd_store:"}],"source_content_type":"text/x-yaml","patch_set":1,"id":"746711ec_dabdb7a3","line":130,"range":{"start_line":130,"start_character":2,"end_line":130,"end_character":23},"updated":"2021-11-23 13:59:16.000000000","message":"Just to rule out this is cinder related issue, can we change this to use file store?","commit_id":"f719e776acdad2d4210db4c5616e2f8d269424fa"},{"author":{"_account_id":29775,"name":"Sandeep Yadav","email":"sandyada@redhat.com","username":"sandeepyadav93"},"change_message_id":"6bd59bd491505d1ab564ec2c0862a6a6e2c673de","unresolved":true,"context_lines":[{"line_number":127,"context_line":"      CinderRbdAvailabilityZone: ceph3-AZ"},{"line_number":128,"context_line":"  CinderBackupBackend: ceph"},{"line_number":129,"context_line":"  GlanceEnabledImportMethods: \u0027glance-direct,web-download,copy-image\u0027"},{"line_number":130,"context_line":"  GlanceApiPolicies:"},{"line_number":131,"context_line":"    glance-copy_image:"},{"line_number":132,"context_line":"      key: \"copy_image\""},{"line_number":133,"context_line":"      value: \"\""}],"source_content_type":"text/x-yaml","patch_set":2,"id":"53fddf0d_be3de69e","line":130,"updated":"2021-11-29 06:52:35.000000000","message":"imho.. Would be good if we use a similar comment here:\n\nhttps://opendev.org/openstack/nova/src/branch/master/playbooks/ceph/glance-copy-policy.yaml#L10-L13","commit_id":"7dcbdb71b3b10a1c9328ead7ebc6cd840b16853e"},{"author":{"_account_id":29775,"name":"Sandeep Yadav","email":"sandyada@redhat.com","username":"sandeepyadav93"},"change_message_id":"eec02559e071f94ea8968f35528d48f4bdbf1c34","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  GlanceApiPolicies:"},{"line_number":135,"context_line":"    glance-copy_image:"},{"line_number":136,"context_line":"      key: \"copy_image\""},{"line_number":137,"context_line":"      value: \"\""},{"line_number":138,"context_line":"  GlanceBackend: cinder"},{"line_number":139,"context_line":"  GlanceStoreDescription: \u0027Cinder glance store\u0027"},{"line_number":140,"context_line":"  GlanceMultistoreConfig:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"373df992_947d25a4","line":137,"updated":"2021-11-29 07:06:13.000000000","message":"In nova-ceph-multistore job also we run a playbook to change the policy runtime to allow non admin users to copy the image.\n\nhttps://opendev.org/openstack/nova/src/branch/master/playbooks/ceph/glance-copy-policy.yaml#L10-L13","commit_id":"579102e54490d7eba2fdb4403103886a5024d4fd"},{"author":{"_account_id":9303,"name":"Abhishek Kekane","email":"akekane@redhat.com","username":"abhishekkekane"},"change_message_id":"8ce4d20965607446702d3f106e080ea173133cae","unresolved":false,"context_lines":[{"line_number":134,"context_line":"  GlanceApiPolicies:"},{"line_number":135,"context_line":"    glance-copy_image:"},{"line_number":136,"context_line":"      key: \"copy_image\""},{"line_number":137,"context_line":"      value: \"\""},{"line_number":138,"context_line":"  GlanceBackend: cinder"},{"line_number":139,"context_line":"  GlanceStoreDescription: \u0027Cinder glance store\u0027"},{"line_number":140,"context_line":"  GlanceMultistoreConfig:"}],"source_content_type":"text/x-yaml","patch_set":3,"id":"dc875f56_bb46e043","line":137,"in_reply_to":"373df992_947d25a4","updated":"2021-11-29 07:11:09.000000000","message":"++","commit_id":"579102e54490d7eba2fdb4403103886a5024d4fd"}]}