)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"733e920af9a5d21a38614835d87cc9ca29babe17","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":1,"id":"eb6d58a9_f2407bdc","updated":"2022-03-01 07:41:55.000000000","message":"The patch itself lgtm, if you want to backport this change, I think it would be better to add a link to a bug report.\n\nI\u0027m also concerned about this failing suddenly in a minor package upgrade without prior deprecation warning. This undermines the impression that c9s is too unstable for us to really be useful.\n\nLast but not least it would be helpful if someone would make the centos jobs passing again, likely by skipping the failing tests until nova gets fixed.","commit_id":"f003197964c8355ec406f06a09e452f8160918ad"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"f3e8f20c8dc35deb72889e1e713fc37579ddb98d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"9ee4fe78_04606494","in_reply_to":"eb6d58a9_f2407bdc","updated":"2022-03-01 19:48:46.000000000","message":"Done","commit_id":"f003197964c8355ec406f06a09e452f8160918ad"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"280596c1a6f91a827ee5bacfc654badc496aa1d2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"e9a8dc9e_78479c97","updated":"2022-03-11 15:27:53.000000000","message":"Latest failure is a whole job timeout due to waiting for volumes to be cleaned up, but I don\u0027t see anything obvious in cinder logs that indicates a problem. Still can\u0027t see how this could be related to the sha1 change, but also agree this seems to be one of the only places this job is failing...","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"3e42bd165cfe99d98cd589780f06448b28736523","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"293a985d_34f82278","updated":"2022-03-07 09:19:13.000000000","message":"The last five failures of nova-ceph-multistore were all on this patch. I still have no idea why this could happen but the numbers look too significant for me to simply ignore this.\n\nhttps://zuul.opendev.org/t/openstack/builds?job_name\u003dnova-ceph-multistore\u0026project\u003dopenstack/devstack","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"2244d603f7b5c1302c73be9c084f6e3ea2deabc3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"ea296ec6_614b9f48","updated":"2022-03-09 08:33:58.000000000","message":"also faced while running a job with c9","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"e9e14c93ef0fe9885cf7c1c261e9adbb8803d787","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"3ffd966c_282140c1","updated":"2022-03-03 12:55:07.000000000","message":"recheck","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"cd437eeba0688f16b007d877c8ee5a67c4d54925","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"fa1172e9_392e1f8c","updated":"2022-03-04 10:08:37.000000000","message":"recheck","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"34e4a5900bd299954382d5100ec849f9c99c3c86","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"fbcd9807_228ed4c9","updated":"2022-03-08 15:21:27.000000000","message":"recheck","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":11628,"name":"Michael Johnson","email":"johnsomor@gmail.com","username":"johnsom"},"change_message_id":"f3e8f20c8dc35deb72889e1e713fc37579ddb98d","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"e51d0416_a59ce05c","updated":"2022-03-01 19:48:46.000000000","message":"recheck\nssh timeout","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"30a9938e46cef92d05ef086f432300b732768ac2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"5d96ee8d_6a90a73f","updated":"2022-03-11 02:05:06.000000000","message":"recheck collect some data","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"309e98314c168524a637d9b180c2c364c3b76a3a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"958e7da0_6897395a","updated":"2022-03-11 02:07:01.000000000","message":"recheck collect some data ... please?","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":22623,"name":"Erik Olof Gunnar Andersson","email":"eandersson@blizzard.com","username":"eoandersson"},"change_message_id":"71828dc1688db10c02e31a007b2c7cf9525ecf9e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"3ce3332c_8c59488f","updated":"2022-03-11 07:11:39.000000000","message":"recheck more data","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"0095c2d35539897cdc488d26f991c62748e94945","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"8d06aeed_e900b765","updated":"2022-03-03 21:15:06.000000000","message":"recheck nova-ceph-multistore seems to be failing pretty often for this patch. I don\u0027t see how this could be related, but would be good if we could get it more stable or else make it n-v","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"e0545a4314c3e55fbb6bf620b37e576d1890ef70","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"33d87684_e83910fa","updated":"2022-03-02 11:59:16.000000000","message":"recheck pypy issue?","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"30a9938e46cef92d05ef086f432300b732768ac2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"9fe1ae69_0ba9060e","in_reply_to":"293a985d_34f82278","updated":"2022-03-11 02:05:06.000000000","message":"\u003e The last five failures of nova-ceph-multistore were all on this patch. I still have no idea why this could happen but the numbers look too significant for me to simply ignore this.\n\nI have an interest in the nova-ceph-multistore job working so I was taking a look here. The N-2 and N-1 failures were caused by mysql going OOM and thus everything after that failing with mysql connection failures in the logs. I doubt that\u0027s related to this patch.\n\nThe N-0 failure looks to be a hard lockup of nova-compute, the likes of which we rarely see. It freezes mid-stream after \"Creating image\" and never utters another log message, not even periodics. That usually is only something like a blocking library call or something, or more often something like a disk-exhaustion sort of thing (which this isn\u0027t). After that step, I think we make some ceph calls (I see no relevant errors there) and some calls to glance (which involve the tls proxy). But, no idea why we\u0027d succeed a thousand times and hard-freeze this time.\n\nI\u0027d like to recheck this maybe a couple more times to see what other data we might collect. I\u0027ll throw a -W on it for the time being.","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e413bea7b1a77cc57f87accbee2f93da2f27ce34","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":2,"id":"68c1fcca_62f3fb9e","in_reply_to":"d85a70c8_1101664a","updated":"2022-03-11 21:01:33.000000000","message":"Yeah. Let\u0027s be sure to do a few rechecks (with good results) before we merge. Given the one hang around the time where we\u0027d be talking to glance and other weirdness I was wondering if maybe we\u0027ve got a client issue? Something that causes us to get stuck occasionally loading the cert with the larger key or something...","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":22623,"name":"Erik Olof Gunnar Andersson","email":"eandersson@blizzard.com","username":"eoandersson"},"change_message_id":"b8053345187ac7edff3d85a7b3f0e064a6845bb4","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":2,"id":"d85a70c8_1101664a","in_reply_to":"e9a8dc9e_78479c97","updated":"2022-03-11 20:28:57.000000000","message":"If changing this to sha256 is causing issues we probably have bigger problems somewhere else as well. I rebased it just to make sure we aren\u0027t missing something.","commit_id":"d0ea02bfc0f85728657eb734f906ce5ad274f1bf"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"121cbe95965189062c8f97ebfa8b5287b0df427b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"7bb264fe_0e8b251d","updated":"2022-03-14 15:53:57.000000000","message":"Well, I dunno why a rebase might have affected this, but recent test runs seem happy. The content of the patch wasn\u0027t really relevant to the failures we were seeing anyway, so it\u0027s hard to say much about it either way.\n\nI think we know this should be a safe thing to do, and now we have some evidence that it\u0027s not super fail-happy (at the moment at least). I\u0027d vote for putting this in and letting it soak in the firehose at this point. I\u0027ll +2 but let someone else +W if they agree.","commit_id":"35bc600da17c7342345fa9c4d0b8078a8388fad1"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"632b802e6d8210405b3bcff2789b1a31b57c2ae4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"1e1717cc_6e729c4e","updated":"2022-03-15 14:32:57.000000000","message":"lgtm, let\u0027s merge, if this will cause an issue somewhere,  it would need to be addressed sooner or later anyway","commit_id":"35bc600da17c7342345fa9c4d0b8078a8388fad1"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"267eaee464b4fdb64a1d3f0191c10d1187d2e3a5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"4c88b6f6_5e8e8e84","updated":"2022-03-14 13:43:39.000000000","message":"recheck","commit_id":"35bc600da17c7342345fa9c4d0b8078a8388fad1"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"617285e17413094a425b423525a0184c297a2d41","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"5543c42d_2801c1e9","updated":"2022-03-12 07:42:58.000000000","message":"recheck","commit_id":"35bc600da17c7342345fa9c4d0b8078a8388fad1"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"0438d54d77270037820e1a0a88735bb88235d46e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"7e28b156_839bf5e4","updated":"2022-03-12 20:02:22.000000000","message":"recheck","commit_id":"35bc600da17c7342345fa9c4d0b8078a8388fad1"},{"author":{"_account_id":22873,"name":"Martin Kopec","email":"mkopec@redhat.com","username":"mkopec"},"change_message_id":"632b802e6d8210405b3bcff2789b1a31b57c2ae4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"1d86e0d8_8abb62b6","in_reply_to":"7bb264fe_0e8b251d","updated":"2022-03-15 14:32:57.000000000","message":"I agree, thank you Dan for testing it out and chasing the errors.","commit_id":"35bc600da17c7342345fa9c4d0b8078a8388fad1"}]}