)]}'
{".zuul.yaml":[{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"c0a5e44cf1e8e84bec65524d88f258906f1f0ac4","unresolved":true,"context_lines":[{"line_number":728,"context_line":"        # from source instead."},{"line_number":729,"context_line":"        OVN_BUILD_FROM_SOURCE: True"},{"line_number":730,"context_line":""},{"line_number":731,"context_line":"- job:"},{"line_number":732,"context_line":"    name: devstack-platform-ubuntu-jammy"},{"line_number":733,"context_line":"    parent: tempest-full-py3"},{"line_number":734,"context_line":"    description: Ubuntu 22.04 LTS (jammy) platform test"}],"source_content_type":"text/x-yaml","patch_set":4,"id":"78791656_34d94ccd","line":731,"updated":"2022-04-28 05:02:00.000000000","message":"let\u0027s also run ovn along with logs to see what\u0027s the issue there, will update the patch.","commit_id":"77feddc83ba4c40e630360160e9a806222ff2908"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"c0a5e44cf1e8e84bec65524d88f258906f1f0ac4","unresolved":true,"context_lines":[{"line_number":756,"context_line":"        q-l3: true"},{"line_number":757,"context_line":"        q-meta: true"},{"line_number":758,"context_line":"        q-metering: true"},{"line_number":759,"context_line":"    group-vars:"},{"line_number":760,"context_line":"      subnode:"},{"line_number":761,"context_line":"        devstack_services:"},{"line_number":762,"context_line":"          # Disable OVN services"},{"line_number":763,"context_line":"          ovn-controller: false"},{"line_number":764,"context_line":"          ovs-vswitchd: false"},{"line_number":765,"context_line":"          ovsdb-server: false"},{"line_number":766,"context_line":"          # Disable Neutron ML2/OVN services"},{"line_number":767,"context_line":"          q-ovn-metadata-agent: false"},{"line_number":768,"context_line":"          # Enable Neutron ML2/OVS services"},{"line_number":769,"context_line":"          q-agt: true"},{"line_number":770,"context_line":""},{"line_number":771,"context_line":"- job:"},{"line_number":772,"context_line":"    name: devstack-no-tls-proxy"}],"source_content_type":"text/x-yaml","patch_set":4,"id":"a5a551e0_28844f38","line":769,"range":{"start_line":759,"start_character":1,"end_line":769,"end_character":21},"updated":"2022-04-28 05:02:00.000000000","message":"shouldn\u0027t be needed as single node job.","commit_id":"77feddc83ba4c40e630360160e9a806222ff2908"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"6ce0abcf50c2ab13feb90035c4dd5bc5874792aa","unresolved":true,"context_lines":[{"line_number":756,"context_line":"        q-l3: true"},{"line_number":757,"context_line":"        q-meta: true"},{"line_number":758,"context_line":"        q-metering: true"},{"line_number":759,"context_line":"    group-vars:"},{"line_number":760,"context_line":"      subnode:"},{"line_number":761,"context_line":"        devstack_services:"},{"line_number":762,"context_line":"          # Disable OVN services"},{"line_number":763,"context_line":"          ovn-controller: false"},{"line_number":764,"context_line":"          ovs-vswitchd: false"},{"line_number":765,"context_line":"          ovsdb-server: false"},{"line_number":766,"context_line":"          # Disable Neutron ML2/OVN services"},{"line_number":767,"context_line":"          q-ovn-metadata-agent: false"},{"line_number":768,"context_line":"          # Enable Neutron ML2/OVS services"},{"line_number":769,"context_line":"          q-agt: true"},{"line_number":770,"context_line":""},{"line_number":771,"context_line":"- job:"},{"line_number":772,"context_line":"    name: devstack-no-tls-proxy"}],"source_content_type":"text/x-yaml","patch_set":4,"id":"fedad7cb_66844e7a","line":769,"range":{"start_line":759,"start_character":1,"end_line":769,"end_character":21},"in_reply_to":"9fb2ae0d_8df35b2c","updated":"2022-04-28 14:21:29.000000000","message":"okk","commit_id":"77feddc83ba4c40e630360160e9a806222ff2908"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"724a5ae7df3e43f0350607f94df7128dad09bf11","unresolved":true,"context_lines":[{"line_number":756,"context_line":"        q-l3: true"},{"line_number":757,"context_line":"        q-meta: true"},{"line_number":758,"context_line":"        q-metering: true"},{"line_number":759,"context_line":"    group-vars:"},{"line_number":760,"context_line":"      subnode:"},{"line_number":761,"context_line":"        devstack_services:"},{"line_number":762,"context_line":"          # Disable OVN services"},{"line_number":763,"context_line":"          ovn-controller: false"},{"line_number":764,"context_line":"          ovs-vswitchd: false"},{"line_number":765,"context_line":"          ovsdb-server: false"},{"line_number":766,"context_line":"          # Disable Neutron ML2/OVN services"},{"line_number":767,"context_line":"          q-ovn-metadata-agent: false"},{"line_number":768,"context_line":"          # Enable Neutron ML2/OVS services"},{"line_number":769,"context_line":"          q-agt: true"},{"line_number":770,"context_line":""},{"line_number":771,"context_line":"- job:"},{"line_number":772,"context_line":"    name: devstack-no-tls-proxy"}],"source_content_type":"text/x-yaml","patch_set":4,"id":"9fb2ae0d_8df35b2c","line":769,"range":{"start_line":759,"start_character":1,"end_line":769,"end_character":21},"in_reply_to":"a5a551e0_28844f38","updated":"2022-04-28 06:08:33.000000000","message":"I just copied this whole block from the bullseye job, it isn\u0027t needed for a single node, but maybe people want to make a multinode variant?\n\nAnyway, this isn\u0027t expected to get merged in this state, I was just testing whether the neutron failures were related to OVN or whether there might be some deeper issue.","commit_id":"77feddc83ba4c40e630360160e9a806222ff2908"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"724a5ae7df3e43f0350607f94df7128dad09bf11","unresolved":true,"context_lines":[{"line_number":743,"context_line":"        \u0027{{ devstack_base_dir }}/data/ovs\u0027: \u0027logs\u0027"},{"line_number":744,"context_line":"        \u0027{{ devstack_base_dir }}/data/ovn\u0027: \u0027logs\u0027"},{"line_number":745,"context_line":"        \u0027{{ devstack_log_dir }}/ovsdb-server-nb.log\u0027: \u0027logs\u0027"},{"line_number":746,"context_line":"        \u0027{{ devstack_log_dir }}/ovsdb-server-sb.log\u0027: \u0027logs\u0027"},{"line_number":747,"context_line":""},{"line_number":748,"context_line":"- job:"},{"line_number":749,"context_line":"    name: devstack-platform-ubuntu-jammy-ovn-source"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"63a5eb06_533f4b40","line":746,"updated":"2022-04-28 06:08:33.000000000","message":"Is this expected to be a lot of output or why don\u0027t we collect this by default?","commit_id":"6c708d4acce35c54838446429d7f628e86ffc7ea"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"6ce0abcf50c2ab13feb90035c4dd5bc5874792aa","unresolved":true,"context_lines":[{"line_number":743,"context_line":"        \u0027{{ devstack_base_dir }}/data/ovs\u0027: \u0027logs\u0027"},{"line_number":744,"context_line":"        \u0027{{ devstack_base_dir }}/data/ovn\u0027: \u0027logs\u0027"},{"line_number":745,"context_line":"        \u0027{{ devstack_log_dir }}/ovsdb-server-nb.log\u0027: \u0027logs\u0027"},{"line_number":746,"context_line":"        \u0027{{ devstack_log_dir }}/ovsdb-server-sb.log\u0027: \u0027logs\u0027"},{"line_number":747,"context_line":""},{"line_number":748,"context_line":"- job:"},{"line_number":749,"context_line":"    name: devstack-platform-ubuntu-jammy-ovn-source"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"e898de9e_d8e10aa0","line":746,"in_reply_to":"63a5eb06_533f4b40","updated":"2022-04-28 14:21:29.000000000","message":"as far as i know it\u0027s not much atleast when debug is not enabled. also these log path are relevant when ovs/ovn are deployed from source, with distro packages path are different. pushing update to patch for this.","commit_id":"6c708d4acce35c54838446429d7f628e86ffc7ea"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"a04db5c1b76c076d57df1c397b88bea447a8bdb6","unresolved":true,"context_lines":[{"line_number":911,"context_line":"    check:"},{"line_number":912,"context_line":"      jobs:"},{"line_number":913,"context_line":"        # Commented green ones"},{"line_number":914,"context_line":"        #- devstack-platform-ubuntu-jammy"},{"line_number":915,"context_line":"        - devstack-platform-ubuntu-jammy-ovn"},{"line_number":916,"context_line":"        #- devstack-platform-ubuntu-jammy-ovn-source"},{"line_number":917,"context_line":"        #- devstack-platform-ubuntu-jammy-ovn-source-master"},{"line_number":918,"context_line":"    gate:"},{"line_number":919,"context_line":"      jobs:"},{"line_number":920,"context_line":"        - devstack"}],"source_content_type":"text/x-yaml","patch_set":12,"id":"6af6c214_171144fb","line":917,"range":{"start_line":914,"start_character":0,"end_line":917,"end_character":59},"updated":"2022-05-05 08:56:42.000000000","message":"ok, finally I got to this :-)\nso these jobs are green, even with ovs-agent","commit_id":"e1b7821de12ce63ff7afa80aec02cad62546621d"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"9843af409fd1a2411c0ac1046de66766221bffdf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"223759f1_34fb7152","updated":"2022-04-27 10:47:39.000000000","message":"recheck","commit_id":"8de0191b932ba15e1780b2d74812cd85aafc5381"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"8598cb7e31d54f9d109e0d3431b9225e36afebdd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"6320766f_5b41f1e4","updated":"2022-05-04 13:24:11.000000000","message":"mirror issue:- libsystemd-dev : Depends: libsystemd0 (\u003d 249.11-0ubuntu3) but 249.11-0ubuntu3.1 is to be installed\n\nhigher version of dependent package already installed during image build.","commit_id":"e1b7821de12ce63ff7afa80aec02cad62546621d"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"8d423f6d62ade5115867b1c08768385bcd736a24","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"c1622eb5_8d70ee27","updated":"2022-05-05 03:52:04.000000000","message":"recheck again","commit_id":"e1b7821de12ce63ff7afa80aec02cad62546621d"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"8b6cbc907ea7f38613654feb4d584a67fa171b13","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"f21e2d22_80368a57","updated":"2022-05-04 18:55:49.000000000","message":"recheck got new image","commit_id":"e1b7821de12ce63ff7afa80aec02cad62546621d"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"6397ee8bb67a496ba0da794baec8c31bacb92749","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"965afd5d_0b4c35e3","updated":"2022-05-04 14:31:14.000000000","message":"recheck issue seems related to phased apt updates, holding a node for debugging","commit_id":"e1b7821de12ce63ff7afa80aec02cad62546621d"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"0c9e6be010a9d75f291a58eaaafa9ce9b9823060","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"5a559e75_0d4b6aac","updated":"2022-05-04 13:54:22.000000000","message":"recheck there is a known issue with reprepro, but the systemd update was mirrored yesterday already, so not sure what\u0027s happening, let\u0027s gather more data","commit_id":"e1b7821de12ce63ff7afa80aec02cad62546621d"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"a4a037005ba76329f8c8f3a70116c00311742ea6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"9eaef15f_e8d76cfb","updated":"2022-05-05 05:43:04.000000000","message":"recheck volume tests failure","commit_id":"e1b7821de12ce63ff7afa80aec02cad62546621d"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"0773e1fb09c48028d0f1115b547e0f4c708d93bf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":14,"id":"0e90324b_91b0ebc1","updated":"2022-05-09 15:17:54.000000000","message":"recheck\ndevstack-platform-ubuntu-jammy-ovn failed with \"No space left on device\", tests are passed","commit_id":"f2d4d425c3503e2a526ff09774956a0937ba355a"},{"author":{"_account_id":8313,"name":"Lajos Katona","display_name":"lajoskatona","email":"katonalala@gmail.com","username":"elajkat","status":"Ericsson Software Technology"},"change_message_id":"b8cab756b01cf453fd7d80daecfb3f5c007ff319","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":14,"id":"1af485b7_26c38235","updated":"2022-05-09 07:28:18.000000000","message":"recheck\ntest_image_web_download_import failed for devstack-platform-ubuntu-jammy-ovn-source","commit_id":"f2d4d425c3503e2a526ff09774956a0937ba355a"}],"lib/neutron_plugins/ovn_agent":[{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"b5d554477db20c156a85e1526439cd3f179d91c0","unresolved":true,"context_lines":[{"line_number":704,"context_line":"            sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem"},{"line_number":705,"context_line":"            sudo ovn-sbctl --db\u003dunix:$OVS_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem"},{"line_number":706,"context_line":"        fi"},{"line_number":707,"context_line":"        sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock -v set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe\u003d60000"},{"line_number":708,"context_line":"        sudo ovn-sbctl --db\u003dunix:$OVS_RUNDIR/ovnsb_db.sock -v set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe\u003d60000"},{"line_number":709,"context_line":"        sudo ss -lntup | grep \u0027:664\u0027"},{"line_number":710,"context_line":"        sleep 10"},{"line_number":711,"context_line":"        sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock -v get-connection"}],"source_content_type":"application/x-shellscript","patch_set":11,"id":"93f1db0a_f5da79c4","line":708,"range":{"start_line":707,"start_character":0,"end_line":708,"end_character":162},"updated":"2022-05-04 11:15:27.000000000","message":"so what i see is it\u0027s working fine and both nb and sb tcp port are open, but then just after 1 second, db seems to be resetting, didn\u0027t got why it\u0027s happening, adding some sleep before it seems to be helping, will try to wait for db before moving further.\n\nIssue is reproducable locally too. but when i ran set-connection it works fine after the failure.","commit_id":"1885b8621a26c66d6b50655fbed7a8180b06d91e"},{"author":{"_account_id":13252,"name":"Dr. Jens Harbott","display_name":"Jens Harbott (frickler)","email":"frickler@offenerstapel.de","username":"jrosenboom"},"change_message_id":"0c9e6be010a9d75f291a58eaaafa9ce9b9823060","unresolved":true,"context_lines":[{"line_number":704,"context_line":"            sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem"},{"line_number":705,"context_line":"            sudo ovn-sbctl --db\u003dunix:$OVS_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem"},{"line_number":706,"context_line":"        fi"},{"line_number":707,"context_line":"        sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock -v set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe\u003d60000"},{"line_number":708,"context_line":"        sudo ovn-sbctl --db\u003dunix:$OVS_RUNDIR/ovnsb_db.sock -v set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe\u003d60000"},{"line_number":709,"context_line":"        sudo ss -lntup | grep \u0027:664\u0027"},{"line_number":710,"context_line":"        sleep 10"},{"line_number":711,"context_line":"        sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock -v get-connection"}],"source_content_type":"application/x-shellscript","patch_set":11,"id":"9c858f59_26f9518b","line":708,"range":{"start_line":707,"start_character":0,"end_line":708,"end_character":162},"in_reply_to":"93f1db0a_f5da79c4","updated":"2022-05-04 13:54:22.000000000","message":"Ah, I\u0027ve seen this before, seems to be a race condition. What I think what happens is:\n\na) The _start_process L696 triggers a systemctl restart, that command returns immediately before the service is even stopped\nb) the wait_for calls pass while the service is still running and about to be stopped\nc) the ovn-*ctl command runs when the service is stopped, resulting in failure","commit_id":"1885b8621a26c66d6b50655fbed7a8180b06d91e"},{"author":{"_account_id":13861,"name":"yatin","email":"ykarel@redhat.com","username":"yatinkarel"},"change_message_id":"dc36c62880a4058c272ce70ccb72da8de813ceea","unresolved":true,"context_lines":[{"line_number":704,"context_line":"            sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem"},{"line_number":705,"context_line":"            sudo ovn-sbctl --db\u003dunix:$OVS_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem"},{"line_number":706,"context_line":"        fi"},{"line_number":707,"context_line":"        sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock -v set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe\u003d60000"},{"line_number":708,"context_line":"        sudo ovn-sbctl --db\u003dunix:$OVS_RUNDIR/ovnsb_db.sock -v set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe\u003d60000"},{"line_number":709,"context_line":"        sudo ss -lntup | grep \u0027:664\u0027"},{"line_number":710,"context_line":"        sleep 10"},{"line_number":711,"context_line":"        sudo ovn-nbctl --db\u003dunix:$OVS_RUNDIR/ovnnb_db.sock -v get-connection"}],"source_content_type":"application/x-shellscript","patch_set":11,"id":"d8813b14_2a679b31","line":708,"range":{"start_line":707,"start_character":0,"end_line":708,"end_character":162},"in_reply_to":"9c858f59_26f9518b","updated":"2022-05-06 11:50:50.000000000","message":"@frickler so i checked you are mostly right in this but there is one additional thing.\n\nb) Wait for calls, checks for 5 seconds and it passes, as during restart .sock files are not available for just some time\nc) ovn-ctl commands passes even if db files are absent, just need socket files to be present. so until db files are available any updates done via ovn-*ctl command are temporary and lost with restart., and due to this we seeing issue later during neutron-api.\n\nDuring init_ovn(before service restart) we drop the dbs, and then during restart sockets are recreated, and we wait for those, those become available too quickly but dbs comparetively takes more time.\n\nSo i think it makes sense to check for dbs first before actually running ovn-*ctl commands as without it we may see issues like we seeing currently in jammy. I will push a independent patch to do this, and then you can rebase your jammy support patch https://review.opendev.org/c/openstack/devstack/+/839359 over it. wdyt?","commit_id":"1885b8621a26c66d6b50655fbed7a8180b06d91e"}]}