)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":29671,"name":"Albin Vass","email":"opendev@albinvass.com","username":"albin_vass"},"change_message_id":"4953854ea8e63e4e1f6d71a30ab71dd0cc38676f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"18e00131_dc185fa9","updated":"2022-07-14 14:12:00.000000000","message":"And ansible-playbooks runs openpty here:\n\nhttps://github.com/ansible/ansible/blob/devel/lib/ansible/plugins/connection/ssh.py#L908\nhttps://github.com/ansible/ansible/blob/devel/lib/ansible/executor/task_executor.py#L1191","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":29671,"name":"Albin Vass","email":"opendev@albinvass.com","username":"albin_vass"},"change_message_id":"86d18ae8af01339646034a502bf0ba6f14d59b6b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"fb00f5a4_ae8e5a70","updated":"2022-07-14 14:09:40.000000000","message":"Here\u0027s the backtrace we see in gdb for a stuck ansible-playbook process:\n\n\n```\n#0  __lll_lock_wait_private (futex\u003d0x7f068c02cc20 \u003clock\u003e) at ./lowlevellock.c:35\n#1  0x00007f068bf88869 in __GI___nss_database_lookup2 (database\u003ddatabase@entry\u003d0x7f068bff6318 \"group\", alternate_name\u003dalternate_name@entry\u003d0x0, defconfig\u003ddefconfig@entry\u003d0x7f068bff489f \"files\", \n    ni\u003d0x7f068c02cbd8 \u003c__nss_group_database\u003e) at nsswitch.c:122\n#2  0x00007f068bf8a1e4 in __GI___nss_group_lookup2 (ni\u003dni@entry\u003d0x7fff1e591318, fct_name\u003dfct_name@entry\u003d0x7f068bff45b0 \"getgrnam_r\", fct2_name\u003dfct2_name@entry\u003d0x0, fctp\u003dfctp@entry\u003d0x7fff1e591328)\n    at XXX-lookup.c:60\n#3  0x00007f068bf31d8b in __getgrnam_r (name\u003dname@entry\u003d0x7f068bff31dc \"tty\", resbuf\u003dresbuf@entry\u003d0x7fff1e5917d0, buffer\u003dbuffer@entry\u003d0x7fff1e5913a0 \"\\260sċ\\006\\177\", buflen\u003d1024, \n    result\u003dresult@entry\u003d0x7fff1e5917c8) at ../nss/getXXbyYY_r.c:269\n#4  0x00007f068bf9fc17 in grantpt (fd\u003dfd@entry\u003d9) at ../sysdeps/unix/grantpt.c:152\n#5  0x00007f068be3d94e in __GI_openpty (amaster\u003d0x7fff1e593958, aslave\u003d0x7fff1e59395c, name\u003d0x0, termp\u003d0x0, winp\u003d0x0) at openpty.c:103\n```","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"56306ed5812db993f715f068190cb1f6a1022133","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"0b45598f_fe4db293","updated":"2022-07-15 17:50:30.000000000","message":"I think this is worth a try if it makes the Zuul images run Ansible 5 more reliably.","commit_id":"92ffabf218c9e05d447499856e2a877fddef9ab0"},{"author":{"_account_id":29671,"name":"Albin Vass","email":"opendev@albinvass.com","username":"albin_vass"},"change_message_id":"2746382f462e3cf9509c7c24927cc58eb58c8bd1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"2795b76a_212aaaef","updated":"2022-07-15 19:51:02.000000000","message":"recheck","commit_id":"92ffabf218c9e05d447499856e2a877fddef9ab0"},{"author":{"_account_id":7118,"name":"Ian Wienand","email":"iwienand@redhat.com","username":"iwienand"},"change_message_id":"91673c14835a9605f0cf9563506721658317fb9f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"af5aa344_257ae0e9","updated":"2022-07-17 23:43:32.000000000","message":"We have been seeing a similar issue on OpenDev system-config jobs too.  I managed to catch one of the jobs that gets stuck and times out, and inside the executor ansible-playbook run we get a backtrace that goes into grantpt()\n\nhttps://paste.opendev.org/show/bJrkQjbKbUmSx4jGHC7i/\n\nthis strongly suggests the same problems as described in the changelog.  with this additional confirmation, merging this seems prudent.","commit_id":"559602910ff57f90e6478e748d17e0e298430c73"}],"Dockerfile":[{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"231aae19efaaa851e73c18c46599429a037febcc","unresolved":true,"context_lines":[{"line_number":70,"context_line":"COPY --from\u003dbuilder /tmp/openshift-install/oc /usr/local/bin/oc"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"RUN echo \u0027APT::Default-Release \"stable\";\u0027 \u003e\u003e /etc/apt/apt.conf.d/99defaultrelease"},{"line_number":73,"context_line":"RUN echo \"deb     http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e /etc/apt/sources.list.d/testing.list"},{"line_number":74,"context_line":"RUN echo \"deb-src http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e\u003e /etc/apt/sources.list.d/testing.list"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"RUN apt-get update \\"}],"source_content_type":"text/x-dockerfile","patch_set":2,"id":"19d61b77_344cd028","line":73,"range":{"start_line":73,"start_character":54,"end_line":73,"end_character":61},"updated":"2022-07-15 15:45:47.000000000","message":"Small preference to use bookworm here so that we don\u0027t continue to accidentally get testing after the image updates to bookworm. I expect if/when image updates happen we\u0027ll also remove this entirely, but if we miss doing so using the bookworm name will be safer.\n\nAlso, http://deb.debian.org/debian appears to be the url for the mirror these images use. Might want to update to that so that the package lists are in sync?","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":29671,"name":"Albin Vass","email":"opendev@albinvass.com","username":"albin_vass"},"change_message_id":"a109cc54ff6f15253b65e736b1b9213818379a00","unresolved":false,"context_lines":[{"line_number":70,"context_line":"COPY --from\u003dbuilder /tmp/openshift-install/oc /usr/local/bin/oc"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"RUN echo \u0027APT::Default-Release \"stable\";\u0027 \u003e\u003e /etc/apt/apt.conf.d/99defaultrelease"},{"line_number":73,"context_line":"RUN echo \"deb     http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e /etc/apt/sources.list.d/testing.list"},{"line_number":74,"context_line":"RUN echo \"deb-src http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e\u003e /etc/apt/sources.list.d/testing.list"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"RUN apt-get update \\"}],"source_content_type":"text/x-dockerfile","patch_set":2,"id":"9adf9129_3d7fcf11","line":73,"range":{"start_line":73,"start_character":54,"end_line":73,"end_character":61},"in_reply_to":"19d61b77_344cd028","updated":"2022-07-15 16:51:22.000000000","message":"Done","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"231aae19efaaa851e73c18c46599429a037febcc","unresolved":true,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":"RUN echo \u0027APT::Default-Release \"stable\";\u0027 \u003e\u003e /etc/apt/apt.conf.d/99defaultrelease"},{"line_number":73,"context_line":"RUN echo \"deb     http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e /etc/apt/sources.list.d/testing.list"},{"line_number":74,"context_line":"RUN echo \"deb-src http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e\u003e /etc/apt/sources.list.d/testing.list"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"RUN apt-get update \\"},{"line_number":77,"context_line":"  \u0026\u0026 apt-get install -y skopeo \\"}],"source_content_type":"text/x-dockerfile","patch_set":2,"id":"e0a8f82e_c934de9f","line":74,"updated":"2022-07-15 15:45:47.000000000","message":"Nit we shouldn\u0027t need the source repository for this.","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":29671,"name":"Albin Vass","email":"opendev@albinvass.com","username":"albin_vass"},"change_message_id":"a109cc54ff6f15253b65e736b1b9213818379a00","unresolved":false,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":"RUN echo \u0027APT::Default-Release \"stable\";\u0027 \u003e\u003e /etc/apt/apt.conf.d/99defaultrelease"},{"line_number":73,"context_line":"RUN echo \"deb     http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e /etc/apt/sources.list.d/testing.list"},{"line_number":74,"context_line":"RUN echo \"deb-src http://ftp.de.debian.org/debian/    testing main contrib non-free\" \u003e\u003e /etc/apt/sources.list.d/testing.list"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"RUN apt-get update \\"},{"line_number":77,"context_line":"  \u0026\u0026 apt-get install -y skopeo \\"}],"source_content_type":"text/x-dockerfile","patch_set":2,"id":"dee6b690_5c23f486","line":74,"in_reply_to":"e0a8f82e_c934de9f","updated":"2022-07-15 16:51:22.000000000","message":"Done","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"231aae19efaaa851e73c18c46599429a037febcc","unresolved":true,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"RUN apt-get update \\"},{"line_number":77,"context_line":"  \u0026\u0026 apt-get install -y skopeo \\"},{"line_number":78,"context_line":"  \u0026\u0026 apt-get -t testing install -y libc-bin \\"},{"line_number":79,"context_line":"  \u0026\u0026 apt-get clean \\"},{"line_number":80,"context_line":"  \u0026\u0026 rm -rf /var/lib/apt/lists/*"},{"line_number":81,"context_line":""}],"source_content_type":"text/x-dockerfile","patch_set":2,"id":"3ea77cec_e14887f6","line":78,"updated":"2022-07-15 15:45:47.000000000","message":"After each of the earlier package installs (including skopeo just above) we log lines like:\n\n  Processing triggers for libc-bin (2.31-13+deb11u3) ...\n  \nBut when we install libc-bin 2.33-7 that doesn\u0027t seem to happen. I half wonder if we shouldn\u0027t install libc-bin prior to skopeo so that the skopeo install can trigger any of those triggers for newer libc-bin?","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":29671,"name":"Albin Vass","email":"opendev@albinvass.com","username":"albin_vass"},"change_message_id":"a109cc54ff6f15253b65e736b1b9213818379a00","unresolved":false,"context_lines":[{"line_number":75,"context_line":""},{"line_number":76,"context_line":"RUN apt-get update \\"},{"line_number":77,"context_line":"  \u0026\u0026 apt-get install -y skopeo \\"},{"line_number":78,"context_line":"  \u0026\u0026 apt-get -t testing install -y libc-bin \\"},{"line_number":79,"context_line":"  \u0026\u0026 apt-get clean \\"},{"line_number":80,"context_line":"  \u0026\u0026 rm -rf /var/lib/apt/lists/*"},{"line_number":81,"context_line":""}],"source_content_type":"text/x-dockerfile","patch_set":2,"id":"14a3ae2a_9fc85232","line":78,"in_reply_to":"3ea77cec_e14887f6","updated":"2022-07-15 16:51:22.000000000","message":"Done","commit_id":"feca15157d37ad3d30001e146e312157b519ad5d"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"3e6ebd8525d628bcc700de2ea6cfbe2e8acea054","unresolved":true,"context_lines":[{"line_number":73,"context_line":"RUN echo \"deb     http://deb.debian.org/debian/    bookworm main contrib non-free\" \u003e /etc/apt/sources.list.d/testing.list"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"RUN apt-get update \\"},{"line_number":76,"context_line":"  \u0026\u0026 apt-get -t testing install -y libc-bin \\"},{"line_number":77,"context_line":"  \u0026\u0026 apt-get install -y skopeo \\"},{"line_number":78,"context_line":"  \u0026\u0026 apt-get clean \\"},{"line_number":79,"context_line":"  \u0026\u0026 rm -rf /var/lib/apt/lists/*"}],"source_content_type":"text/x-dockerfile","patch_set":3,"id":"89540af4_f507bad3","line":76,"range":{"start_line":76,"start_character":16,"end_line":76,"end_character":23},"updated":"2022-07-15 16:39:17.000000000","message":"I think this \u0027testing\u0027 string needs to be \u0027bookworm\u0027 now to match the apt source list entry.","commit_id":"7efca48fa5f60d1bfbf15d7a029fa32ce474132f"},{"author":{"_account_id":29671,"name":"Albin Vass","email":"opendev@albinvass.com","username":"albin_vass"},"change_message_id":"a109cc54ff6f15253b65e736b1b9213818379a00","unresolved":false,"context_lines":[{"line_number":73,"context_line":"RUN echo \"deb     http://deb.debian.org/debian/    bookworm main contrib non-free\" \u003e /etc/apt/sources.list.d/testing.list"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"RUN apt-get update \\"},{"line_number":76,"context_line":"  \u0026\u0026 apt-get -t testing install -y libc-bin \\"},{"line_number":77,"context_line":"  \u0026\u0026 apt-get install -y skopeo \\"},{"line_number":78,"context_line":"  \u0026\u0026 apt-get clean \\"},{"line_number":79,"context_line":"  \u0026\u0026 rm -rf /var/lib/apt/lists/*"}],"source_content_type":"text/x-dockerfile","patch_set":3,"id":"66abdfbd_f3d254d0","line":76,"range":{"start_line":76,"start_character":16,"end_line":76,"end_character":23},"in_reply_to":"89540af4_f507bad3","updated":"2022-07-15 16:51:22.000000000","message":"Done","commit_id":"7efca48fa5f60d1bfbf15d7a029fa32ce474132f"}]}