)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":26285,"name":"wu.chunyang","email":"wchy1001@gmail.com","username":"wu.chunyang"},"change_message_id":"11ac08a507d74794bf3505af9fba96b0bff6e013","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"a8104db7_990ec854","updated":"2026-07-05 15:06:05.000000000","message":"I am not sure this is a correct fix. AFAIK, once network isolation is disabled, the default route is entirely up to operators. They may use the mgmt-network as the default gateway, or use tenant networks instead — even if they don’t configure the mgmt-network at all.\n\nIn CI deployments, I don’t think we need to test network_isolation\u003ddisable anymore. We should remove this config soon to avoid network configuration confusion. What do you think about this?","commit_id":"5c30def872632e122917a852c22e8a6b47485a5f"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"4910e7254c92b93cd1cf322573e4f7e4bb4fde90","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"3b283b1d_f1dc1409","updated":"2026-07-14 04:41:59.000000000","message":"recheck","commit_id":"968f29bd0dfc2eaf7f50b186755d3a845a8fd939"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"f04a0572b1d639666c2a8fc673801e36549600a4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"1d0a85d5_22f81aff","updated":"2026-07-22 05:26:11.000000000","message":"Thanks! Ready to merge.","commit_id":"02efba18ef10ac09aae3438b05cc1ddbdc431054"}],"integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/40-fix-default-route":[{"author":{"_account_id":26285,"name":"wu.chunyang","email":"wchy1001@gmail.com","username":"wu.chunyang"},"change_message_id":"11ac08a507d74794bf3505af9fba96b0bff6e013","unresolved":true,"context_lines":[{"line_number":45,"context_line":"echo \"Management interface: ${mgmt_iface}\""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# Disable interfaces associated with other default routes"},{"line_number":48,"context_line":"for route in \"${default_routes[@]}\"; do"},{"line_number":49,"context_line":"    if [[ \"$route\" \u003d~ ^default\\ .*dev\\ ([^[:space:]]+) ]]; then"},{"line_number":50,"context_line":"        iface\u003d\"${BASH_REMATCH[1]}\""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"        # Skip the interface using TROVE_MGMT_GATEWAY"},{"line_number":53,"context_line":"        if [[ \"$iface\" !\u003d \"$mgmt_iface\" ]]; then"},{"line_number":54,"context_line":"            echo \"Disabling interface ${iface} (extra default route detected)\""},{"line_number":55,"context_line":"            # Disable the interface entirely because we need to get a single"},{"line_number":56,"context_line":"            # default route, which is provided by trove-mgmt network. Also"},{"line_number":57,"context_line":"            # guest-agent in network isolation mode will remove this interface"},{"line_number":58,"context_line":"            # anyway immediately after startup."},{"line_number":59,"context_line":"            # Trove-mgmt interfaice becomes the only default route and would"},{"line_number":60,"context_line":"            # provide an access to the Internet."},{"line_number":61,"context_line":"            ip link set dev \"$iface\" down"},{"line_number":62,"context_line":"        fi"},{"line_number":63,"context_line":"    fi"},{"line_number":64,"context_line":"done"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"},{"line_number":67,"context_line":"    ip route show"}],"source_content_type":"application/x-shellscript","patch_set":3,"id":"2a044639_dcc0e03b","line":64,"range":{"start_line":48,"start_character":0,"end_line":64,"end_character":4},"updated":"2026-07-05 15:06:05.000000000","message":"I’m confused about this part. When you take the interface down, when do you bring it back up? Under network isolation mode, this logic is handled by the docker host-nic plugin.\n\neven if we have to fix this problem, I think using router metrics is more sutiable.  for example:\n\nnetplan set network.ethernets.${iface}.dhcp4-overrides.route-metric\u003d500\nnetplay apply","commit_id":"5c30def872632e122917a852c22e8a6b47485a5f"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"4e0c0914962b9c8dafa46bd5da0e0d7b34eab203","unresolved":false,"context_lines":[{"line_number":45,"context_line":"echo \"Management interface: ${mgmt_iface}\""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# Disable interfaces associated with other default routes"},{"line_number":48,"context_line":"for route in \"${default_routes[@]}\"; do"},{"line_number":49,"context_line":"    if [[ \"$route\" \u003d~ ^default\\ .*dev\\ ([^[:space:]]+) ]]; then"},{"line_number":50,"context_line":"        iface\u003d\"${BASH_REMATCH[1]}\""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"        # Skip the interface using TROVE_MGMT_GATEWAY"},{"line_number":53,"context_line":"        if [[ \"$iface\" !\u003d \"$mgmt_iface\" ]]; then"},{"line_number":54,"context_line":"            echo \"Disabling interface ${iface} (extra default route detected)\""},{"line_number":55,"context_line":"            # Disable the interface entirely because we need to get a single"},{"line_number":56,"context_line":"            # default route, which is provided by trove-mgmt network. Also"},{"line_number":57,"context_line":"            # guest-agent in network isolation mode will remove this interface"},{"line_number":58,"context_line":"            # anyway immediately after startup."},{"line_number":59,"context_line":"            # Trove-mgmt interfaice becomes the only default route and would"},{"line_number":60,"context_line":"            # provide an access to the Internet."},{"line_number":61,"context_line":"            ip link set dev \"$iface\" down"},{"line_number":62,"context_line":"        fi"},{"line_number":63,"context_line":"    fi"},{"line_number":64,"context_line":"done"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"},{"line_number":67,"context_line":"    ip route show"}],"source_content_type":"application/x-shellscript","patch_set":3,"id":"c956d9a8_550c4ed0","line":64,"range":{"start_line":48,"start_character":0,"end_line":64,"end_character":4},"in_reply_to":"2a044639_dcc0e03b","updated":"2026-07-08 06:32:07.000000000","message":"\u003e When you take the interface down, when do you bring it back up?\n\nNever. The client network should be disabled in network isolation mode and later handled by the Docker hostnic driver.\n\n\u003e Under network isolation mode, this logic is handled by the docker host-nic plugin.\n\nClient network interface is disabled [here](https://opendev.org/openstack/trove/src/branch/master/trove/cmd/guest.py#L77) at guest startup, until that moment we have a default gateway conflict, which is addressed by this commit. This problem is causing intermittent ERROR states in test instances and long execution due to DNS resolution failures invoked from `trove-image-loader` and `pip install`.\n\n___\n\nI\u0027ve tested the approach with netplan set/apply in my local system, and it looks promising. I\u0027ll return with a new patchset asap.","commit_id":"5c30def872632e122917a852c22e8a6b47485a5f"},{"author":{"_account_id":26285,"name":"wu.chunyang","email":"wchy1001@gmail.com","username":"wu.chunyang"},"change_message_id":"91449c53ac72b37dc9d177e90c49ed590dcc5ca0","unresolved":false,"context_lines":[{"line_number":45,"context_line":"echo \"Management interface: ${mgmt_iface}\""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# Disable interfaces associated with other default routes"},{"line_number":48,"context_line":"for route in \"${default_routes[@]}\"; do"},{"line_number":49,"context_line":"    if [[ \"$route\" \u003d~ ^default\\ .*dev\\ ([^[:space:]]+) ]]; then"},{"line_number":50,"context_line":"        iface\u003d\"${BASH_REMATCH[1]}\""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"        # Skip the interface using TROVE_MGMT_GATEWAY"},{"line_number":53,"context_line":"        if [[ \"$iface\" !\u003d \"$mgmt_iface\" ]]; then"},{"line_number":54,"context_line":"            echo \"Disabling interface ${iface} (extra default route detected)\""},{"line_number":55,"context_line":"            # Disable the interface entirely because we need to get a single"},{"line_number":56,"context_line":"            # default route, which is provided by trove-mgmt network. Also"},{"line_number":57,"context_line":"            # guest-agent in network isolation mode will remove this interface"},{"line_number":58,"context_line":"            # anyway immediately after startup."},{"line_number":59,"context_line":"            # Trove-mgmt interfaice becomes the only default route and would"},{"line_number":60,"context_line":"            # provide an access to the Internet."},{"line_number":61,"context_line":"            ip link set dev \"$iface\" down"},{"line_number":62,"context_line":"        fi"},{"line_number":63,"context_line":"    fi"},{"line_number":64,"context_line":"done"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"},{"line_number":67,"context_line":"    ip route show"}],"source_content_type":"application/x-shellscript","patch_set":3,"id":"f6955420_82ecc9cb","line":64,"range":{"start_line":48,"start_character":0,"end_line":64,"end_character":4},"in_reply_to":"30783de0_020af282","updated":"2026-07-08 14:09:20.000000000","message":"yes, this works even when the network_isolation is disabled.","commit_id":"5c30def872632e122917a852c22e8a6b47485a5f"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"005dd81268c535c60e4605d79cbb94083f9286df","unresolved":false,"context_lines":[{"line_number":45,"context_line":"echo \"Management interface: ${mgmt_iface}\""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# Disable interfaces associated with other default routes"},{"line_number":48,"context_line":"for route in \"${default_routes[@]}\"; do"},{"line_number":49,"context_line":"    if [[ \"$route\" \u003d~ ^default\\ .*dev\\ ([^[:space:]]+) ]]; then"},{"line_number":50,"context_line":"        iface\u003d\"${BASH_REMATCH[1]}\""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"        # Skip the interface using TROVE_MGMT_GATEWAY"},{"line_number":53,"context_line":"        if [[ \"$iface\" !\u003d \"$mgmt_iface\" ]]; then"},{"line_number":54,"context_line":"            echo \"Disabling interface ${iface} (extra default route detected)\""},{"line_number":55,"context_line":"            # Disable the interface entirely because we need to get a single"},{"line_number":56,"context_line":"            # default route, which is provided by trove-mgmt network. Also"},{"line_number":57,"context_line":"            # guest-agent in network isolation mode will remove this interface"},{"line_number":58,"context_line":"            # anyway immediately after startup."},{"line_number":59,"context_line":"            # Trove-mgmt interfaice becomes the only default route and would"},{"line_number":60,"context_line":"            # provide an access to the Internet."},{"line_number":61,"context_line":"            ip link set dev \"$iface\" down"},{"line_number":62,"context_line":"        fi"},{"line_number":63,"context_line":"    fi"},{"line_number":64,"context_line":"done"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"},{"line_number":67,"context_line":"    ip route show"}],"source_content_type":"application/x-shellscript","patch_set":3,"id":"30783de0_020af282","line":64,"range":{"start_line":48,"start_character":0,"end_line":64,"end_character":4},"in_reply_to":"c956d9a8_550c4ed0","updated":"2026-07-08 09:19:34.000000000","message":"New patchset works for me. `ip ro sh` during startup inside guest:\n```\ndefault via 192.168.254.1 dev ens4 proto dhcp src 192.168.254.228 metric 50 \ndefault via 10.0.0.1 dev ens3 proto dhcp src 10.0.0.14 metric 100 \n8.8.8.8 via 192.168.254.1 dev ens4 proto dhcp src 192.168.254.228 metric 50 \n10.0.0.0/26 dev ens3 proto kernel scope link src 10.0.0.14 metric 100 \n10.0.0.1 dev ens3 proto dhcp scope link src 10.0.0.14 metric 100 \n10.0.0.2 dev ens3 proto dhcp scope link src 10.0.0.14 metric 100 \n127.0.0.53 dev ens3 proto dhcp scope host metric 100 \n169.254.169.254 via 192.168.254.2 dev ens4 proto dhcp src 192.168.254.228 metric 50 \n169.254.169.254 via 10.0.0.2 dev ens3 proto dhcp src 10.0.0.14 metric 100 \n192.168.254.0/24 dev ens4 proto kernel scope link src 192.168.254.228 metric 50 \n192.168.254.1 dev ens4 proto dhcp scope link src 192.168.254.228 metric 50 \n192.168.254.2 dev ens4 proto dhcp scope link src 192.168.254.228 metric 50 \n```\n\nI\u0027m sure this will fix the problem.","commit_id":"5c30def872632e122917a852c22e8a6b47485a5f"},{"author":{"_account_id":26285,"name":"wu.chunyang","email":"wchy1001@gmail.com","username":"wu.chunyang"},"change_message_id":"26242a8b355f3ab7f9c132c2233800c6b769d436","unresolved":false,"context_lines":[{"line_number":45,"context_line":"echo \"Management interface: ${mgmt_iface}\""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# Disable interfaces associated with other default routes"},{"line_number":48,"context_line":"for route in \"${default_routes[@]}\"; do"},{"line_number":49,"context_line":"    if [[ \"$route\" \u003d~ ^default\\ .*dev\\ ([^[:space:]]+) ]]; then"},{"line_number":50,"context_line":"        iface\u003d\"${BASH_REMATCH[1]}\""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"        # Skip the interface using TROVE_MGMT_GATEWAY"},{"line_number":53,"context_line":"        if [[ \"$iface\" !\u003d \"$mgmt_iface\" ]]; then"},{"line_number":54,"context_line":"            echo \"Disabling interface ${iface} (extra default route detected)\""},{"line_number":55,"context_line":"            # Disable the interface entirely because we need to get a single"},{"line_number":56,"context_line":"            # default route, which is provided by trove-mgmt network. Also"},{"line_number":57,"context_line":"            # guest-agent in network isolation mode will remove this interface"},{"line_number":58,"context_line":"            # anyway immediately after startup."},{"line_number":59,"context_line":"            # Trove-mgmt interfaice becomes the only default route and would"},{"line_number":60,"context_line":"            # provide an access to the Internet."},{"line_number":61,"context_line":"            ip link set dev \"$iface\" down"},{"line_number":62,"context_line":"        fi"},{"line_number":63,"context_line":"    fi"},{"line_number":64,"context_line":"done"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"},{"line_number":67,"context_line":"    ip route show"}],"source_content_type":"application/x-shellscript","patch_set":3,"id":"700d6bf0_7fecc48f","line":64,"range":{"start_line":48,"start_character":0,"end_line":64,"end_character":4},"in_reply_to":"d077e83d_e2fa4f18","updated":"2026-07-12 13:23:58.000000000","message":"thanks for the fix.","commit_id":"5c30def872632e122917a852c22e8a6b47485a5f"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"9975baa68274cbb96c42b6e13a3693f4a1f741b9","unresolved":false,"context_lines":[{"line_number":45,"context_line":"echo \"Management interface: ${mgmt_iface}\""},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"# Disable interfaces associated with other default routes"},{"line_number":48,"context_line":"for route in \"${default_routes[@]}\"; do"},{"line_number":49,"context_line":"    if [[ \"$route\" \u003d~ ^default\\ .*dev\\ ([^[:space:]]+) ]]; then"},{"line_number":50,"context_line":"        iface\u003d\"${BASH_REMATCH[1]}\""},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"        # Skip the interface using TROVE_MGMT_GATEWAY"},{"line_number":53,"context_line":"        if [[ \"$iface\" !\u003d \"$mgmt_iface\" ]]; then"},{"line_number":54,"context_line":"            echo \"Disabling interface ${iface} (extra default route detected)\""},{"line_number":55,"context_line":"            # Disable the interface entirely because we need to get a single"},{"line_number":56,"context_line":"            # default route, which is provided by trove-mgmt network. Also"},{"line_number":57,"context_line":"            # guest-agent in network isolation mode will remove this interface"},{"line_number":58,"context_line":"            # anyway immediately after startup."},{"line_number":59,"context_line":"            # Trove-mgmt interfaice becomes the only default route and would"},{"line_number":60,"context_line":"            # provide an access to the Internet."},{"line_number":61,"context_line":"            ip link set dev \"$iface\" down"},{"line_number":62,"context_line":"        fi"},{"line_number":63,"context_line":"    fi"},{"line_number":64,"context_line":"done"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"},{"line_number":67,"context_line":"    ip route show"}],"source_content_type":"application/x-shellscript","patch_set":3,"id":"d077e83d_e2fa4f18","line":64,"range":{"start_line":48,"start_character":0,"end_line":64,"end_character":4},"in_reply_to":"f6955420_82ecc9cb","updated":"2026-07-10 12:49:48.000000000","message":"Nope, the solution with metric doesn\u0027t work, though.\n\n```\nubuntu@trove-tempest-testinstancesslpostgresql-instance-516993632:~$ ip ro sh\ndefault via 192.168.254.1 dev ens4 proto dhcp src 192.168.254.177 metric 50 \ndefault via 10.0.0.1 dev ens3 proto dhcp src 10.0.0.21 metric 100 \n8.8.8.8 via 192.168.254.1 dev ens4 proto dhcp src 192.168.254.177 metric 50 \n10.0.0.0/26 dev ens3 proto kernel scope link src 10.0.0.21 metric 100 \n10.0.0.1 dev ens3 proto dhcp scope link src 10.0.0.21 metric 100 \n10.0.0.2 dev ens3 proto dhcp scope link src 10.0.0.21 metric 100 \n127.0.0.53 dev ens3 proto dhcp scope host metric 100 \n169.254.169.254 via 192.168.254.2 dev ens4 proto dhcp src 192.168.254.177 metric 50 \n169.254.169.254 via 10.0.0.2 dev ens3 proto dhcp src 10.0.0.21 metric 100 \n192.168.254.0/24 dev ens4 proto kernel scope link src 192.168.254.177 metric 50 \n192.168.254.1 dev ens4 proto dhcp scope link src 192.168.254.177 metric 50 \n192.168.254.2 dev ens4 proto dhcp scope link src 192.168.254.177 metric 50 \nubuntu@trove-tempest-testinstancesslpostgresql-instance-516993632:~$ curl https://pypi.org -v\n* Could not resolve host: pypi.org\n* Closing connection\ncurl: (6) Could not resolve host: pypi.org\nubuntu@trove-tempest-testinstancesslpostgresql-instance-516993632:~$ sudo ip link set down ens3\nubuntu@trove-tempest-testinstancesslpostgresql-instance-516993632:~$ curl https://pypi.org -v\n* Host pypi.org:443 was resolved.\n* IPv6: 2a04:4e42::223, 2a04:4e42:400::223, 2a04:4e42:200::223, 2a04:4e42:600::223\n* IPv4: 151.101.0.223, 151.101.64.223, 151.101.128.223, 151.101.192.223\n*   Trying 151.101.0.223:443...\n* Connected to pypi.org (151.101.0.223) port 443\n* ALPN: curl offers h2,http/1.1\n...\n```\n\nRolling back the solution with ip link down","commit_id":"5c30def872632e122917a852c22e8a6b47485a5f"},{"author":{"_account_id":26285,"name":"wu.chunyang","email":"wchy1001@gmail.com","username":"wu.chunyang"},"change_message_id":"91449c53ac72b37dc9d177e90c49ed590dcc5ca0","unresolved":true,"context_lines":[{"line_number":49,"context_line":"# Default metric is 100. Network with lower metric will be prioritized."},{"line_number":50,"context_line":"netplan set \"network.ethernets.${mgmt_iface}.dhcp4-overrides.route-metric\u003d${ROUTE_METRIC}\""},{"line_number":51,"context_line":"# Without this option dhcp will be disabled with the override provided above."},{"line_number":52,"context_line":"netplan set network.ethernets.enp2s0.dhcp4\u003dtrue"},{"line_number":53,"context_line":"netplan apply"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"}],"source_content_type":"application/x-shellscript","patch_set":5,"id":"3e7b907d_e64dbd08","line":52,"range":{"start_line":52,"start_character":30,"end_line":52,"end_character":36},"updated":"2026-07-08 14:09:20.000000000","message":"Is this a typo?","commit_id":"61f737a6f1f1681ddef041fa1e282a34e3ad0187"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"fcaa1c8f9996aa0bf8f010e83f5cac7663080121","unresolved":false,"context_lines":[{"line_number":49,"context_line":"# Default metric is 100. Network with lower metric will be prioritized."},{"line_number":50,"context_line":"netplan set \"network.ethernets.${mgmt_iface}.dhcp4-overrides.route-metric\u003d${ROUTE_METRIC}\""},{"line_number":51,"context_line":"# Without this option dhcp will be disabled with the override provided above."},{"line_number":52,"context_line":"netplan set network.ethernets.enp2s0.dhcp4\u003dtrue"},{"line_number":53,"context_line":"netplan apply"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then"}],"source_content_type":"application/x-shellscript","patch_set":5,"id":"df6fdd1d_25639a48","line":52,"range":{"start_line":52,"start_character":30,"end_line":52,"end_character":36},"in_reply_to":"3e7b907d_e64dbd08","updated":"2026-07-09 15:57:40.000000000","message":"Oops, sorry. Yes, I\u0027m a little bit unfocused after vacation 😄\nFixed.","commit_id":"61f737a6f1f1681ddef041fa1e282a34e3ad0187"}],"integration/scripts/files/elements/guest-agent/post-install.d/31-enable-guest-agent-systemd":[{"author":{"_account_id":26285,"name":"wu.chunyang","email":"wchy1001@gmail.com","username":"wu.chunyang"},"change_message_id":"26242a8b355f3ab7f9c132c2233800c6b769d436","unresolved":true,"context_lines":[{"line_number":11,"context_line":"fi"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"if [[ ${DEV_MODE} \u003d\u003d \"true\" ]]; then"},{"line_number":14,"context_line":"    systemctl enable $(svc-map trove-image-loader)"},{"line_number":15,"context_line":"fi"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"if [[ ${SYNC_LOG_TO_CONTROLLER} \u003d\u003d \"True\" ]]; then"}],"source_content_type":"application/x-shellscript","patch_set":7,"id":"60f1ad64_043fea90","line":14,"range":{"start_line":14,"start_character":31,"end_line":14,"end_character":50},"updated":"2026-07-12 13:23:58.000000000","message":"Hello, why is the trove-image-loader service only available in dev mode now ?","commit_id":"36ddaaefef787a6fee70e304ba1fa06cdbaeaa24"},{"author":{"_account_id":26285,"name":"wu.chunyang","email":"wchy1001@gmail.com","username":"wu.chunyang"},"change_message_id":"2cff014dcd26e786f7dd61e8cbce655ee806396d","unresolved":true,"context_lines":[{"line_number":11,"context_line":"fi"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"if [[ ${DEV_MODE} \u003d\u003d \"true\" ]]; then"},{"line_number":14,"context_line":"    systemctl enable $(svc-map trove-image-loader)"},{"line_number":15,"context_line":"fi"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"if [[ ${SYNC_LOG_TO_CONTROLLER} \u003d\u003d \"True\" ]]; then"}],"source_content_type":"application/x-shellscript","patch_set":7,"id":"695c77ed_584c3213","line":14,"range":{"start_line":14,"start_character":31,"end_line":14,"end_character":50},"in_reply_to":"052c712f_0d785ade","updated":"2026-07-19 07:58:43.000000000","message":"Yes, you are right — the initial purpose of this service is to pre-download the images. It certainly competes with guest-agent for resources, but this is an IO-bound task and should not cost much CPU time. You mentioned \"downloading and unpacking the same required image from the registry in parallel\" — this is actually expected behavior. When guest-agent reaches the image download step, several minutes have typically already passed since boot. Regardless of whether the image-loader service has finished by then, the guest-agent can leverage the cache (partial or complete) that the image-loader has already downloaded. It wouldn\u0027t download the same layers twice because they share the same Docker storage — the Docker daemon handles this deduplication. As for \"This slows down the overall startup process and sometimes produces unreliable test results\" — I haven\u0027t encountered this issue personally, but it\u0027s worth investigating further.\n\nEven if you decide to move this service to dev-only, considering it has already been included in the stable branch, I\u0027d suggest proposing this change in a separate commit with a release note.","commit_id":"36ddaaefef787a6fee70e304ba1fa06cdbaeaa24"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"3beaa0eb26767b47feff997ef6eec1d1235347c2","unresolved":true,"context_lines":[{"line_number":11,"context_line":"fi"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"if [[ ${DEV_MODE} \u003d\u003d \"true\" ]]; then"},{"line_number":14,"context_line":"    systemctl enable $(svc-map trove-image-loader)"},{"line_number":15,"context_line":"fi"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"if [[ ${SYNC_LOG_TO_CONTROLLER} \u003d\u003d \"True\" ]]; then"}],"source_content_type":"application/x-shellscript","patch_set":7,"id":"052c712f_0d785ade","line":14,"range":{"start_line":14,"start_character":31,"end_line":14,"end_character":50},"in_reply_to":"60f1ad64_043fea90","updated":"2026-07-14 11:01:28.000000000","message":"Hi! I don\u0027t see a reason why it should run in production images 🤔\n\nAs far as I understand, it\u0027s intended to \"pre-cache\" the required images for the guest agent, for example, to prevent timeouts during the container startup process. But in practice, it does the opposite: it competes with the guest agent for resources, downloading and unpacking the same required image from the registry in parallel. This slows down the overall startup process and sometimes produces unreliable test results.\n\nSo my view of `trove-image-loader.service` is:\n\n* For **development**, we can run it (and run it sequentially **before** `guest-agent-dev.service`, as I\u0027m proposing in this patch, [here](https://review.opendev.org/c/openstack/trove/+/992900/8/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service#3)).\n  It allows us to see how much time is spent downloading images from the registry and makes it easier to debug future registry-related issues.\n* For **production**, we **definitely don\u0027t** need it.\n  Please correct me if I\u0027m wrong.\n\nAlso, in production, cloud operators should provide the fastest and most reliable container registry possible, so image downloads complete in just a few seconds.","commit_id":"36ddaaefef787a6fee70e304ba1fa06cdbaeaa24"},{"author":{"_account_id":36080,"name":"Erkin Mussurmankulov","display_name":"Eric","email":"erkin.mussurmankulov@ps.kz","username":"mongoose404","status":"PS Cloud services employee"},"change_message_id":"04a553abe811b66f7e864cc2b39e44f12ae456e6","unresolved":true,"context_lines":[{"line_number":11,"context_line":"fi"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"if [[ ${DEV_MODE} \u003d\u003d \"true\" ]]; then"},{"line_number":14,"context_line":"    systemctl enable $(svc-map trove-image-loader)"},{"line_number":15,"context_line":"fi"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"if [[ ${SYNC_LOG_TO_CONTROLLER} \u003d\u003d \"True\" ]]; then"}],"source_content_type":"application/x-shellscript","patch_set":7,"id":"4f3390a1_8e8fa428","line":14,"range":{"start_line":14,"start_character":31,"end_line":14,"end_character":50},"in_reply_to":"695c77ed_584c3213","updated":"2026-07-20 08:40:17.000000000","message":"Ok, let\u0027s do it like that then. I will move the `trove-image-loader` service to DEV only in this MR and create a nested MR for running it sequentially.\n\nHere is an example when tests hit this issue:\n\n[Test results logs](https://zuul.opendev.org/t/openstack/build/b065bff02b604b52aed0f9371432b34f/logs)\n\nTake a look at [trove-guestagent.log](https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_b06/openstack/b065bff02b604b52aed0f9371432b34f/controller/logs/guest-agent-logs/13073d78-f7b5-49b2-bf81-ef31a2512164/log/trove/trove-guestagent_log.txt), and [syslog](https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_b06/openstack/b065bff02b604b52aed0f9371432b34f/controller/logs/guest-agent-logs/13073d78-f7b5-49b2-bf81-ef31a2512164/log/syslog).\n\nThe `trove-image-loader` is still pulling images while the `guest-agent` creates a container, and both are stuck, resulting in a test timeout failure.\n\nThis issue may be observed during busy hours, when Zuul isn\u0027t working fast enough.","commit_id":"36ddaaefef787a6fee70e304ba1fa06cdbaeaa24"}]}