)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"901a00b8ac26e60a3a5b6e0c268ad5f3ed058fb3","unresolved":false,"context_lines":[{"line_number":9,"context_line":"TODO: Clarify if per tenant-provider min-ready handling is the expected"},{"line_number":10,"context_line":"behavior. The alternative would be a global min-ready that has to"},{"line_number":11,"context_line":"consider which providers are available in a tenant so we can at least"},{"line_number":12,"context_line":"provide the number of min-ready nodes in a tenant."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Change-Id: I14d61719da6d4719cd147edeb6b35d7cae4ee93e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"41c17e7e_f09e4b89","line":12,"updated":"2024-09-23 23:46:03.000000000","message":"I think global min-ready is what we want.  It is most close to the current nodepool behavior.  It is also necessary for systems which may have slower or even completely idle tenants.  Opendev is such a system.  In those cases, a per-tenant min-ready would mean permanently idle nodes and lost capacity.\n\nConceptually, we may be able to think about implementing global min-ready like this:\n\n  For each tenant:\n    For each provider:\n      For each label:\n        If min-ready is satisfied by ready or min-ready building nodes on this endpoint: continue\n        else, launch min-ready nodes\n\nThat should produce the desired behavior.  Which provider actually launches the nodes is somewhat arbitrary, but from each provider\u0027s perspective, min-ready is satisfied one way or another.","commit_id":"abfdf0f243e15c041668dba7cf6a83451be4e7a9"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"44d08f1686a60cef9d5cbe8006be0331ad9ff9d3","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"265e988b_b973c556","updated":"2024-10-10 08:25:36.000000000","message":"PS20 resolved a merge conflict","commit_id":"43fd705c946262ba4f308eec2f2dd482bbede141"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"2c1a24a8f57ef8eb4968cd37b2a73e36308f1416","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"eee2f101_0cd032f8","updated":"2024-10-10 10:19:30.000000000","message":"recheck","commit_id":"43fd705c946262ba4f308eec2f2dd482bbede141"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"f169ed71a444a61431bea9282ee1f9a8f6f65781","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":24,"id":"7411805c_e29094f5","updated":"2024-10-14 11:51:28.000000000","message":"recheck","commit_id":"4b6c14227055045f503ab9fc0be08a57582fcde8"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"61cb557d1ce2867149079dc280c75167769fb842","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"47f53918_a06acea1","updated":"2024-10-24 08:16:39.000000000","message":"recheck","commit_id":"e8f0a37e1f7e7932b0489eb1b2d01174e030794d"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"27dd58825ee4e0b5617ea63968f012f45a31914a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":28,"id":"99eb74c5_5e5eaea8","updated":"2024-10-24 10:28:58.000000000","message":"recheck","commit_id":"e8f0a37e1f7e7932b0489eb1b2d01174e030794d"}],"tests/fixtures/zuul-launcher-multi-region.conf":[{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"647e1a5a22619a73dc8daba241a41078279ce1dc","unresolved":true,"context_lines":[{"line_number":44,"context_line":"[connection aws-europe]"},{"line_number":45,"context_line":"driver\u003daws"},{"line_number":46,"context_line":"access_key_id\u003dfake"},{"line_number":47,"context_line":"secret_access_key\u003dfake"}],"source_content_type":"text/plain","patch_set":19,"id":"2dd770ef_8715304f","line":47,"updated":"2024-10-09 20:25:58.000000000","message":"I don\u0027t think it\u0027s relevant for this change, but this stands out since this isn\u0027t how I would expect this to be set up.\n\nNormally I\u0027d expect a single AWS connection with a single credential, and that credential has access to multiple regions.  Each region would be a section with the same connection.\n\nSince this is early work, I\u0027m a little worried about copy-pasta from this and people getting the idea that this is how you have multiple regions.  Is this necessary here, or can we put two sections on the same connection?","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"311d1a9e016c4f17a8e94fdfa46cf47995bc95de","unresolved":true,"context_lines":[{"line_number":44,"context_line":"[connection aws-europe]"},{"line_number":45,"context_line":"driver\u003daws"},{"line_number":46,"context_line":"access_key_id\u003dfake"},{"line_number":47,"context_line":"secret_access_key\u003dfake"}],"source_content_type":"text/plain","patch_set":19,"id":"f8212fde_3176f431","line":47,"in_reply_to":"2dd770ef_8715304f","updated":"2024-10-09 21:57:16.000000000","message":"Take a look at https://review.opendev.org/c/zuul/zuul/+/931994 please.\n\nThat\u0027s what I was imagining, and I think it\u0027s better to use that for tests if we can.  If that\u0027s equivalent, I think we should go with that; if it isn\u0027t, then I may still be missing something in my understanding.","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"1a6460315a499933085938e87130dbadfc74f21b","unresolved":false,"context_lines":[{"line_number":44,"context_line":"[connection aws-europe]"},{"line_number":45,"context_line":"driver\u003daws"},{"line_number":46,"context_line":"access_key_id\u003dfake"},{"line_number":47,"context_line":"secret_access_key\u003dfake"}],"source_content_type":"text/plain","patch_set":19,"id":"863eba38_5176419a","line":47,"in_reply_to":"f8212fde_3176f431","updated":"2024-10-10 09:23:27.000000000","message":"Ack, I don\u0027t think we need a second AWS connection.","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"}],"zuul/launcher/server.py":[{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"647e1a5a22619a73dc8daba241a41078279ce1dc","unresolved":true,"context_lines":[{"line_number":378,"context_line":"                                tags\u003dtags,"},{"line_number":379,"context_line":"                            )"},{"line_number":380,"context_line":"                        ready_nodes[label.name].remove(node)"},{"line_number":381,"context_line":"                        log.debug(\"Assigned min-ready node %s\", node.uuid)"},{"line_number":382,"context_line":"                        break"},{"line_number":383,"context_line":"                    except Exception:"},{"line_number":384,"context_line":"                        log.exception(\"Faild to assign ready node %s\", node)"}],"source_content_type":"text/x-python","patch_set":19,"id":"a09bc1d8_e4765de4","line":381,"updated":"2024-10-09 20:25:58.000000000","message":"I\u0027d say \"assigned ready node\" since we will have ready nodes that are not the result of min-ready (nodes that were created for aborted jobs).","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"1a6460315a499933085938e87130dbadfc74f21b","unresolved":false,"context_lines":[{"line_number":378,"context_line":"                                tags\u003dtags,"},{"line_number":379,"context_line":"                            )"},{"line_number":380,"context_line":"                        ready_nodes[label.name].remove(node)"},{"line_number":381,"context_line":"                        log.debug(\"Assigned min-ready node %s\", node.uuid)"},{"line_number":382,"context_line":"                        break"},{"line_number":383,"context_line":"                    except Exception:"},{"line_number":384,"context_line":"                        log.exception(\"Faild to assign ready node %s\", node)"}],"source_content_type":"text/x-python","patch_set":19,"id":"516e2970_69cd8366","line":381,"in_reply_to":"4c4d3280_4ae5ddba","updated":"2024-10-10 09:23:27.000000000","message":"Ack, that makes sense!","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"311d1a9e016c4f17a8e94fdfa46cf47995bc95de","unresolved":true,"context_lines":[{"line_number":378,"context_line":"                                tags\u003dtags,"},{"line_number":379,"context_line":"                            )"},{"line_number":380,"context_line":"                        ready_nodes[label.name].remove(node)"},{"line_number":381,"context_line":"                        log.debug(\"Assigned min-ready node %s\", node.uuid)"},{"line_number":382,"context_line":"                        break"},{"line_number":383,"context_line":"                    except Exception:"},{"line_number":384,"context_line":"                        log.exception(\"Faild to assign ready node %s\", node)"}],"source_content_type":"text/x-python","patch_set":19,"id":"4c4d3280_4ae5ddba","line":381,"in_reply_to":"a09bc1d8_e4765de4","updated":"2024-10-09 21:57:16.000000000","message":"(I rolled this into my followup change 931994).","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"647e1a5a22619a73dc8daba241a41078279ce1dc","unresolved":true,"context_lines":[{"line_number":513,"context_line":"                    with self.createZKContext(node._lock, self.log) as ctx:"},{"line_number":514,"context_line":"                        node.updateAttributes(ctx, state\u003dstate)"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"            # Mark outdated nodes w/o a request for cleanup"},{"line_number":517,"context_line":"            if not request and not self._hasProvider(node):"},{"line_number":518,"context_line":"                state \u003d node.State.OUTDATED"},{"line_number":519,"context_line":"                log.debug(\"Marking node %s as %s\", node, state)"}],"source_content_type":"text/x-python","patch_set":19,"id":"b9e117de_8c5ea479","line":516,"updated":"2024-10-09 20:25:58.000000000","message":"These are nodes without a request *and* this launcher does not have a provider for it; so the idea is another launcher will handle that, right?\n\nI guess we would need a two-launcher test case to test this, right?  That\u0027s probably worth doing at some point (but probably doesn\u0027t have to happen in this change).","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"78d71bff45ea54b262dc371e7add726940d9e423","unresolved":false,"context_lines":[{"line_number":513,"context_line":"                    with self.createZKContext(node._lock, self.log) as ctx:"},{"line_number":514,"context_line":"                        node.updateAttributes(ctx, state\u003dstate)"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"            # Mark outdated nodes w/o a request for cleanup"},{"line_number":517,"context_line":"            if not request and not self._hasProvider(node):"},{"line_number":518,"context_line":"                state \u003d node.State.OUTDATED"},{"line_number":519,"context_line":"                log.debug(\"Marking node %s as %s\", node, state)"}],"source_content_type":"text/x-python","patch_set":19,"id":"59f80ba1_2fcfb1db","line":516,"in_reply_to":"3d6c3762_2cf7bb4d","updated":"2024-10-15 15:48:57.000000000","message":"Second point fixed in https://review.opendev.org/c/zuul/zuul/+/932179","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"1a6460315a499933085938e87130dbadfc74f21b","unresolved":true,"context_lines":[{"line_number":513,"context_line":"                    with self.createZKContext(node._lock, self.log) as ctx:"},{"line_number":514,"context_line":"                        node.updateAttributes(ctx, state\u003dstate)"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"            # Mark outdated nodes w/o a request for cleanup"},{"line_number":517,"context_line":"            if not request and not self._hasProvider(node):"},{"line_number":518,"context_line":"                state \u003d node.State.OUTDATED"},{"line_number":519,"context_line":"                log.debug(\"Marking node %s as %s\", node, state)"}],"source_content_type":"text/x-python","patch_set":19,"id":"ca8743aa_720c88df","line":516,"in_reply_to":"b9e117de_8c5ea479","updated":"2024-10-10 09:23:27.000000000","message":"We expect all launchers to have a complete view of all providers. So when we can not find a provider, this means that the label config changed and we no longer have a provider that\u0027s using this particular label config.\n\nSince we won\u0027t be able to use nodes with an outdated label config we mark them for cleanup.\n\nHaving said that, I\u0027m wondering if we need to make sure our local provider layouts are up-to-date. Otherwise there might be race-conditions when a launcher already updated to a newer provider, created min-ready nodes and the launcher processing such a min-ready node can\u0027t find a matching provider yet.","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"42b9345bf5ecd9ed2c34b93f9bad6b79809915d7","unresolved":true,"context_lines":[{"line_number":513,"context_line":"                    with self.createZKContext(node._lock, self.log) as ctx:"},{"line_number":514,"context_line":"                        node.updateAttributes(ctx, state\u003dstate)"},{"line_number":515,"context_line":""},{"line_number":516,"context_line":"            # Mark outdated nodes w/o a request for cleanup"},{"line_number":517,"context_line":"            if not request and not self._hasProvider(node):"},{"line_number":518,"context_line":"                state \u003d node.State.OUTDATED"},{"line_number":519,"context_line":"                log.debug(\"Marking node %s as %s\", node, state)"}],"source_content_type":"text/x-python","patch_set":19,"id":"3d6c3762_2cf7bb4d","line":516,"in_reply_to":"ca8743aa_720c88df","updated":"2024-10-15 15:43:09.000000000","message":"So \"OUTDATED\" means \"human must deal with this\"?\n\nYour second point does indeed sound like a hole we should close.","commit_id":"766ee0e849c81be906c689dd2284d6393a4a25ec"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"daadcc60064f7137745ac68987ab3bf3bc6fbfba","unresolved":true,"context_lines":[{"line_number":638,"context_line":"    def _processMinReady(self):"},{"line_number":639,"context_line":"        if not self.api.nodes_cache.waitForSync("},{"line_number":640,"context_line":"                timeout\u003dself.CACHE_SYNC_TIMEOUT):"},{"line_number":641,"context_line":"            self.log.warning(\"Timeout waiting %ss for node cache to sync\")"},{"line_number":642,"context_line":"            return"},{"line_number":643,"context_line":""},{"line_number":644,"context_line":"        for label, provider in self._getMissingMinReadySlots():"}],"source_content_type":"text/x-python","patch_set":27,"id":"b5d72f82_69813c21","line":641,"range":{"start_line":641,"start_character":30,"end_line":641,"end_character":37},"updated":"2024-10-23 23:45:09.000000000","message":"Missing variable argument (self.CACHE_SYNC_TIMEOUT).","commit_id":"87279d4d30ac741e3f6599189132fd15ec050307"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"13503e44ff7106df1a96e5cece5d6cbe5b0146f1","unresolved":false,"context_lines":[{"line_number":638,"context_line":"    def _processMinReady(self):"},{"line_number":639,"context_line":"        if not self.api.nodes_cache.waitForSync("},{"line_number":640,"context_line":"                timeout\u003dself.CACHE_SYNC_TIMEOUT):"},{"line_number":641,"context_line":"            self.log.warning(\"Timeout waiting %ss for node cache to sync\")"},{"line_number":642,"context_line":"            return"},{"line_number":643,"context_line":""},{"line_number":644,"context_line":"        for label, provider in self._getMissingMinReadySlots():"}],"source_content_type":"text/x-python","patch_set":27,"id":"9de93baf_2b68b5a9","line":641,"range":{"start_line":641,"start_character":30,"end_line":641,"end_character":37},"in_reply_to":"b5d72f82_69813c21","updated":"2024-10-24 05:26:51.000000000","message":"Done","commit_id":"87279d4d30ac741e3f6599189132fd15ec050307"}]}