)]}'
{"cyborg/agent/manager.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"b4abf22f398c1192a587eb17f975f00bd7f700d4","unresolved":false,"context_lines":[{"line_number":136,"context_line":"        drv \u003d self._rt.get_driver_for_type(device.type)"},{"line_number":137,"context_line":"        drv.cleanup(device)"},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"    def _reconcile_device_states(self):"},{"line_number":140,"context_line":"        context \u003d cyborg_context.get_admin_context()"},{"line_number":141,"context_line":"        devices \u003d objects.Device.get_list_by_hostname("},{"line_number":142,"context_line":"            context,"}],"source_content_type":"text/x-python","patch_set":5,"id":"2f27c529_8ce636cb","line":139,"updated":"2026-08-06 17:16:53.000000000","message":"The new _reconcile_device_states method performs multiple DB queries (Device.get_list_by_hostname, Deployable.get_list_by_device_id, AttachHandle.get_ah_list_by_deployable_id) and save operations without any try/except. If any DB operation fails, the exception propagates through init_host() and p...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: A transient DB connectivity issue during agent startup causes the agent to fail to start entirely, with no self-healing. In production, this means the compute node\u0027s accelerator management is offline until manual intervention.\n\n**Suggestion**:\nWrap the body of _reconcile_device_states in a try/except that logs the exception and allows init_host to complete, so the agent can start and retry reconciliation later. Follow the pattern used in conductor/manager.py init_host.","commit_id":"489f005f20718c5aefe514802d25c711b39cb55f"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"a279a661ed4541a8a59e4348d387a41a55127a19","unresolved":false,"context_lines":[{"line_number":194,"context_line":"                        device.uuid,"},{"line_number":195,"context_line":"                    )"},{"line_number":196,"context_line":"                    if device.supports_cleaning:"},{"line_number":197,"context_line":"                        self.cleanup_device(context, device)"},{"line_number":198,"context_line":"                    else:"},{"line_number":199,"context_line":"                        device.device_state \u003d constants.DEVICE_STATE_AVAILABLE"},{"line_number":200,"context_line":"                        device.save(context)"}],"source_content_type":"text/x-python","patch_set":7,"id":"b2f4843a_917c4ec2","line":197,"updated":"2026-08-07 16:56:31.000000000","message":"During init_host(), _do_reconcile_device_states calls self.cleanup_device(context, device) inline for each device in ALLOCATED state with no active ARQs. cleanup_device submits to a thread pool but then calls future.result(timeout\u003dCONF.nvme.cleanup_timeout), blocking for up to the cleanup timeout...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: Agent startup could be blocked for minutes if multiple devices need cleanup or if cleanup operations are slow. This delays the agent from accepting RPC calls, potentially causing timeouts in conductor communication.\n\n**Suggestion**:\nSubmit cleanup_device asynchronously (fire-and-forget) during reconciliation, or process cleanup in the background after init_host completes. If synchronous cleanup is intentional, add a LOG.warning before each device cleanup so operators can diagnose slow startup.","commit_id":"d32c1d46e96368254925b7b78b204a15747229a4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"400236530b0ba26cd12a4a88bacab617f0abc9af","unresolved":false,"context_lines":[{"line_number":195,"context_line":"                        device.uuid,"},{"line_number":196,"context_line":"                    )"},{"line_number":197,"context_line":"                    if device.supports_cleaning:"},{"line_number":198,"context_line":"                        self.cleanup_device(context, device)"},{"line_number":199,"context_line":"                    else:"},{"line_number":200,"context_line":"                        device.device_state \u003d constants.DEVICE_STATE_AVAILABLE"},{"line_number":201,"context_line":"                        device.save(context)"}],"source_content_type":"text/x-python","patch_set":8,"id":"1f78239f_37c6373f","line":198,"updated":"2026-08-09 05:46:24.000000000","message":"_do_reconcile_device_states calls self.cleanup_device(context, device) for each ALLOCATED device with no active ARQs that supports cleaning. cleanup_device submits work to a thread executor and then blocks on future.result(timeout\u003dCONF.nvme.cleanup_timeout). NVMe cleanup operations (sanitize, wri...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: After an agent crash with multiple NVMe devices left in ALLOCATED state, the agent restart could be blocked for a long time (cumulative cleanup_timeout across all devices), delaying service availability.\n\n**Suggestion**:\nConsider submitting cleanup_device tasks asynchronously for all affected devices and collecting results, or document the expected startup delay in the release note so operators can set expectations. Alternatively, set device state to CLEANING and submit cleanup without blocking, relying on the next restart to verify completion.","commit_id":"aa365ba4e164ea5ba2cde6123bd0b29840c31270"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1548399c5773e0b159af5038a88af275c2793227","unresolved":false,"context_lines":[{"line_number":169,"context_line":"            context,"},{"line_number":170,"context_line":"            self.resource_provider_name,"},{"line_number":171,"context_line":"        )"},{"line_number":172,"context_line":"        for device in devices:"},{"line_number":173,"context_line":"            if device.device_state in ("},{"line_number":174,"context_line":"                constants.DEVICE_STATE_CLEANING,"},{"line_number":175,"context_line":"                constants.DEVICE_STATE_PENDING_CLEANING,"}],"source_content_type":"text/x-python","patch_set":9,"id":"e896994b_8aa43f89","line":172,"updated":"2026-08-11 10:06:05.000000000","message":"The `_do_reconcile_device_states` loop iterates over all devices but has no per-device try/except. If processing any device raises (e.g., device.save DB error, query failure), the exception propagates to the outer `except Exception` in `_reconcile_device_states`, which logs and swallows it. All r...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: If a transient DB error or unexpected exception occurs while processing device 2 of 10, devices 3-10 remain in stale states (CLEANING, ALLOCATED) until the next full agent restart. For devices stuck in ALLOCATED, this means they cannot accept new binds and are effectively stranded.\n\n**Suggestion**:\nWrap the per-device processing body inside the for-loop with a try/except so that one device failure logs an error and continues to the next device, rather than aborting the entire reconciliation pass.","commit_id":"80ebe043e9f22e5e91805f3e8ab8e0f892de066e"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"e3d961254ccfea871042d060af7c6b2e15ceecb1","unresolved":false,"context_lines":[{"line_number":154,"context_line":"                device.uuid,"},{"line_number":155,"context_line":"            )"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    def _reconcile_device_states(self):"},{"line_number":158,"context_line":"        try:"},{"line_number":159,"context_line":"            self._do_reconcile_device_states()"},{"line_number":160,"context_line":"        except Exception:"}],"source_content_type":"text/x-python","patch_set":10,"id":"e08874fb_1f7852d9","line":157,"updated":"2026-08-11 16:14:09.000000000","message":"The _reconcile_device_states() method wraps _do_reconcile_device_states() in a try/except that catches all exceptions and logs them, allowing the agent to continue starting. No test verifies this behavior — if _do_reconcile_device_states raises, the catch-and-log path is unexercised.\n\n**Severity**: SUGGESTION | **Confidence**: 0.8\n\n**Benefit**: If the exception handling in _reconcile_device_states is broken or removed accidentally, no test will catch it. The agent could crash on startup when reconciliation fails, defeating the purpose of the graceful degradation design.\n\n**Recommendation**:\nAdd a test that patches _do_reconcile_device_states to raise a known exception, calls _reconcile_device_states, and asserts no exception propagates and the expected log message is emitted.","commit_id":"9f298f81dea27546ba8d442cba1d6cf343f4fbc2"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":227,"context_line":"                        device.uuid,"},{"line_number":228,"context_line":"                    )"},{"line_number":229,"context_line":"                    if device.supports_cleaning:"},{"line_number":230,"context_line":"                        device.device_state \u003d ("},{"line_number":231,"context_line":"                            constants.DEVICE_STATE_PENDING_CLEANING"},{"line_number":232,"context_line":"                        )"},{"line_number":233,"context_line":"                        device.save(context)"}],"source_content_type":"text/x-python","patch_set":14,"id":"6feefc09_8423a18f","line":230,"updated":"2026-08-13 14:52:40.000000000","message":"In _do_reconcile_device_states, when an ALLOCATED device with no ARQs supports cleaning, the code sets device_state to PENDING_CLEANING (line 230), saves (line 233), then calls cleanup_device (line 234). cleanup_device immediately sets device_state to CLEANING (line 135) and saves again, making t...\n\n**Severity**: SUGGESTION | **Confidence**: 0.9\n\n**Benefit**: The extra DB save is wasteful and the PENDING_CLEANING state is never observable by any consumer. It also makes the code harder to reason about because it implies a meaningful intermediate state.\n\n**Recommendation**:\nRemove the PENDING_CLEANING intermediate save since cleanup_device will immediately overwrite it to CLEANING, or adjust cleanup_device to accept the device in PENDING_CLEANING without re-saving.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":110,"context_line":"        self._cleanup_executor.shutdown(wait\u003dTrue)"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"    def init_host(self):"},{"line_number":113,"context_line":"        \"\"\"Hook called by RPCService.start() after the RPC server is up.\"\"\""},{"line_number":114,"context_line":"        try:"},{"line_number":115,"context_line":"            while True:"},{"line_number":116,"context_line":"                found, done \u003d data_migrations.backfill_device_state()"}],"source_content_type":"text/x-python","patch_set":16,"id":"dab5d221_70214a8a","line":113,"updated":"2026-08-15 17:10:38.000000000","message":"The loop repeats while found is truthy, but the only exit besides found\u003d\u003d0 relies on backfill_device_state making progress. If device_update fails for one device (or a device row is re-selected without its state changing), each batch returns found\u003e0 with done\u003d\u003d0 and init_host spins forever, block...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A single un-updatable device row (DB constraint, stale replica read) prevents cyborg-agent from ever completing init_host, taking the compute agent down with no timeout.\n\n**Priority**: Before merge\n**Why This Matters**: A single un-updatable device row (DB constraint, stale replica read) prevents cyborg-agent from ever completing init_host, taking the compute agent down with no timeout.\n\n**Recommendation**:\nBreak out of the loop when done \u003d\u003d 0 as well (log an error directing the operator to cyborg-dbsync), matching the progress-check pattern used by the conductor loop.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":223,"context_line":"            elif device.device_state \u003d\u003d constants.DEVICE_STATE_ALLOCATED:"},{"line_number":224,"context_line":"                if not self._has_active_arqs(context, device):"},{"line_number":225,"context_line":"                    LOG.warning("},{"line_number":226,"context_line":"                        \u0027Device %s allocated with no ARQs, triggering cleanup\u0027,"},{"line_number":227,"context_line":"                        device.uuid,"},{"line_number":228,"context_line":"                    )"},{"line_number":229,"context_line":"                    if device.supports_cleaning:"}],"source_content_type":"text/x-python","patch_set":16,"id":"d3012f51_ee85a4d3","line":226,"updated":"2026-08-15 17:10:38.000000000","message":"_do_reconcile_device_states() treats every ALLOCATED device with no in-use attach handles as needing cleanup and calls cleanup_device() for NVMe devices, which submits a sanitize/write-zeroes/destroy operation via the driver. The only evidence is the DB rows present at agent restart, with no oper...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: Any condition that leaves device_state\u003dallocated with attach handles marked not in_use while data is still resident (partial unbind, failed deallocate, conductor crash between ARQ unbind and AH update, or a brand-new NVMe device imported as allocated) results in an automatic destructive sanitize/...\n\n**Priority**: Before merge\n**Why This Matters**: Any condition that leaves device_state\u003dallocated with attach handles marked not in_use while data is still resident (partial unbind, failed deallocate, conductor crash between ARQ unbind and AH update, or a brand-new NVMe device imported as allocated) results in an automatic destructive sanitize/...\n\n**Recommendation**:\nDo not auto-run destructive cleanup at startup from DB inference alone: move such devices to PENDING_CLEANING/error and surface them for operator action (or require an opt-in config like [nvme] auto_cleanup_on_startup), and/or verify device contents/namespace state before invoking drv.cleanup(). Distinguish NVMe-owned pool devices from operator data by owner/trait checks.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"}],"cyborg/agent/resource_tracker.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1548399c5773e0b159af5038a88af275c2793227","unresolved":false,"context_lines":[{"line_number":89,"context_line":"                    )"},{"line_number":90,"context_line":"                seen[addr] \u003d name"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"    TYPE_TO_DRIVER \u003d {"},{"line_number":93,"context_line":"        constants.DEVICE_NVME: \u0027nvme_driver\u0027,"},{"line_number":94,"context_line":"    }"},{"line_number":95,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9b156410_70ab9a73","line":92,"updated":"2026-08-11 10:06:05.000000000","message":"The new `get_driver_for_type()` method on ResourceTracker has branching logic: it looks up a driver name in TYPE_TO_DRIVER, checks whether that driver exists in acc_drivers, returns it on success, or raises InvalidDriver on two separate failure paths. None of these branches are exercised by unit...\n\n**Severity**: SUGGESTION | **Confidence**: 0.8\n\n**Benefit**: Future changes to the driver lookup logic (e.g., adding a new device type to TYPE_TO_DRIVER) have no test coverage to catch regressions in the mapping or error paths.\n\n**Recommendation**:\nAdd unit tests covering: (1) successful lookup for a registered type, (2) InvalidDriver raised for an unregistered device type, (3) InvalidDriver raised when the mapped driver name is not present in acc_drivers.","commit_id":"80ebe043e9f22e5e91805f3e8ab8e0f892de066e"}],"cyborg/common/exception.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"400236530b0ba26cd12a4a88bacab617f0abc9af","unresolved":false,"context_lines":[{"line_number":449,"context_line":"    _msg_fmt \u003d _(\"Device cleanup failed: %(reason)s.\")"},{"line_number":450,"context_line":""},{"line_number":451,"context_line":""},{"line_number":452,"context_line":"class DeviceNotAvailable(CyborgException):"},{"line_number":453,"context_line":"    _msg_fmt \u003d _(\"Device %(device)s is not available (state\u003d%(state)s).\")"},{"line_number":454,"context_line":""},{"line_number":455,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"cbde4799_188de315","line":452,"updated":"2026-08-09 05:46:24.000000000","message":"The commit message explicitly states that raising DeviceNotAvailable gives \u0027a clear 409 response.\u0027 However, DeviceNotAvailable inherits from CyborgException whose default code is HTTPStatus.INTERNAL_SERVER_ERROR (500). The codebase already provides a Conflict exception with code \u003d HTTPStatus.CONF...\n\n**Severity**: HIGH | **Confidence**: 0.9\n\n**Risk**: API clients (including Nova) will receive HTTP 500 instead of HTTP 409 when binding to an unavailable device. This prevents clients from distinguishing a transient server error from a legitimate resource conflict, and contradicts the documented behavior.\n\n**Priority**: Before merge\n**Why This Matters**: API clients (including Nova) will receive HTTP 500 instead of HTTP 409 when binding to an unavailable device. This prevents clients from distinguishing a transient server error from a legitimate resource conflict, and contradicts the documented behavior.\n\n**Recommendation**:\nEither change the parent class to Conflict (which sets code \u003d HTTPStatus.CONFLICT) or add \u0027code \u003d HTTPStatus.CONFLICT\u0027 to DeviceNotAvailable. Example: class DeviceNotAvailable(Conflict):","commit_id":"aa365ba4e164ea5ba2cde6123bd0b29840c31270"}],"cyborg/objects/ext_arq.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1c42ac4ef4832d77a7af971f82ef2dbe19eb0e7b","unresolved":false,"context_lines":[{"line_number":275,"context_line":"        )"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"    def bind(self, context, deployable):"},{"line_number":278,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":279,"context_line":"        if ("},{"line_number":280,"context_line":"            device.device_state is not None"},{"line_number":281,"context_line":"            and device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE"}],"source_content_type":"text/x-python","patch_set":2,"id":"96af9652_f7d27a99","line":278,"updated":"2026-08-06 11:26:24.000000000","message":"The new bind guard raises DeviceNotAvailable before _allocate_attach_handle() is reached. The only code that transitions the ARQ to BIND_FAILED lives inside _allocate_attach_handle\u0027s except block (line 269). As a result, when the guard fires, the ARQ state is never updated to BIND_FAILED and rema...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: An ARQ whose target device is cleaning or errored stays in BIND_STARTED indefinitely. Nova, which expects a terminal state (Bound or BindFailed), may hang waiting for the ARQ to resolve. The ARQ is not usable until manually fixed or deleted.\n\n**Priority**: Before merge\n**Why This Matters**: An ARQ whose target device is cleaning or errored stays in BIND_STARTED indefinitely. Nova, which expects a terminal state (Bound or BindFailed), may hang waiting for the ARQ to resolve. The ARQ is not usable until manually fixed or deleted.\n\n**Recommendation**:\nBefore raising DeviceNotAvailable, call self.update_check_state(context, constants.ARQ_BIND_FAILED) to transition the ARQ to a terminal failure state, or wrap the guard so the exception is caught by the same error-handling path as _allocate_attach_handle failures.","commit_id":"747be871aa6f1c3b461784ac663dea1dfece0da2"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1c42ac4ef4832d77a7af971f82ef2dbe19eb0e7b","unresolved":false,"context_lines":[{"line_number":285,"context_line":"                state\u003ddevice.device_state,"},{"line_number":286,"context_line":"            )"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"        if device.device_state is not None:"},{"line_number":289,"context_line":"            device.device_state \u003d constants.DEVICE_STATE_ALLOCATED"},{"line_number":290,"context_line":"            device.save(context)"},{"line_number":291,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"5ed48eef_cde39381","line":288,"updated":"2026-08-06 11:26:24.000000000","message":"The new code in bind() transitions the device to ALLOCATED and saves it before calling _allocate_attach_handle(). If _allocate_attach_handle() or update_check_state() subsequently raises, the device remains stuck in ALLOCATED with no rollback path. Future bind attempts are rejected by the new gua...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A failed bind (e.g. attach-handle allocation failure) leaves the device in ALLOCATED state. It becomes unavailable for future binds because the guard rejects non-AVAILABLE states. Recovery requires an agent restart so _reconcile_device_states can detect and fix the orphaned ALLOCATED device.\n\n**Priority**: Before merge\n**Why This Matters**: A failed bind (e.g. attach-handle allocation failure) leaves the device in ALLOCATED state. It becomes unavailable for future binds because the guard rejects non-AVAILABLE states. Recovery requires an agent restart so _reconcile_device_states can detect and fix the orphaned ALLOCATED device.\n\n**Recommendation**:\nWrap the post-guard logic in a try/except that reverts device.device_state to its previous value (e.g. AVAILABLE) and saves before re-raising on failure. Alternatively, set ALLOCATED only after _allocate_attach_handle succeeds.","commit_id":"747be871aa6f1c3b461784ac663dea1dfece0da2"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"06e27c3c94c934e718a9274621bd17b09420246c","unresolved":false,"context_lines":[{"line_number":288,"context_line":""},{"line_number":289,"context_line":"        self._allocate_attach_handle(context, deployable)"},{"line_number":290,"context_line":""},{"line_number":291,"context_line":"        if device.device_state is not None:"},{"line_number":292,"context_line":"            device.device_state \u003d constants.DEVICE_STATE_ALLOCATED"},{"line_number":293,"context_line":"            device.save(context)"},{"line_number":294,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"79ccc1b1_e72d4ae1","line":291,"updated":"2026-08-07 07:37:52.000000000","message":"The bind method sets device.device_state to ALLOCATED after a successful bind (lines 291-293). The bind guard at lines 279-287 rejects any bind where the device is not AVAILABLE. So a second ARQ targeting a different deployable on the same physical device is rejected with DeviceNotAvailable. Cybo...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: Any device with multiple deployables (multi-VF GPUs, multi-slot FPGAs, multi-port accelerators) can only serve one ARQ at a time. Additional ARQs for the same physical device will receive DeviceNotAvailable errors and fail to bind, breaking multi-instance accelerator sharing.\n\n**Priority**: Before merge\n**Why This Matters**: Any device with multiple deployables (multi-VF GPUs, multi-slot FPGAs, multi-port accelerators) can only serve one ARQ at a time. Additional ARQs for the same physical device will receive DeviceNotAvailable errors and fail to bind, breaking multi-instance accelerator sharing.\n\n**Recommendation**:\nEither (a) do not set device to ALLOCATED in bind() — instead rely solely on the cleaning/error guard to reject binds during cleanup, or (b) track the count of active ARQs per device and only set ALLOCATED on first bind, returning to AVAILABLE when the last ARQ unbinds, or (c) move the state tracking to the deployable level rather than the device level.","commit_id":"56a46f8cc76b7172f473bf27edec3d40fc5fc6f6"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":296,"context_line":"                state\u003ddevice.device_state,"},{"line_number":297,"context_line":"            )"},{"line_number":298,"context_line":""},{"line_number":299,"context_line":"        self._allocate_attach_handle(context, deployable)"},{"line_number":300,"context_line":"        placement_reserved \u003d False"},{"line_number":301,"context_line":"        try:"},{"line_number":302,"context_line":"            device.device_state \u003d constants.DEVICE_STATE_ALLOCATED"}],"source_content_type":"text/x-python","patch_set":14,"id":"efb0575d_e7eb7e31","line":299,"updated":"2026-08-13 14:52:40.000000000","message":"The _allocate_attach_handle() call at line 299 sits outside the try/except block (lines 301-316), so if it partially succeeds (AttachHandle.allocate creates the DB row but MDEV creation fails), the exception propagates directly without invoking _rollback_bind(). The allocated attach handle is lea...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: If _allocate_attach_handle partially succeeds (AH allocated in DB, then MDEV step fails), the attach handle remains allocated in the database with no ARQ properly referencing it after the bind failure. Over time this can accumulate orphaned attach handles and cause resource accounting drift.\n\n**Priority**: Before merge\n**Why This Matters**: If _allocate_attach_handle partially succeeds (AH allocated in DB, then MDEV step fails), the attach handle remains allocated in the database with no ARQ properly referencing it after the bind failure. Over time this can accumulate orphaned attach handles and cause resource accounting drift.\n\n**Recommendation**:\nMove the _allocate_attach_handle() call inside the try block so that _rollback_bind() can deallocate the attach handle on failure. Ensure _rollback_bind handles the case where attach_handle_id is set but deployable_id and device state have not yet been modified.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":324,"context_line":"            self.attach_handle_id is not None"},{"line_number":325,"context_line":"        ):"},{"line_number":326,"context_line":"            try:"},{"line_number":327,"context_line":"                self._deallocate_attach_handle("},{"line_number":328,"context_line":"                    context, self.attach_handle_id, self.arq.hostname"},{"line_number":329,"context_line":"                )"},{"line_number":330,"context_line":"            except Exception:"}],"source_content_type":"text/x-python","patch_set":14,"id":"cce7bcc2_6cb94add","line":327,"updated":"2026-08-13 14:52:40.000000000","message":"During bind rollback, _rollback_bind calls _deallocate_attach_handle which internally calls update_check_state(context, ARQ_UNBIND_FAILED) on its own failure (line 397). This transitions the ARQ to UnbindFailed. Then bind()\u0027s except block calls update_check_state(ARQ_BIND_FAILED), but the transfo...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: If attach handle deallocation fails during rollback, the ARQ ends up in UnbindFailed state. The subsequent update_check_state(ARQ_BIND_FAILED) raises ARQBadState, masking the original bind error with a state machine exception and leaving the ARQ in an inconsistent state.\n\n**Priority**: Before merge\n**Why This Matters**: If attach handle deallocation fails during rollback, the ARQ ends up in UnbindFailed state. The subsequent update_check_state(ARQ_BIND_FAILED) raises ARQBadState, masking the original bind error with a state machine exception and leaving the ARQ in an inconsistent state.\n\n**Recommendation**:\nWrap the _deallocate_attach_handle call inside _rollback_bind so that the ARQ_UNBIND_FAILED state transition it triggers does not conflict with the subsequent ARQ_BIND_FAILED transition. Consider having _deallocate_attach_handle skip state transitions when called from a rollback context.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":332,"context_line":"                    \u0027Failed to rollback attach handle for ARQ %s\u0027,"},{"line_number":333,"context_line":"                    self.arq.uuid,"},{"line_number":334,"context_line":"                )"},{"line_number":335,"context_line":"            self.attach_handle_id \u003d None"},{"line_number":336,"context_line":""},{"line_number":337,"context_line":"        if self.obj_attr_is_set(\u0027deployable_id\u0027) and ("},{"line_number":338,"context_line":"            self.deployable_id is not None"}],"source_content_type":"text/x-python","patch_set":14,"id":"413dce10_ee5035c1","line":335,"updated":"2026-08-13 14:52:40.000000000","message":"In _rollback_bind, self.attach_handle_id is set to None at line 335 but is only persisted to the database if the deployable_id block (lines 337-347) executes and saves. If the exception occurred before self.deployable_id was set (line 307), the deployable_id block is skipped and attach_handle_id\u003d...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: After a rollback where deployable_id was not yet set, the ExtARQ\u0027s attach_handle_id in the DB still references the (now deallocated) attach handle. Subsequent unbind or delete operations on this ARQ may attempt to deallocate an already-freed handle or reference stale data.\n\n**Priority**: Before merge\n**Why This Matters**: After a rollback where deployable_id was not yet set, the ExtARQ\u0027s attach_handle_id in the DB still references the (now deallocated) attach handle. Subsequent unbind or delete operations on this ARQ may attempt to deallocate an already-freed handle or reference stale data.\n\n**Recommendation**:\nAfter setting self.attach_handle_id \u003d None at line 335, call self.save(context) to persist the change independently of the deployable_id block.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"185588e997ef9c4f150634af7d46351face4a8bd","unresolved":false,"context_lines":[{"line_number":289,"context_line":""},{"line_number":290,"context_line":"    def bind(self, context, deployable):"},{"line_number":291,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":292,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":293,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":294,"context_line":"            raise exception.DeviceNotAvailable("},{"line_number":295,"context_line":"                device\u003ddevice.uuid,"}],"source_content_type":"text/x-python","patch_set":15,"id":"f50aa4b0_38427faf","line":292,"updated":"2026-08-14 16:24:03.000000000","message":"The guard \u0027device.device_state !\u003d DEVICE_STATE_AVAILABLE\u0027 rejects devices whose device_state is NULL (old rows before the best-effort startup backfill, or newly reported devices before conductor\u0027s periodic upsert backfills AVAILABLE) and also rejects a second bind of a shared deployable (e.g. vGP...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: Upgraded clouds where device_state has not yet been backfilled reject all binds with DeviceNotAvailable. Device profiles allocating more than one ARQ against the same deployable (vGPU) fail on the second bind, breaking Nova requests for multiple vGPUs of one PGPU.\n\n**Priority**: Before merge\n**Why This Matters**: Upgraded clouds where device_state has not yet been backfilled reject all binds with DeviceNotAvailable. Device profiles allocating more than one ARQ against the same deployable (vGPU) fail on the second bind, breaking Nova requests for multiple vGPUs of one PGPU.\n\n**Recommendation**:\nTreat NULL device_state as available, and gate on attach-handle availability rather than device-level state for deployables with num_accelerators \u003e 1 (e.g. only reject cleaning/pending_cleaning/error states), or at minimum document the multi-ARQ limitation in the release note.","commit_id":"a27febdebfc2cfd835b251cd3848f3191655479c"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"185588e997ef9c4f150634af7d46351face4a8bd","unresolved":false,"context_lines":[{"line_number":319,"context_line":"        # if (self.arq.state \u003d\u003d constants.ARQ_DELETING"},{"line_number":320,"context_line":"        #         or self.arq.state \u003d\u003d ARQ_UNBOUND):"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"    def _rollback_bind(self, context, deployable, device, placement_reserved):"},{"line_number":323,"context_line":"        if self.obj_attr_is_set(\u0027attach_handle_id\u0027) and ("},{"line_number":324,"context_line":"            self.attach_handle_id is not None"},{"line_number":325,"context_line":"        ):"}],"source_content_type":"text/x-python","patch_set":15,"id":"2d8fef66_51e6555e","line":322,"updated":"2026-08-14 16:24:03.000000000","message":"bind() calls _rollback_bind inside its except clause intending to always reach update_check_state(context, ARQ_BIND_FAILED). But _rollback_bind invokes _deallocate_attach_handle, which itself calls update_check_state(context, ARQ_UNBIND_FAILED) and re-raises on failure; if that nested state trans...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A bind failure occurring after the ARQ reaches Bound can surface ARQBadState instead of the original error, skipping the BindFailed terminal state Nova expects, and leaving attach handle / device / Placement state inconsistently rolled back.\n\n**Priority**: Before merge\n**Why This Matters**: A bind failure occurring after the ARQ reaches Bound can surface ARQBadState instead of the original error, skipping the BindFailed terminal state Nova expects, and leaving attach handle / device / Placement state inconsistently rolled back.\n\n**Recommendation**:\nDo not call update_check_state(ARQ_UNBIND_FAILED) from the rollback path; use update_state directly or catch exception.ARQBadState there. Also wrap _rollback_bind so bind()\u0027s handler can always reach update_check_state(context, ARQ_BIND_FAILED) before re-raising the original exception.","commit_id":"a27febdebfc2cfd835b251cd3848f3191655479c"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":288,"context_line":"        )"},{"line_number":289,"context_line":""},{"line_number":290,"context_line":"    def bind(self, context, deployable):"},{"line_number":291,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":292,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":293,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":294,"context_line":"            raise exception.DeviceNotAvailable("}],"source_content_type":"text/x-python","patch_set":16,"id":"d1653107_1755a1bc","line":291,"updated":"2026-08-15 17:10:38.000000000","message":"bind() now raises DeviceNotAvailable unless device_state is exactly AVAILABLE, but device_state is set to ALLOCATED after the first successful bind and only returns to AVAILABLE after cleanup. A second ARQ (e.g. another vGPU instance) targeting the same physical device is rejected with a terminal...\n\n**Severity**: HIGH | **Confidence**: 0.9\n\n**Risk**: Any deployment where a device (particularly vGPU deployables with num_accelerators \u003e 1, or multiple ARQs from one instance) needs more than one concurrent attach handle now fails binding with ARQ_BIND_FAILED; Nova receives a terminal bind status. This is a functional regression for vGPU and multi...\n\n**Priority**: Before merge\n**Why This Matters**: Any deployment where a device (particularly vGPU deployables with num_accelerators \u003e 1, or multiple ARQs from one instance) needs more than one concurrent attach handle now fails binding with ARQ_BIND_FAILED; Nova receives a terminal bind status. This is a functional regression for vGPU and multi...\n\n**Recommendation**:\nScope the guard to the states it is meant to protect (reject PENDING_CLEANING/CLEANING/ERROR) while continuing to allow binding when the device is AVAILABLE or ALLOCATED, or track per-deployable free attach_handle/in_use counts instead of a single device-level state. At minimum document/limit the guard to single-tenant devices like NVMe and add a test for a second concurrent bind.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":289,"context_line":""},{"line_number":290,"context_line":"    def bind(self, context, deployable):"},{"line_number":291,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":292,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":293,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":294,"context_line":"            raise exception.DeviceNotAvailable("},{"line_number":295,"context_line":"                device\u003ddevice.uuid,"}],"source_content_type":"text/x-python","patch_set":16,"id":"9856bacb_3a515868","line":292,"updated":"2026-08-15 17:10:38.000000000","message":"On device-state rejection bind() first calls update_check_state(context, ARQ_BIND_FAILED) and then raises DeviceNotAvailable. ARQ_BIND_FAILED\u0027s transform scope is [BIND_STARTED, BOUND]; update_check_state itself raises ARQBadState when the transition is illegal, which would replace the intended D...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: When the guard fires while the ARQ is not in BIND_STARTED/BOUND (e.g. INITIAL/UNBOUND), Nova/conductor receives ARQBadState instead of DeviceNotAvailable, hiding the real reason; the new tests only cover the BIND_STARTED case so this is untested.\n\n**Suggestion**:\nWrap the update_check_state call defensively or set the failure state only when the current state is within the allowed scope, and ensure DeviceNotAvailable is the exception propagated. Add a unit test where the ARQ is in a non-transformable state.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":319,"context_line":"        # if (self.arq.state \u003d\u003d constants.ARQ_DELETING"},{"line_number":320,"context_line":"        #         or self.arq.state \u003d\u003d ARQ_UNBOUND):"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"    def _rollback_bind(self, context, deployable, device, placement_reserved):"},{"line_number":323,"context_line":"        if self.obj_attr_is_set(\u0027attach_handle_id\u0027) and ("},{"line_number":324,"context_line":"            self.attach_handle_id is not None"},{"line_number":325,"context_line":"        ):"}],"source_content_type":"text/x-python","patch_set":16,"id":"eb14db6b_cd4057f6","line":322,"updated":"2026-08-15 17:10:38.000000000","message":"_rollback_bind clears attach_handle_id/deployable_id in memory but self.save() is only called when deployable_id was set; after an attach-handle allocation failure the saved ARQ row can still reference the old attach handle. When self.save() or device.save() themselves fail, the rollback swallows...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A failure during the primary save path (the common trigger for rollback) leaves the device permanently ALLOCATED with Placement inventory reserved; combined with CF-001\u0027s guard, all future binds to that device are rejected until manual DB surgery. A stale attach_handle_id in the ARQ row can also...\n\n**Priority**: Before merge\n**Why This Matters**: A failure during the primary save path (the common trigger for rollback) leaves the device permanently ALLOCATED with Placement inventory reserved; combined with CF-001\u0027s guard, all future binds to that device are rejected until manual DB surgery. A stale attach_handle_id in the ARQ row can also...\n\n**Recommendation**:\nPersist the attach_handle_id reset in all cases (save after clearing both fields), and when device.save() fails during rollback, mark the device ERROR (fail closed) and log an actionable error instead of silently continuing, so the inconsistent state is at least visible and excluded from binding.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"}],"cyborg/tests/unit/agent/test_manager.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"93de7e09f828c593325106444e1f3735463f901f","unresolved":false,"context_lines":[{"line_number":420,"context_line":"        mock_conf.host \u003d \u0027compute-0\u0027"},{"line_number":421,"context_line":"        am \u003d self._create_manager_with_mocks()"},{"line_number":422,"context_line":""},{"line_number":423,"context_line":"        device \u003d mock.Mock(device_state\u003d\u0027cleaning\u0027, uuid\u003d\u0027dev-1\u0027)"},{"line_number":424,"context_line":"        mock_get_devs.return_value \u003d [device]"},{"line_number":425,"context_line":""},{"line_number":426,"context_line":"        am._reconcile_device_states()"}],"source_content_type":"text/x-python","patch_set":11,"id":"9a72172d_7125798f","line":423,"updated":"2026-08-12 09:59:45.000000000","message":"The test relies on Mock\u0027s default truthy return for device.supports_cleaning instead of explicitly setting it to True. This makes the test ambiguous about which code path is exercised (NVMe vs non-NVMe) and fragile if the code later changes to use the value differently.\n\n**Severity**: SUGGESTION | **Confidence**: 0.8\n\n**Benefit**: The test passes accidentally rather than intentionally. A future change that uses supports_cleaning as a non-boolean would break the test in a confusing way. Reviewers cannot easily understand what device type scenario is being tested.\n\n**Recommendation**:\nExplicitly set supports_cleaning\u003dTrue on the mock device (e.g., mock.Mock(device_state\u003d\u0027cleaning\u0027, uuid\u003d\u0027dev-1\u0027, supports_cleaning\u003dTrue)) and rename the test to indicate it covers the NVMe (cleaning-capable) path, e.g. test_reconcile_cleaning_nvme_to_error_reserved.","commit_id":"cc9cb0b00ea1c5abb432862b2b25fa2b9231562e"}],"cyborg/tests/unit/objects/test_extarq.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"06e27c3c94c934e718a9274621bd17b09420246c","unresolved":false,"context_lines":[{"line_number":687,"context_line":"        device \u003d mock.Mock(device_state\u003d\u0027available\u0027, uuid\u003d\u0027dev-uuid\u0027)"},{"line_number":688,"context_line":"        mock_get_dev.return_value \u003d device"},{"line_number":689,"context_line":""},{"line_number":690,"context_line":"        obj_extarq.bind(self.context, dep)"},{"line_number":691,"context_line":""},{"line_number":692,"context_line":"        self.assertEqual(\u0027allocated\u0027, device.device_state)"},{"line_number":693,"context_line":"        device.save.assert_called_once_with(self.context)"}],"source_content_type":"text/x-python","patch_set":6,"id":"8220758a_4dc4c84c","line":690,"updated":"2026-08-07 07:37:52.000000000","message":"The new bind tests verify rejection of cleaning/error devices and acceptance of available/null-state devices, but none test the scenario where a second ARQ attempts to bind to a device that is already ALLOCATED from a prior bind. This is the critical edge case introduced by setting device to ALLO...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: The multi-deployable bind regression described in CF-001 would not be caught by the current test suite, giving false confidence that the bind guard is safe.\n\n**Suggestion**:\nAdd a test that binds two ARQs to different deployables on the same device and verifies the second bind succeeds (or document the intended single-ARQ-per-device behavior explicitly if that is the design intent).","commit_id":"56a46f8cc76b7172f473bf27edec3d40fc5fc6f6"}]}